예제 #1
0
파일: user.py 프로젝트: klb3713/BYSJ
 def getUserBookTags(self):
     results = self.service.GetUserBookTags(self.api.url)
     if results:
         book_tags, count = results
     else:
         return False
     
     if not book_tags:
         return False
     user = None
     users = User.objects(uid=self.api.api_id)
     if users:
         user = users[0]
     else:
         return False
     
     try:
         user_book_related = None
         user_book_related, created = UserBookRelated.objects.get_or_create(
                 uid=user.uid, auto_save=False,
                 defaults={'uid':user.uid, 'book_tags':book_tags})
         if not created:
             user_book_related.book_tags.extend(book_tags)
         try:
             user_book_related.save()
         except mongoengine.base.ValidationError, error:
             logger.debug("[ERROR]%s: %s when save %s at api:%s" %(error, \
                     error.to_dict(), user_book_related, self.api.url))
     except Exception, error:
         logger.debug("[ERROR]%s: UserBookRelated.objects.get_or_create(uid=%s ..."\
                 %(error, self.api.api_id))
예제 #2
0
파일: user.py 프로젝트: klb3713/BYSJ
 def getUser(self):
     user = self.service.GetPeople(self.api.url)
     if user:
         exist_user = User.objects(uid=user.uid)
         if not exist_user:
             try:
                 user.save()
             except mongoengine.base.ValidationError, error:
                 logger.debug("[ERROR]%s: %s at uid:%s" %(error, error.to_dict(), user.uid))
                 return False
         else:
             return exist_user[0]
예제 #3
0
파일: user.py 프로젝트: klb3713/BYSJ
 def getProfile(self):
     user_api = Api(url='http://api.douban.com/people/'+self.uid,
                api_id=self.uid, api_type='USER')
     user = UserProcessor(user_api, 'default').getUser()
     logger.debug('%s'%(user))
     if user:
         profile = {}
         profile['uid'] = user.uid
         profile['nickname'] = user.nickname
         profile['signature'] = user.signature
         profile['image_link'] = user.image_link
         profile['website'] = user.link
         profile['introduction'] = user.content
         return profile
     else:
         return None
예제 #4
0
파일: book.py 프로젝트: klb3713/BYSJ
 def getBook(self, oldbook=None):
     book = self.service.GetBook(self.api.url)
     if book:
         if oldbook:
             oldbook.summary = book.summary
             oldbook.common_tags = book.common_tags
             oldbook.rating = book.rating
             try:
                 oldbook.save()
                 return oldbook
             except mongoengine.base.ValidationError, error:
                 logger.debug("[ERROR]%s: %s at bid:%s" %(error, error.to_dict(), oldbook.bid))
         
         try:
             book.save()
         except mongoengine.base.ValidationError, error:
             logger.debug("[ERROR]%s: %s at bid:%s" %(error, error.to_dict(), book.bid))
예제 #5
0
파일: user.py 프로젝트: klb3713/BYSJ
 def getUserContacts(self, recursion=False):
     results = self.service.GetContacts(self.api.url)
     if results:
         users, count = results
     else:
         return False
     
     new_users = []
     new_users_data = []
     indexs = []
     for index, user in enumerate(users):
         exist_user = User.objects(uid=user.uid)
         if exist_user:
             users[index] = exist_user[0]
         else:
             new_users.append(user)
             new_user = user.__dict__.get('_data')
             for key, value in new_user.items():
                 if value is None:
                     new_user.pop(key)
             new_user.pop('image')
             new_users_data.append(new_user)
             indexs.append(index)
     if new_users:
         ids = self.db.user.insert(new_users_data)
         for i, index in enumerate(indexs):
             users[index].id = ids[i]
     
     try:
         uid = self.api.api_id
         user_related = None
         user_related, created = UserRelated.objects.get_or_create(
                 uid=uid, auto_save=False,
                 defaults = {'uid':uid, 'contacts':users})
         if not created:
             user_related.contacts.extend(users)
         try:
             user_related.save()
         except mongoengine.base.ValidationError, error:
             logger.debug("[ERROR]%s: %s when save %s at api:%s" \
                     %(error, error.to_dict(), user_related, self.api.url))
     except Exception, error:
         logger.debug("[ERROR]%s: UserRelated.objects.get_or_create(uid=%s ..."\
                 %(error, self.api.api_id))
예제 #6
0
파일: user.py 프로젝트: klb3713/BYSJ
 def getUserFriends(self, recursion=False):
     users = self.service.GetFriends(self.api.url)
     if results:
         users, count = results
     else:
         return False
     
     new_users = []
     for index, user in enumerate(users):
         exist_user = User.objects(uid=user.uid)
         if exist_user:
             users[index] = exist_user[0]
         else:
             new_users.append(user)
             try:
                 user.save()
             except mongoengine.base.ValidationError, error:
                 logger.debug("[ERROR]%s: %s when save uid:%s at api:%s" \
                         %(error, error.to_dict(), user.uid, self.api.url))
예제 #7
0
파일: service.py 프로젝트: klb3713/BYSJ
 def Get(self, uri, converter, extra_headers={}, *args, **kwargs):
     auth_header = self.client.get_auth_header('GET', uri)
     if auth_header:
         extra_headers.update(auth_header)
     elif self.api_key:
         param = 'apikey=' + self.api_key
         if '?' in uri:
             uri += '&' + param
         else:
             uri += '?' + param
     #print 'extra_headers:', extra_headers  #TODO remove!
     request = Request(uri)
     if extra_headers:
         for key, value in extra_headers.items():
             request.add_header(key, value)
     try:
         response = urlopen(request)
         return converter(response.read())
     except URLError, e:
         if hasattr(e, 'reason'):
             logger.debug('[ERROR]Failed to reach a server. Reason: %s' %(e.reason))
         elif hasattr(e, 'code'):
             logger.debug('[ERROR]The server couldn\'t fulfill the request. Error code: %s' %(e.code))
         logger.debug('-------Error Url: ' + uri)
         return None
예제 #8
0
파일: crawl.py 프로젝트: klb3713/BYSJ
def processData(op):
    stop = False
    while(not stop):
        data_thread = DataProcessThread(op)
        data_thread.start()
        time.sleep(1)
        logger.debug("Now at:" + str(DataProcessThread.index))
        if op == 'FIX_BOOK_COLLECTION' and \
        DataProcessThread.index >= DataProcessThread.book_collections_size:
            stop = True
        elif op == 'book_related' and \
        DataProcessThread.index >= DataProcessThread.book_related_size:
            stop = True
        elif op == 'FIX_BOOK_REVIEW' and \
        DataProcessThread.index >= DataProcessThread.book_review_size:
            stop = True
        elif op == 'FIX_NOTE' and \
        DataProcessThread.index >= DataProcessThread.note_size:
            stop = True
        elif op == 'FIX_RECOMMENDATION' and \
        DataProcessThread.index >= DataProcessThread.recommendation_size:
            stop = True
예제 #9
0
파일: book.py 프로젝트: klb3713/BYSJ
 def getBookTags(self):
     results = self.service.GetBookTags(self.api.url)
     if results:
         book_tags, count = results
     else:
         return False
     
     book = None
     books = Book.objects(bid=self.api.api_id)
     if books:
         book = books[0]
     else:
         return False
     
     if not book_tags:
         return False
     else:
         db = MongoDB.getConnection('mining')
         for tag in book_tags:
             db.book_tag.update({'title':tag.title},
                                {'$addToSet':{'books':book.bid},
                                 '$inc':{'count':tag.count}},
                                upsert=True)
     try:
         book_related = None
         book_related, created = BookRelated.objects.get_or_create(\
                 bid=book.bid, auto_save=False, defaults=\
                 {'bid':book.bid, 'book':book, 'tags':book_tags})
         if not created:
             book_related.tags.extend(book_tags)
         try:
             book_related.save()
         except mongoengine.base.ValidationError, error:
             logger.debug("[ERROR]%s: %s when save %s at api:%s" %(error, \
                     error.to_dict(), book_related, self.api.url))
     except Exception, error:
         logger.debug("[ERROR]%s: BookRelated.objects.get_or_create(uid=%s ..."\
                 %(error, self.api.api_id))
예제 #10
0
파일: user.py 프로젝트: klb3713/BYSJ
 def getUserRecommendations(self):
     results = self.service.GetUserRecommendations(self.api.url)
     if results:
         recommendations, count, user = results
     else:
         return False
     
     if not recommendations or not user:
         return False
     recommendation_datas = []
     for recommendation in recommendations:
         recommendation_data = recommendation.__dict__.get('_data')
         for key, value in recommendation_data.items():
             if value is None:
                 recommendation_data.pop(key)
         recommendation_data['author'] = DBRef('user', user.id)
         recommendation_datas.append(recommendation_data)
     
     if recommendation_datas:
         ids = self.db.recommendation.insert(recommendation_datas, check_keys=False)
         for index, object_id in enumerate(ids):
             recommendations[index].id = object_id
     
     try:
         user_related = None
         user_related, created = UserRelated.objects.get_or_create(
                 uid=user.uid, auto_save=False,
                 defaults={'uid':user.uid, 'recommendations':recommendations})
         if not created:
             user_related.recommendations.extend(recommendations)
         try:
             user_related.save()
         except mongoengine.base.ValidationError, error:
             logger.debug("[ERROR]%s: %s when save %s at api:%s" \
                     %(error, error.to_dict(), user_related, self.api.url))
     except Exception, error:
         logger.debug("[ERROR]%s: UserRelated.objects.get_or_create(uid=%s ..."\
                 %(error, self.api.api_id))
예제 #11
0
파일: book.py 프로젝트: klb3713/BYSJ
    def getBookCollections(self):
        results = self.service.GetBookCollections(self.api.url)
        if results:
            collections, count, user = results
        else:
            return False

        if not user or not collections:
            return False
        collection_datas = []
        for collection in collections:
            collection_data = collection.__dict__.get('_data')
            for key, value in collection_data.items():
                if value is None:
                    collection_data.pop(key)
            collection_data['author'] = DBRef('user', user.id)
            if collection.book:
                #添加该书到book_tag集合
                db = MongoDB.getConnection('mining')
                for tag in collection.tags:
                    db.book_tag.update({'title':tag},
                                {'$addToSet':{'books':collection.book.bid},
                                 '$inc':{'count':1}},
                                 upsert=True)
                collection_data['book'] = DBRef('book', collection.book.id)
            collection_datas.append(collection_data)
        
        if collection_datas:
            ids = self.db.book_collection.insert(collection_datas, check_keys=False)
            for index, object_id in enumerate(ids):
                collections[index].id = object_id
            
            try:
                user_book_related = None
                user_book_related, created = UserBookRelated.objects.get_or_create(\
                        uid=self.api.api_id, auto_save=False, defaults=\
                        {'uid':self.api.api_id, 'book_collections':collections})
                if not created:
                    user_book_related.book_collections.extend(collections)
                try:
                    user_book_related.save()
                except mongoengine.base.ValidationError, error:
                    logger.debug("[ERROR]%s: %s when save %s at api:%s" %(error, \
                            error.to_dict(), user_book_related, self.api.url))
            except Exception, error:
                logger.debug("[ERROR]%s: UserBookRelated.objects.get_or_create(uid=%s ..."\
                        %(error, self.api.api_id))
        
            if user_book_related and created:
                try:
                    user.user_book_related = user_book_related
                    user.save()
                except mongoengine.base.ValidationError, error:
                    logger.debug("[ERROR]%s: %s when save %s at api:%s" \
                                 %(error, error.to_dict(), user, self.api.url))
예제 #12
0
파일: book.py 프로젝트: klb3713/BYSJ
 def getBookReviews(self):
     results = self.service.GetBookReviews(self.api.url)
     if results:
         book_reviews, count, book = results
     else:
         return False
     
     if not book or not book_reviews:
         return False
     book_review_datas = []
     indexs = []
     for index, book_review in enumerate(book_reviews):
         exist = BookReview.objects(rid=book_review.rid)
         if exist:
             book_reviews[index] = exist[0]
         else:
             book_review_data = book_review.__dict__.get('_data')
             for key, value in book_review_data.items():
                 if value is None:
                     book_review_data.pop(key)
             if book_review.author:
                 book_review_data['author'] = DBRef('user', book_review.author.id)
             book_review_data['book'] = DBRef('book', book.id)
             book_review_datas.append(book_review_data)
             indexs.append(index)
     
     if book_review_datas:
         ids = self.db.book_review.insert(book_review_datas, check_keys=False)
         for i, index in enumerate(indexs):
             book_reviews[index].id = ids[i]
         
         try:
             book_related = None
             book_related, created = BookRelated.objects.get_or_create(\
                     bid=book.bid, auto_save=False, defaults=\
                     {'bid':book.bid, 'reviews':book_reviews})
             if not created:
                 book_related.reviews.extend(book_reviews)
             try:
                 book_related.save()
             except mongoengine.base.ValidationError, error:
                 logger.debug("[ERROR]%s: %s when save %s at api:%s" %(error, \
                         error.to_dict(), book_related, self.api.url))
         except Exception, error:
             logger.debug("[ERROR]%s: BookRelated.objects.get_or_create(uid=%s ..."\
                     %(error, self.api.api_id))
     
         if book_related and created:
             try:
                 book.book_related = book_related
                 book.save()
             except mongoengine.base.ValidationError, error:
                 logger.debug("[ERROR]%s: %s when save %s at api:%s" \
                              %(error, error.to_dict(), book, self.api.url))
예제 #13
0
파일: book.py 프로젝트: klb3713/BYSJ
                                    '$inc':{'count':tag.count}},
                                   upsert=True)
        try:
            book_related = None
            book_related, created = BookRelated.objects.get_or_create(\
                    bid=book.bid, auto_save=False, defaults=\
                    {'bid':book.bid, 'book':book, 'tags':book_tags})
            if not created:
                book_related.tags.extend(book_tags)
            try:
                book_related.save()
            except mongoengine.base.ValidationError, error:
                logger.debug("[ERROR]%s: %s when save %s at api:%s" %(error, \
                        error.to_dict(), book_related, self.api.url))
        except Exception, error:
            logger.debug("[ERROR]%s: BookRelated.objects.get_or_create(uid=%s ..."\
                    %(error, self.api.api_id))
        
        if book_related and created:
            try:
                book.book_related = book_related
                book.save()
            except mongoengine.base.ValidationError, error:
                logger.debug("[ERROR]%s: %s when save %s at api:%s" 
                             %(error, error.to_dict(), book, self.api.url))
        
        #logger.debug('[%s]Api:%s is processed!' %(datetime.now(), self.api.url))
        return book_tags


예제 #14
0
파일: user.py 프로젝트: klb3713/BYSJ
                    logger.debug("[ERROR]%s: %s when save uid:%s at api:%s" \
                            %(error, error.to_dict(), user.uid, self.api.url))
        

        try:
            uid = self.api.api_id
            user_related = None
            user_related, created = UserRelated.objects.get_or_create(
                    uid=uid, auto_save=False,
                    defaults = {'uid':uid, 'friends':users})
            if not created:
                user_related.friends.extend(users)
            try:
                user_related.save()
            except mongoengine.base.ValidationError, error:
                logger.debug("[ERROR]%s: %s when save %s at api:%s" \
                        %(error, error.to_dict(), user_related, self.api.url))
        except Exception, error:
            logger.debug("[ERROR]%s: UserRelated.objects.get_or_create(uid=%s ..."\
                    %(error, self.api.api_id))
        
        if count:
            start_index = int(count.get('startIndex'))
            total_results = int(count.get('totalResults'))
            items_per_page = int(count.get('itemsPerPage'))
            if start_index + items_per_page < total_results:
                user_contacts_api = self.api_operation.apiGenerator(\
                                    'USER_CONTACTS', self.api.api_id,\
                                    start_index=start_index+items_per_page)
                api = Api(url=user_contacts_api, api_id=self.api.api_id,\
                          api_type='USER_CONTACTS')
                api.save()