def getAllPosts(access_token, user, now, year_ago, important_friends, posts): response = FQL('SELECT gid FROM group_member WHERE uid=me()', access_token) groups = [str(item['gid']) for item in response['data']] logging.debug("Number of groups: " + str(len(groups))) queries = [ 'SELECT post_id, message, created_time, updated_time FROM stream WHERE source_id IN (' + str(important_friends)[5:-2] + ') AND created_time < ' + str(now) + ' ORDER BY created_time DESC LIMIT 100 OFFSET 0', # 'SELECT post_id, message, created_time, updated_time FROM stream WHERE source_id IN (' + str(important_friends)[5:-2] + ') AND created_time < ' + str(now) + ' ORDER BY created_time DESC LIMIT 100 OFFSET 100', # 'SELECT post_id, message, created_time, updated_time FROM stream WHERE source_id IN (' + str(important_friends)[5:-2] + ') AND created_time < ' + str(now) + ' ORDER BY created_time DESC LIMIT 100 OFFSET 200', # 'SELECT post_id, message, created_time, updated_time FROM stream WHERE source_id IN (' + str(important_friends)[5:-2] + ') AND created_time < ' + str(now) + ' ORDER BY created_time DESC LIMIT 100 OFFSET 300', 'SELECT post_id, message, created_time, updated_time FROM stream WHERE source_id IN (' + str(groups)[1:-1] + ') AND created_time < ' + str(now) + ' ORDER BY created_time DESC LIMIT 100 OFFSET 0', # 'SELECT post_id, message, created_time, updated_time FROM stream WHERE source_id IN (' + str(groups)[1:-1] + ') AND created_time < ' + str(now) + ' ORDER BY created_time DESC LIMIT 100 OFFSET 100', # 'g_posts200': 'SELECT post_id, message, created_time, updated_time FROM stream WHERE source_id IN (' + str(groups)[1:-1] + ') AND created_time < ' + str(now) + ' ORDER BY created_time DESC LIMIT 100 OFFSET 200', # 'g_posts300': 'SELECT post_id, message, created_time, updated_time FROM stream WHERE source_id IN (' + str(groups)[1:-1] + ') AND created_time < ' + str(now) + ' ORDER BY created_time DESC LIMIT 100 OFFSET 300', ] response = FQL_batch(access_token, queries) data = flatten([json.loads(answer['body']) for answer in response]) logging.debug("Number of posts: " + str(len(data))) posts_to_put = [] posts.extend([post for post in data if (int(post['updated_time']) > user.last_timestamp and int(post['created_time']) > year_ago)]) posts_to_put.extend([post for post in data if (int(post['created_time']) > user.last_timestamp and int(post['created_time']) > year_ago)]) if len(posts_to_put) == 0: logging.debug('Facebook: no new posts') return sentencePutter = SentencePutter(models.Posts) for post in posts_to_put: sentencePutter.put(post['message'], user.uid, str(post['post_id']), int(post['created_time'])) sentencePutter.flush()
def getAllMessages(access_token, user, now, year_ago, important_friends): queries = { 'threads0': 'SELECT thread_id, updated_time, message_count, recipients FROM thread WHERE folder_id=0 ORDER BY updated_time DESC LIMIT 50 OFFSET 0', # 'threads50': 'SELECT thread_id, updated_time, message_count, recipients FROM thread WHERE folder_id=0 ORDER BY updated_time DESC LIMIT 50 OFFSET 50', # 'threads100': 'SELECT thread_id, updated_time, message_count, recipients FROM thread WHERE folder_id=0 ORDER BY updated_time DESC LIMIT 50 OFFSET 100', # 'threads150': 'SELECT thread_id, updated_time, message_count, recipients FROM thread WHERE folder_id=0 ORDER BY updated_time DESC LIMIT 50 OFFSET 150' } response = FQL_multi(access_token, queries) # Begin Friends tmp = [] for result_set in response['data']: tmp.extend([{'message_count': thread['message_count'], 'recipients': thread['recipients']} for thread in result_set['fql_result_set']]) tmp = [item['recipients'] for item in sorted(tmp, key=lambda item: item['message_count'], reverse=True)[0:10]] for sublist in tmp: important_friends.extend(map(str, sublist)) # End Friends threads_ids = [] for result_set in response['data']: threads_ids.extend([str(thread['thread_id']) for thread in result_set['fql_result_set'] if int(thread['updated_time']) > user.last_timestamp]) if len(threads_ids) == 0: logging.debug('Facebook: no new threads') return threads = dict(zip(threads_ids, [0] * len(threads_ids))) offset = 0 active_threads = threads.keys() sentencePutter = SentencePutter(models.Messages) _counter = 0 while len(active_threads) > 0 and _counter < 4: logging.debug("Active threads: " + str(len(active_threads))) queries = ['SELECT body, message_id, thread_id, created_time FROM message WHERE thread_id IN (' + str(small_list)[1:-1] + ') AND created_time < ' + str(now) + ' ORDER BY created_time DESC LIMIT 6000 OFFSET ' + str(offset) for small_list in bundle(map(str, active_threads), 20)] response = FQL_batch(access_token, queries) messages = flatten([json.loads(answer['body']) for answer in response]) logging.debug("Number of messages: " + str(len(messages))) for msg in messages: threads[str(msg['thread_id'])] += 1 sentencePutter.put(msg['body'], user.uid, str(msg['message_id']), int(msg['created_time'])) if messages[0]['created_time'] < year_ago: break offset += 30 active_threads = [thread_id for thread_id in threads.keys() if threads[thread_id] >= offset] _counter += 1 sentencePutter.flush()
def getAllComments(access_token, user, now, year_ago, posts_ids): logging.debug('Beginning getAllComments') logging.debug("Number of posts with new comments: " + str(len(posts_ids))) response = FQL_batch(access_token, ['SELECT comments FROM stream WHERE post_id IN (' + str(small_list)[1:-1] + ')' for small_list in bundle(posts_ids, 20)]) data = flatten([json.loads(answer['body']) for answer in response]) logging.debug("Number of comments: " + str(len(data))) comments_lists = [comments['comments']['comment_list'] for comments in data] sentencePutter = SentencePutter(models.Comments) counter = 0 comments_lists = sorted(flatten(comments_lists), key=lambda comment: int(comment['time']), reverse=True) for comment in comments_lists: if int(comment['time']) > user.last_timestamp and int(comment['time']) > year_ago: sentencePutter.put(comment['text'], user.uid, str(comment['id']), int(comment['time'])) counter += 1 logging.debug("Number of putted comments: " + str(counter))
def post(self): try: body = json.loads(self.request.body) except: self.response.write('400 invalid json in request body') return sp = SentencePutter(Comments) sp.put(sentence=body['sentence'], uid=body['uid'], fbid=body['fbid'], timestamp=body['timestamp']) sp.put(sentence=body['sentence1'], uid=body['uid'], fbid=body['fbid'], timestamp=body['timestamp']) sp.put(sentence=body['sentence2'], uid=body['uid'], fbid=body['fbid'], timestamp=body['timestamp']) sp.put(sentence=body['sentence'], uid=body['uid'], fbid=body['fbid'], timestamp=body['timestamp']) sp.put(sentence=body['sentence'], uid=body['uid'], fbid=body['fbid'], timestamp=body['timestamp']) sp.put(sentence=body['sentence'], uid=body['uid'], fbid=body['fbid'], timestamp=body['timestamp']) sp.put(sentence=body['sentence'], uid=body['uid'], fbid=body['fbid'], timestamp=body['timestamp']) sp.flush() response = {'status': 'success'} self.response.write(json.dumps(response))