def getAllPosts(access_token, user, now, year_ago, important_friends, posts):
response = FQL('SELECT gid FROM group_member WHERE uid=me()', access_token)
groups = [str(item['gid']) for item in response['data']]
logging.debug("Number of groups: " + str(len(groups)))
queries = [
'SELECT post_id, message, created_time, updated_time FROM stream WHERE source_id IN (' + str(important_friends)[5:-2] + ') AND created_time < ' + str(now) + ' ORDER BY created_time DESC LIMIT 100 OFFSET 0',
# 'SELECT post_id, message, created_time, updated_time FROM stream WHERE source_id IN (' + str(important_friends)[5:-2] + ') AND created_time < ' + str(now) + ' ORDER BY created_time DESC LIMIT 100 OFFSET 100',
# 'SELECT post_id, message, created_time, updated_time FROM stream WHERE source_id IN (' + str(important_friends)[5:-2] + ') AND created_time < ' + str(now) + ' ORDER BY created_time DESC LIMIT 100 OFFSET 200',
# 'SELECT post_id, message, created_time, updated_time FROM stream WHERE source_id IN (' + str(important_friends)[5:-2] + ') AND created_time < ' + str(now) + ' ORDER BY created_time DESC LIMIT 100 OFFSET 300',
'SELECT post_id, message, created_time, updated_time FROM stream WHERE source_id IN (' + str(groups)[1:-1] + ') AND created_time < ' + str(now) + ' ORDER BY created_time DESC LIMIT 100 OFFSET 0',
# 'SELECT post_id, message, created_time, updated_time FROM stream WHERE source_id IN (' + str(groups)[1:-1] + ') AND created_time < ' + str(now) + ' ORDER BY created_time DESC LIMIT 100 OFFSET 100',
# 'g_posts200': 'SELECT post_id, message, created_time, updated_time FROM stream WHERE source_id IN (' + str(groups)[1:-1] + ') AND created_time < ' + str(now) + ' ORDER BY created_time DESC LIMIT 100 OFFSET 200',
# 'g_posts300': 'SELECT post_id, message, created_time, updated_time FROM stream WHERE source_id IN (' + str(groups)[1:-1] + ') AND created_time < ' + str(now) + ' ORDER BY created_time DESC LIMIT 100 OFFSET 300',
]
response = FQL_batch(access_token, queries)
data = flatten([json.loads(answer['body']) for answer in response])
logging.debug("Number of posts: " + str(len(data)))
posts_to_put = []
posts.extend([post for post in data if (int(post['updated_time']) > user.last_timestamp and int(post['created_time']) > year_ago)])
posts_to_put.extend([post for post in data if (int(post['created_time']) > user.last_timestamp and int(post['created_time']) > year_ago)])
if len(posts_to_put) == 0:
logging.debug('Facebook: no new posts')
return
sentencePutter = SentencePutter(models.Posts)
for post in posts_to_put:
sentencePutter.put(post['message'], user.uid, str(post['post_id']), int(post['created_time']))
sentencePutter.flush()
def getAllMessages(access_token, user, now, year_ago, important_friends):
queries = {
'threads0': 'SELECT thread_id, updated_time, message_count, recipients FROM thread WHERE folder_id=0 ORDER BY updated_time DESC LIMIT 50 OFFSET 0',
# 'threads50': 'SELECT thread_id, updated_time, message_count, recipients FROM thread WHERE folder_id=0 ORDER BY updated_time DESC LIMIT 50 OFFSET 50',
# 'threads100': 'SELECT thread_id, updated_time, message_count, recipients FROM thread WHERE folder_id=0 ORDER BY updated_time DESC LIMIT 50 OFFSET 100',
# 'threads150': 'SELECT thread_id, updated_time, message_count, recipients FROM thread WHERE folder_id=0 ORDER BY updated_time DESC LIMIT 50 OFFSET 150'
}
response = FQL_multi(access_token, queries)
# Begin Friends
tmp = []
for result_set in response['data']:
tmp.extend([{'message_count': thread['message_count'], 'recipients': thread['recipients']} for thread in result_set['fql_result_set']])
tmp = [item['recipients'] for item in sorted(tmp, key=lambda item: item['message_count'], reverse=True)[0:10]]
for sublist in tmp:
important_friends.extend(map(str, sublist))
# End Friends
threads_ids = []
for result_set in response['data']:
threads_ids.extend([str(thread['thread_id']) for thread in result_set['fql_result_set'] if int(thread['updated_time']) > user.last_timestamp])
if len(threads_ids) == 0:
logging.debug('Facebook: no new threads')
return
threads = dict(zip(threads_ids, [0] * len(threads_ids)))
offset = 0
active_threads = threads.keys()
sentencePutter = SentencePutter(models.Messages)
_counter = 0
while len(active_threads) > 0 and _counter < 4:
logging.debug("Active threads: " + str(len(active_threads)))
queries = ['SELECT body, message_id, thread_id, created_time FROM message WHERE thread_id IN (' + str(small_list)[1:-1] + ') AND created_time < ' + str(now) + ' ORDER BY created_time DESC LIMIT 6000 OFFSET ' + str(offset) for small_list in bundle(map(str, active_threads), 20)]
response = FQL_batch(access_token, queries)
messages = flatten([json.loads(answer['body']) for answer in response])
logging.debug("Number of messages: " + str(len(messages)))
for msg in messages:
threads[str(msg['thread_id'])] += 1
sentencePutter.put(msg['body'], user.uid, str(msg['message_id']), int(msg['created_time']))
if messages[0]['created_time'] < year_ago:
break
offset += 30
active_threads = [thread_id for thread_id in threads.keys() if threads[thread_id] >= offset]
_counter += 1
sentencePutter.flush()
def getAllComments(access_token, user, now, year_ago, posts_ids):
logging.debug('Beginning getAllComments')
logging.debug("Number of posts with new comments: " + str(len(posts_ids)))
response = FQL_batch(access_token, ['SELECT comments FROM stream WHERE post_id IN (' + str(small_list)[1:-1] + ')' for small_list in bundle(posts_ids, 20)])
data = flatten([json.loads(answer['body']) for answer in response])
logging.debug("Number of comments: " + str(len(data)))
comments_lists = [comments['comments']['comment_list'] for comments in data]
sentencePutter = SentencePutter(models.Comments)
counter = 0
comments_lists = sorted(flatten(comments_lists), key=lambda comment: int(comment['time']), reverse=True)
for comment in comments_lists:
if int(comment['time']) > user.last_timestamp and int(comment['time']) > year_ago:
sentencePutter.put(comment['text'], user.uid, str(comment['id']), int(comment['time']))
counter += 1
logging.debug("Number of putted comments: " + str(counter))
def post(self):
try:
body = json.loads(self.request.body)
except:
self.response.write('400 invalid json in request body')
return
sp = SentencePutter(Comments)
sp.put(sentence=body['sentence'], uid=body['uid'], fbid=body['fbid'], timestamp=body['timestamp'])
sp.put(sentence=body['sentence1'], uid=body['uid'], fbid=body['fbid'], timestamp=body['timestamp'])
sp.put(sentence=body['sentence2'], uid=body['uid'], fbid=body['fbid'], timestamp=body['timestamp'])
sp.put(sentence=body['sentence'], uid=body['uid'], fbid=body['fbid'], timestamp=body['timestamp'])
sp.put(sentence=body['sentence'], uid=body['uid'], fbid=body['fbid'], timestamp=body['timestamp'])
sp.put(sentence=body['sentence'], uid=body['uid'], fbid=body['fbid'], timestamp=body['timestamp'])
sp.put(sentence=body['sentence'], uid=body['uid'], fbid=body['fbid'], timestamp=body['timestamp'])
sp.flush()
response = {'status': 'success'}
self.response.write(json.dumps(response))
