def populate_users_posts(ds, de, all_posts ,all_users): #ds, de is utc start_id = get_id('posts','created_on', ds) end_id = get_id('posts', 'created_on', de) print start_id, end_id start_id -= 1 while 1: print start_id count = db_slave.post.find({'_id': {'$gt': start_id, '$lt': end_id}}).limit(1).count(True) if count == 0: break for r in db_slave.post.find({'_id': {'$gt': start_id, '$lt': end_id}},{'del':1,'c':1,'nc':1,'nl':1,'l':1,'u':1}).sort('_id',1).limit(10000): #if r['t'] in [1,2,3,5,7]: #temp = all_posts.get(r['u']) or [] #temp.append(r) #all_posts[r['u']] = temp start_id = r['_id'] if r.get('l') and not r.get('del'): guid = r.get('l') if __is_virtual_loc(guid): continue #post_data = dict(r) del r['l'] all_posts[r['_id']] = r all_users.add(r['u'])
def populate_users_posts(ds, de, all_posts): #ds, de is utc start_id = get_id('posts','created_on', ds) end_id = get_id('posts', 'created_on', de) print start_id, end_id start_id -= 1 while 1: print start_id count = db_slave.post.find({'_id': {'$gt': start_id, '$lt': end_id}}).limit(1).count(True) if count == 0: break for r in db_slave.post.find({'_id': {'$gt': start_id, '$lt': end_id}}).sort('_id',1).limit(10000): if r['t'] in [1,2,3,5,7]: temp = all_posts.get(r['u']) or [] temp.append(r) all_posts[r['u']] = temp start_id = r['_id']