def make(date): end_ts = datetime2ts(date) start_ts = end_ts - 24*60*60 db_name = get_leveldb('impotant', end_ts) daily_user_important_bucket = leveldb.LevelDB(os.path.join(LEVELDBPATH, db_name), block_cache_size=8 * (2 << 25), write_buffer_size=8 * (2 << 25)) batch = leveldb.WriteBatch() query_dict = {'timestamp': {'$gt': start_ts, '$lt': end_ts}, 'reposts_count': {'$gt': 500}} statuses_count, get_statuses_results = statuses_search.search(query=query_dict, field=['user', '_id', 'reposts_count']) print 'total statuses: %s' % statuses_count print 'writing to levelDB %s...' % db_name count = 0 uid_important = {} for status in get_statuses_results(): if count % 10000 == 0: print 'current count: %s' % count uid = status['user'] reposts_count = status['reposts_count'] followers_count = 0 user_count, get_user_results = user_search.search(query={'_id': uid}) if user_count == 1: for user in get_user_results(): followers_count = user['followers_count'] important = 0.9 * reposts_count + 0.1 * followers_count if uid not in uid_important: uid_important[uid] = 0 important += uid_important[uid] uid_important[uid] = important print uid,important batch.Put(str(uid), str(important)) count += 1 daily_user_important_bucket.Write(batch, sync=True) print 'done.'