def re_snowball_friends(olddbname, oldcomname, newdbname, newcomname): newdb = dbt.db_connect_no_auth(newdbname) newcom = newdb[newcomname] newnet = newdb['net'] newcom.create_index("id", unique=True) newcom.create_index([('level', pymongo.ASCENDING), ('following_prelevel_node', pymongo.ASCENDING)], unique=False) newcom.create_index([('level', pymongo.ASCENDING), ('follower_prelevel_node', pymongo.ASCENDING)], unique=False) newnet.create_index([("user", pymongo.ASCENDING), ("follower", pymongo.ASCENDING)], unique=True) '''Reteive ED core users''' ed_users = iot.get_values_one_field(olddbname, oldcomname, 'id', {'level': 1}) list_size = len(ed_users) length = int(math.ceil(list_size/100.0)) for index in xrange(length): index_begin = index*100 index_end = min(list_size, index_begin+100) lookup.lookup_user_list(ed_users[index_begin:index_end], newcom, 1, 'N') level = 1 while True: # Each call of snowball_following and snowball_follower only process up to 200 users print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followings of seeds for sample db', level following_flag = following.snowball_following(newcom, newnet, level, 'N') print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followees of seeds for sample db', level follower_flag = follower.snowball_follower(newcom, newnet, level, 'N') if following_flag == False and follower_flag == False: break else: continue
def network_snowball(dbname, mode='N'): db = dbt.db_connect_no_auth(dbname) ed_poi = db['ccom'] ed_net = db['cnet'] stream_users = db['poi'] # echelon = dbt.db_connect_no_auth('echelon') # echelon_poi = echelon['poi'] ed_poi.create_index("id", unique=True) ed_poi.create_index([('level', pymongo.ASCENDING), ('following_prelevel_node', pymongo.ASCENDING)], unique=False) ed_poi.create_index([('level', pymongo.ASCENDING), ('follower_prelevel_node', pymongo.ASCENDING)], unique=False) ed_net.create_index([("user", pymongo.ASCENDING), ("follower", pymongo.ASCENDING), ("type", pymongo.ASCENDING)], unique=True) while True: ed_seed = profiles_check.seed_all_profile(stream_users) length = len(ed_seed) if length == 0: print datetime.datetime.now().strftime( "%Y-%m-%d-%H-%M-%S"), 'no seed users, finished!' break else: print 'seed users: ', length lookup.trans_seed_to_poi(ed_seed, ed_poi) continue statis = '' level = 1 while level < 3: print datetime.datetime.now().strftime( "%Y-%m-%d-%H-%M-%S" ), 'Snowball followings of seeds for sample db', level following_flag = following.snowball_following(ed_poi, ed_net, level, mode) print datetime.datetime.now().strftime( "%Y-%m-%d-%H-%M-%S" ), 'Snowball followees of seeds for sample db', level follower_flag = follower.snowball_follower(ed_poi, ed_net, level, mode) # count = ed_poi.count() # try: # # nsize, esize = nt.size_gaint_comp_net_db(ed_net) # # s = 'Start_level: ' + str(level) + ' all_users: ' + \ # # str(count) + ' size_gc:' + str(nsize) + ' ed_gc: ' + str(esize) + '\n' # print s # statis += s # except networkx.exception.NetworkXPointlessConcept: # nsize = 0 # pass if (following_flag == False and follower_flag == False): return statis else: level += 1 continue
def network_snowball(dbname, mode='N'): db = dbt.db_connect_no_auth(dbname) ed_poi = db['ccom'] ed_net = db['cnet'] stream_users = db['poi'] # echelon = dbt.db_connect_no_auth('echelon') # echelon_poi = echelon['poi'] ed_poi.create_index("id", unique=True) ed_poi.create_index([('level', pymongo.ASCENDING), ('following_prelevel_node', pymongo.ASCENDING)], unique=False) ed_poi.create_index([('level', pymongo.ASCENDING), ('follower_prelevel_node', pymongo.ASCENDING)], unique=False) ed_net.create_index([("user", pymongo.ASCENDING), ("follower", pymongo.ASCENDING), ("type", pymongo.ASCENDING)], unique=True) while True: ed_seed = profiles_check.seed_all_profile(stream_users) length = len(ed_seed) if length==0: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'no seed users, finished!' break else: print 'seed users: ', length lookup.trans_seed_to_poi(ed_seed, ed_poi) continue statis = '' level = 1 while level < 3: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followings of seeds for sample db', level following_flag = following.snowball_following(ed_poi, ed_net, level, mode) print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followees of seeds for sample db', level follower_flag = follower.snowball_follower(ed_poi, ed_net, level, mode) # count = ed_poi.count() # try: # # nsize, esize = nt.size_gaint_comp_net_db(ed_net) # # s = 'Start_level: ' + str(level) + ' all_users: ' + \ # # str(count) + ' size_gc:' + str(nsize) + ' ed_gc: ' + str(esize) + '\n' # print s # statis += s # except networkx.exception.NetworkXPointlessConcept: # nsize = 0 # pass if (following_flag == False and follower_flag == False): return statis else: level += 1 continue
def snowball_friends(dbname, com_name, net_name): db = dbt.db_connect_no_auth(dbname) ed_poi = db[com_name] ed_net = db[net_name] # echelon = dbt.db_connect_no_auth('echelon') # echelon_poi = echelon['poi'] ed_poi.create_index("id", unique=True) ed_poi.create_index([('level', pymongo.ASCENDING), ('following_prelevel_node', pymongo.ASCENDING)], unique=False) ed_poi.create_index([('level', pymongo.ASCENDING), ('follower_prelevel_node', pymongo.ASCENDING)], unique=False) ed_net.create_index([("user", pymongo.ASCENDING), ("follower", pymongo.ASCENDING), ("type", pymongo.ASCENDING)], unique=True) # while True: # ed_seed = profiles_check.seed_all_profile(stream_users) # if len(ed_seed) == 0: # print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'no seed users, finished!' # break # else: # lookup.trans_seed_to_poi(ed_seed, ed_poi) # continue ed_users = iot.get_values_one_field(dbname, 'stream_user', 'id') list_size = len(ed_users) length = int(math.ceil(list_size/100.0)) for index in xrange(length): index_begin = index*100 index_end = min(list_size, index_begin+100) lookup.lookup_user_list(ed_users[index_begin:index_end], ed_poi, 1, 'DP') level = 1 while True: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followings of seeds for sample db', level following_flag = following.snowball_following(ed_poi, ed_net, level, 'DP') print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followees of seeds for sample db', level follower_flag = follower.snowball_follower(ed_poi, ed_net, level, 'DP') if following_flag == False and follower_flag == False: break else: level = level + 1 continue
def follow_net(dbname='ed', username='******', netname='net'): # Collect follow network among users com = dbt.db_connect_col(dbname, username) net = dbt.db_connect_col(dbname, netname) net.create_index([("user", pymongo.ASCENDING), ("follower", pymongo.ASCENDING)], unique=True) level = 1 while level < 2: # Each call of snowball_following and snowball_follower only process up to 200 users print datetime.datetime.now().strftime( "%Y-%m-%d-%H-%M-%S" ), 'Snowball followings of seeds for sample db', level following_flag = following.snowball_following(com, net, level, 'N') print datetime.datetime.now().strftime( "%Y-%m-%d-%H-%M-%S" ), 'Snowball followees of seeds for sample db', level follower_flag = follower.snowball_follower(com, net, level, 'N') if following_flag == False and follower_flag == False: level += 1 continue
def snowball_friends(): db = dbt.db_connect_no_auth('ed') ed_poi = db['com'] ed_net = db['net'] # stream_users = db['poi'] # echelon = dbt.db_connect_no_auth('echelon') # echelon_poi = echelon['poi'] ed_poi.create_index("id", unique=True) ed_poi.create_index([('level', pymongo.ASCENDING), ('following_prelevel_node', pymongo.ASCENDING)], unique=False) ed_poi.create_index([('level', pymongo.ASCENDING), ('follower_prelevel_node', pymongo.ASCENDING)], unique=False) ed_net.create_index([("user", pymongo.ASCENDING), ("follower", pymongo.ASCENDING), ("type", pymongo.ASCENDING)], unique=True) # while True: # ed_seed = profiles_check.seed_all_profile(stream_users) # if len(ed_seed)==0: # print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'no seed users, finished!' # break # else: # lookup.trans_seed_to_poi(ed_seed, ed_poi) # continue level = 1 while True: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followings of seeds for sample db', level following_flag = following.snowball_following(ed_poi, ed_net, level, 'ED') print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followees of seeds for sample db', level follower_flag = follower.snowball_follower(ed_poi, ed_net, level, 'ED') if following_flag == False and follower_flag == False: break else: level = level + 1 continue
def re_snowball_friends(olddbname, oldcomname, newdbname, newcomname): newdb = dbt.db_connect_no_auth(newdbname) newcom = newdb[newcomname] newnet = newdb['net'] newcom.create_index("id", unique=True) newcom.create_index([('level', pymongo.ASCENDING), ('following_prelevel_node', pymongo.ASCENDING)], unique=False) newcom.create_index([('level', pymongo.ASCENDING), ('follower_prelevel_node', pymongo.ASCENDING)], unique=False) newnet.create_index([("user", pymongo.ASCENDING), ("follower", pymongo.ASCENDING)], unique=True) '''Reteive ED core users''' ed_users = iot.get_values_one_field(olddbname, oldcomname, 'id') list_size = len(ed_users) print '%d users to process' %list_size length = int(math.ceil(list_size/100.0)) for index in xrange(length): index_begin = index*100 index_end = min(list_size, index_begin+100) lookup.lookup_user_list(ed_users[index_begin:index_end], newcom, 1, 'N') '''Snowball sampling round''' level = 1 while level < 2: # Each call of snowball_following and snowball_follower only process up to 200 users print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followings of seeds for sample db', level following_flag = following.snowball_following(newcom, newnet, level, 'N') print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followees of seeds for sample db', level follower_flag = follower.snowball_follower(newcom, newnet, level, 'N') if following_flag == False and follower_flag == False: level += 1 continue
ed_net.create_index([("user", pymongo.ASCENDING), ("follower", pymongo.ASCENDING), ("type", pymongo.ASCENDING)], unique=True) while True: try: ed_seed = profiles_check.seed_all_profile(echelon_poi) # print ed_seed lookup.trans_seed_to_poi(ed_seed, ed_poi) level = 1 while True: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followings of seeds for sample db', level following_flag = following.snowball_following(ed_poi, ed_net, level, 'ED') print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followees of seeds for sample db', level follower_flag = follower.snowball_follower(ed_poi, ed_net, level, 'ED') if following_flag==False and follower_flag==False: break else: level += 1 continue if level == 1: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'No new seed users, sleep 15 mins' time.sleep(15*60) continue except Exception as details: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'exception stop, re-run' continue
ed_poi.create_index("id", unique=True) ed_poi.create_index([('level', pymongo.ASCENDING), ('following_prelevel_node', pymongo.ASCENDING)], unique=False) ed_poi.create_index([('level', pymongo.ASCENDING), ('follower_prelevel_node', pymongo.ASCENDING)], unique=False) ed_net.create_index([("user", pymongo.ASCENDING), ("follower", pymongo.ASCENDING), ("type", pymongo.ASCENDING)], unique=True) while True: ed_seed = profiles_check.seed_all_profile(stream_users) if len(ed_seed) == 0: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'no seed users, finished!' break else: lookup.trans_seed_to_poi(ed_seed, ed_poi) continue while True: level = 1 print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followings of seeds for sample db', level following_flag = following.snowball_following(ed_poi, ed_net, level, check='YG') print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followees of seeds for sample db', level follower_flag = follower.snowball_follower(ed_poi, ed_net, level, check='YG') if following_flag == False and follower_flag == False: break
unique=False) ed_poi.create_index([('level', pymongo.ASCENDING), ('follower_prelevel_node', pymongo.ASCENDING)], unique=False) ed_net.create_index([("user", pymongo.ASCENDING), ("follower", pymongo.ASCENDING), ("type", pymongo.ASCENDING)], unique=True) while True: ed_seed = profiles_check.seed_all_profile(stream_users) if len(ed_seed) == 0: print datetime.datetime.now().strftime( "%Y-%m-%d-%H-%M-%S"), 'no seed users, finished!' break else: lookup.trans_seed_to_poi(ed_seed, ed_poi) continue while True: level = 1 print datetime.datetime.now().strftime( "%Y-%m-%d-%H-%M-%S" ), 'Snowball followings of seeds for sample db', level following_flag = following.snowball_following(ed_poi, ed_net, level) print datetime.datetime.now().strftime( "%Y-%m-%d-%H-%M-%S" ), 'Snowball followees of seeds for sample db', level follower_flag = follower.snowball_follower(ed_poi, ed_net, level) if following_flag == False and follower_flag == False: break
("type", pymongo.ASCENDING)], unique=True) while True: ed_seed = profiles_check.seed_all_profile(stream_users) if len(ed_seed) == 0: print datetime.datetime.now().strftime( "%Y-%m-%d-%H-%M-%S"), 'no seed users, finished!' break else: lookup.trans_seed_to_poi(ed_seed, ed_poi) continue while True: level = 1 print datetime.datetime.now().strftime( "%Y-%m-%d-%H-%M-%S" ), 'Snowball followings of seeds for sample db', level following_flag = following.snowball_following(ed_poi, ed_net, level, check='YG') print datetime.datetime.now().strftime( "%Y-%m-%d-%H-%M-%S" ), 'Snowball followees of seeds for sample db', level follower_flag = follower.snowball_follower(ed_poi, ed_net, level, check='YG') if following_flag == False and follower_flag == False: break