def re_snowball_friends(olddbname, oldcomname, newdbname, newcomname):
newdb = dbt.db_connect_no_auth(newdbname)
newcom = newdb[newcomname]
newnet = newdb['net']
newcom.create_index("id", unique=True)
newcom.create_index([('level', pymongo.ASCENDING),
('following_prelevel_node', pymongo.ASCENDING)],
unique=False)
newcom.create_index([('level', pymongo.ASCENDING),
('follower_prelevel_node', pymongo.ASCENDING)],
unique=False)
newnet.create_index([("user", pymongo.ASCENDING),
("follower", pymongo.ASCENDING)],
unique=True)
'''Reteive ED core users'''
ed_users = iot.get_values_one_field(olddbname, oldcomname, 'id', {'level': 1})
list_size = len(ed_users)
length = int(math.ceil(list_size/100.0))
for index in xrange(length):
index_begin = index*100
index_end = min(list_size, index_begin+100)
lookup.lookup_user_list(ed_users[index_begin:index_end], newcom, 1, 'N')
level = 1
while True:
# Each call of snowball_following and snowball_follower only process up to 200 users
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followings of seeds for sample db', level
following_flag = following.snowball_following(newcom, newnet, level, 'N')
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followees of seeds for sample db', level
follower_flag = follower.snowball_follower(newcom, newnet, level, 'N')
if following_flag == False and follower_flag == False:
break
else:
continue
def network_snowball(dbname, mode='N'):
db = dbt.db_connect_no_auth(dbname)
ed_poi = db['ccom']
ed_net = db['cnet']
stream_users = db['poi']
# echelon = dbt.db_connect_no_auth('echelon')
# echelon_poi = echelon['poi']
ed_poi.create_index("id", unique=True)
ed_poi.create_index([('level', pymongo.ASCENDING),
('following_prelevel_node', pymongo.ASCENDING)],
unique=False)
ed_poi.create_index([('level', pymongo.ASCENDING),
('follower_prelevel_node', pymongo.ASCENDING)],
unique=False)
ed_net.create_index([("user", pymongo.ASCENDING),
("follower", pymongo.ASCENDING),
("type", pymongo.ASCENDING)],
unique=True)
while True:
ed_seed = profiles_check.seed_all_profile(stream_users)
length = len(ed_seed)
if length == 0:
print datetime.datetime.now().strftime(
"%Y-%m-%d-%H-%M-%S"), 'no seed users, finished!'
break
else:
print 'seed users: ', length
lookup.trans_seed_to_poi(ed_seed, ed_poi)
continue
statis = ''
level = 1
while level < 3:
print datetime.datetime.now().strftime(
"%Y-%m-%d-%H-%M-%S"
), 'Snowball followings of seeds for sample db', level
following_flag = following.snowball_following(ed_poi, ed_net, level,
mode)
print datetime.datetime.now().strftime(
"%Y-%m-%d-%H-%M-%S"
), 'Snowball followees of seeds for sample db', level
follower_flag = follower.snowball_follower(ed_poi, ed_net, level, mode)
# count = ed_poi.count()
# try:
# # nsize, esize = nt.size_gaint_comp_net_db(ed_net)
# # s = 'Start_level: ' + str(level) + ' all_users: ' + \
# # str(count) + ' size_gc:' + str(nsize) + ' ed_gc: ' + str(esize) + '\n'
# print s
# statis += s
# except networkx.exception.NetworkXPointlessConcept:
# nsize = 0
# pass
if (following_flag == False and follower_flag == False):
return statis
else:
level += 1
continue
def network_snowball(dbname, mode='N'):
db = dbt.db_connect_no_auth(dbname)
ed_poi = db['ccom']
ed_net = db['cnet']
stream_users = db['poi']
# echelon = dbt.db_connect_no_auth('echelon')
# echelon_poi = echelon['poi']
ed_poi.create_index("id", unique=True)
ed_poi.create_index([('level', pymongo.ASCENDING),
('following_prelevel_node', pymongo.ASCENDING)],
unique=False)
ed_poi.create_index([('level', pymongo.ASCENDING),
('follower_prelevel_node', pymongo.ASCENDING)],
unique=False)
ed_net.create_index([("user", pymongo.ASCENDING),
("follower", pymongo.ASCENDING),
("type", pymongo.ASCENDING)],
unique=True)
while True:
ed_seed = profiles_check.seed_all_profile(stream_users)
length = len(ed_seed)
if length==0:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'no seed users, finished!'
break
else:
print 'seed users: ', length
lookup.trans_seed_to_poi(ed_seed, ed_poi)
continue
statis = ''
level = 1
while level < 3:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followings of seeds for sample db', level
following_flag = following.snowball_following(ed_poi, ed_net, level, mode)
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followees of seeds for sample db', level
follower_flag = follower.snowball_follower(ed_poi, ed_net, level, mode)
# count = ed_poi.count()
# try:
# # nsize, esize = nt.size_gaint_comp_net_db(ed_net)
# # s = 'Start_level: ' + str(level) + ' all_users: ' + \
# # str(count) + ' size_gc:' + str(nsize) + ' ed_gc: ' + str(esize) + '\n'
# print s
# statis += s
# except networkx.exception.NetworkXPointlessConcept:
# nsize = 0
# pass
if (following_flag == False and follower_flag == False):
return statis
else:
level += 1
continue
def snowball_friends(dbname, com_name, net_name):
db = dbt.db_connect_no_auth(dbname)
ed_poi = db[com_name]
ed_net = db[net_name]
# echelon = dbt.db_connect_no_auth('echelon')
# echelon_poi = echelon['poi']
ed_poi.create_index("id", unique=True)
ed_poi.create_index([('level', pymongo.ASCENDING),
('following_prelevel_node', pymongo.ASCENDING)],
unique=False)
ed_poi.create_index([('level', pymongo.ASCENDING),
('follower_prelevel_node', pymongo.ASCENDING)],
unique=False)
ed_net.create_index([("user", pymongo.ASCENDING),
("follower", pymongo.ASCENDING),
("type", pymongo.ASCENDING)],
unique=True)
# while True:
# ed_seed = profiles_check.seed_all_profile(stream_users)
# if len(ed_seed) == 0:
# print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'no seed users, finished!'
# break
# else:
# lookup.trans_seed_to_poi(ed_seed, ed_poi)
# continue
ed_users = iot.get_values_one_field(dbname, 'stream_user', 'id')
list_size = len(ed_users)
length = int(math.ceil(list_size/100.0))
for index in xrange(length):
index_begin = index*100
index_end = min(list_size, index_begin+100)
lookup.lookup_user_list(ed_users[index_begin:index_end], ed_poi, 1, 'DP')
level = 1
while True:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followings of seeds for sample db', level
following_flag = following.snowball_following(ed_poi, ed_net, level, 'DP')
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followees of seeds for sample db', level
follower_flag = follower.snowball_follower(ed_poi, ed_net, level, 'DP')
if following_flag == False and follower_flag == False:
break
else:
level = level + 1
continue
def follow_net(dbname='ed', username='******', netname='net'):
# Collect follow network among users
com = dbt.db_connect_col(dbname, username)
net = dbt.db_connect_col(dbname, netname)
net.create_index([("user", pymongo.ASCENDING),
("follower", pymongo.ASCENDING)],
unique=True)
level = 1
while level < 2:
# Each call of snowball_following and snowball_follower only process up to 200 users
print datetime.datetime.now().strftime(
"%Y-%m-%d-%H-%M-%S"
), 'Snowball followings of seeds for sample db', level
following_flag = following.snowball_following(com, net, level, 'N')
print datetime.datetime.now().strftime(
"%Y-%m-%d-%H-%M-%S"
), 'Snowball followees of seeds for sample db', level
follower_flag = follower.snowball_follower(com, net, level, 'N')
if following_flag == False and follower_flag == False:
level += 1
continue
def snowball_friends():
db = dbt.db_connect_no_auth('ed')
ed_poi = db['com']
ed_net = db['net']
# stream_users = db['poi']
# echelon = dbt.db_connect_no_auth('echelon')
# echelon_poi = echelon['poi']
ed_poi.create_index("id", unique=True)
ed_poi.create_index([('level', pymongo.ASCENDING),
('following_prelevel_node', pymongo.ASCENDING)],
unique=False)
ed_poi.create_index([('level', pymongo.ASCENDING),
('follower_prelevel_node', pymongo.ASCENDING)],
unique=False)
ed_net.create_index([("user", pymongo.ASCENDING),
("follower", pymongo.ASCENDING),
("type", pymongo.ASCENDING)],
unique=True)
# while True:
# ed_seed = profiles_check.seed_all_profile(stream_users)
# if len(ed_seed)==0:
# print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'no seed users, finished!'
# break
# else:
# lookup.trans_seed_to_poi(ed_seed, ed_poi)
# continue
level = 1
while True:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followings of seeds for sample db', level
following_flag = following.snowball_following(ed_poi, ed_net, level, 'ED')
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followees of seeds for sample db', level
follower_flag = follower.snowball_follower(ed_poi, ed_net, level, 'ED')
if following_flag == False and follower_flag == False:
break
else:
level = level + 1
continue
def re_snowball_friends(olddbname, oldcomname, newdbname, newcomname):
newdb = dbt.db_connect_no_auth(newdbname)
newcom = newdb[newcomname]
newnet = newdb['net']
newcom.create_index("id", unique=True)
newcom.create_index([('level', pymongo.ASCENDING),
('following_prelevel_node', pymongo.ASCENDING)],
unique=False)
newcom.create_index([('level', pymongo.ASCENDING),
('follower_prelevel_node', pymongo.ASCENDING)],
unique=False)
newnet.create_index([("user", pymongo.ASCENDING),
("follower", pymongo.ASCENDING)],
unique=True)
'''Reteive ED core users'''
ed_users = iot.get_values_one_field(olddbname, oldcomname, 'id')
list_size = len(ed_users)
print '%d users to process' %list_size
length = int(math.ceil(list_size/100.0))
for index in xrange(length):
index_begin = index*100
index_end = min(list_size, index_begin+100)
lookup.lookup_user_list(ed_users[index_begin:index_end], newcom, 1, 'N')
'''Snowball sampling round'''
level = 1
while level < 2:
# Each call of snowball_following and snowball_follower only process up to 200 users
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followings of seeds for sample db', level
following_flag = following.snowball_following(newcom, newnet, level, 'N')
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followees of seeds for sample db', level
follower_flag = follower.snowball_follower(newcom, newnet, level, 'N')
if following_flag == False and follower_flag == False:
level += 1
continue
ed_net.create_index([("user", pymongo.ASCENDING),
("follower", pymongo.ASCENDING),
("type", pymongo.ASCENDING)],
unique=True)
while True:
try:
ed_seed = profiles_check.seed_all_profile(echelon_poi)
# print ed_seed
lookup.trans_seed_to_poi(ed_seed, ed_poi)
level = 1
while True:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followings of seeds for sample db', level
following_flag = following.snowball_following(ed_poi, ed_net, level, 'ED')
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followees of seeds for sample db', level
follower_flag = follower.snowball_follower(ed_poi, ed_net, level, 'ED')
if following_flag==False and follower_flag==False:
break
else:
level += 1
continue
if level == 1:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'No new seed users, sleep 15 mins'
time.sleep(15*60)
continue
except Exception as details:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'exception stop, re-run'
continue
ed_poi.create_index("id", unique=True)
ed_poi.create_index([('level', pymongo.ASCENDING),
('following_prelevel_node', pymongo.ASCENDING)],
unique=False)
ed_poi.create_index([('level', pymongo.ASCENDING),
('follower_prelevel_node', pymongo.ASCENDING)],
unique=False)
ed_net.create_index([("user", pymongo.ASCENDING),
("follower", pymongo.ASCENDING),
("type", pymongo.ASCENDING)],
unique=True)
while True:
ed_seed = profiles_check.seed_all_profile(stream_users)
if len(ed_seed) == 0:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'no seed users, finished!'
break
else:
lookup.trans_seed_to_poi(ed_seed, ed_poi)
continue
while True:
level = 1
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followings of seeds for sample db', level
following_flag = following.snowball_following(ed_poi, ed_net, level, check='YG')
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'Snowball followees of seeds for sample db', level
follower_flag = follower.snowball_follower(ed_poi, ed_net, level, check='YG')
if following_flag == False and follower_flag == False:
break
unique=False)
ed_poi.create_index([('level', pymongo.ASCENDING),
('follower_prelevel_node', pymongo.ASCENDING)],
unique=False)
ed_net.create_index([("user", pymongo.ASCENDING),
("follower", pymongo.ASCENDING),
("type", pymongo.ASCENDING)],
unique=True)
while True:
ed_seed = profiles_check.seed_all_profile(stream_users)
if len(ed_seed) == 0:
print datetime.datetime.now().strftime(
"%Y-%m-%d-%H-%M-%S"), 'no seed users, finished!'
break
else:
lookup.trans_seed_to_poi(ed_seed, ed_poi)
continue
while True:
level = 1
print datetime.datetime.now().strftime(
"%Y-%m-%d-%H-%M-%S"
), 'Snowball followings of seeds for sample db', level
following_flag = following.snowball_following(ed_poi, ed_net, level)
print datetime.datetime.now().strftime(
"%Y-%m-%d-%H-%M-%S"
), 'Snowball followees of seeds for sample db', level
follower_flag = follower.snowball_follower(ed_poi, ed_net, level)
if following_flag == False and follower_flag == False:
break
("type", pymongo.ASCENDING)],
unique=True)
while True:
ed_seed = profiles_check.seed_all_profile(stream_users)
if len(ed_seed) == 0:
print datetime.datetime.now().strftime(
"%Y-%m-%d-%H-%M-%S"), 'no seed users, finished!'
break
else:
lookup.trans_seed_to_poi(ed_seed, ed_poi)
continue
while True:
level = 1
print datetime.datetime.now().strftime(
"%Y-%m-%d-%H-%M-%S"
), 'Snowball followings of seeds for sample db', level
following_flag = following.snowball_following(ed_poi,
ed_net,
level,
check='YG')
print datetime.datetime.now().strftime(
"%Y-%m-%d-%H-%M-%S"
), 'Snowball followees of seeds for sample db', level
follower_flag = follower.snowball_follower(ed_poi,
ed_net,
level,
check='YG')
if following_flag == False and follower_flag == False:
break
