def trans(db1, db2):
db2.create_index("id", unique=True)
for user in db1.find({}):
if profiles_check.check_ed(user) == True:
try:
db2.insert(user)
except pymongo.errors.DuplicateKeyError:
pass
def count_eds(poidb):
print 'start count'
count = 0
for user in poidb.find({}):
if profiles_check.check_ed(user):
print user['screen_name']
count += 1
print count
def states_change(dbname1, dbname2, comname1, comname2):
db1 = dbt.db_connect_no_auth(dbname1)
db2 = dbt.db_connect_no_auth(dbname2)
com1 = db1[comname1]
com2 = db2[comname2]
count = 0
index = 0
for user1 in com1.find({'level': 1}):
index += 1
user1_ed = profiles_check.check_ed(user1)
user2 = com2.find_one({'id': user1['id']})
if user2:
user2_ed = profiles_check.check_ed(user2)
if user1_ed != user2_ed:
print user1['id']
count += 1
print count
print index
def states_change(dbname1, dbname2, comname1, comname2):
db1 = dbt.db_connect_no_auth(dbname1)
db2 = dbt.db_connect_no_auth(dbname2)
com1 = db1[comname1]
com2 = db2[comname2]
count = 0
index = 0
for user1 in com1.find({'level': 1}):
index += 1
user1_ed = profiles_check.check_ed(user1)
user2 = com2.find_one({'id': user1['id']})
if user2:
user2_ed = profiles_check.check_ed(user2)
if user1_ed != user2_ed:
print user1['id']
count += 1
print count
print index
def output_net_user_data(dbname, comname, netname):
'''
Output the social network (two-ground) and user's ED states into local files
'''
g = gt.load_network(dbname, netname)
gt.summary(g)
com = dbt.db_connect_col(dbname, comname)
for v in g.vs:
user = com.find_one({'id': int(v['name'])})
v['l'] = user['level']
v['ed'] = profiles_check.check_ed(user)
g.write_graphml(dbname+'-'+netname+'.graphml')
def check_users(stream, user_info):
while True:
count = stream.count({'user_extracted': {'$exists': False}})
if count == 0:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'no new stream, sleep 2 hours'
time.sleep(2*60*60)
continue
else:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'extract users from stream amount:', count
for tweet in stream.find({'user_extracted':{'$exists': False}},
['id_str', 'user']).limit(min(100, count)):
user = tweet['user']
if profiles_check.check_ed(user) == True:
try:
user_info.insert(user)
except pymongo.errors.DuplicateKeyError:
pass
stream.update({'id': int(tweet['id_str'])}, {'$set':{"user_extracted": True
}}, upsert=False)
def ed_friend_num(dbname, comname, netname, flag):
'''
Only one-round social network are used
'''
filter_user = {'level': 1}
users = iot.get_values_one_field(dbname, comname, 'id', filter_user)
net = gt.load_network(dbname, netname)
com = dbt.db_connect_col(dbname, comname)
data = []
for uid in users:
row = [uid, flag]
exist = True
try:
v = net.vs.find(name=str(uid))
except ValueError:
exist = False
if exist:
followees = set([int(net.vs[v]['name']) for v in net.successors(str(uid))])
followers = set([int(net.vs[v]['name']) for v in net.predecessors(str(uid))])
common = followees.intersection(followers)
followees = followees - common
followers = followers - common
for fids in [followees, followers, common]:
if len(fids) > 0:
print uid in fids
print len(fids)
ed_num, noned_num = 0, 0
for fid in fids:
ed_flag = profiles_check.check_ed(com.find_one({'id': fid}))
if ed_flag:
ed_num += 1
else:
noned_num += 1
row.extend([ed_num, noned_num])
else:
row.extend([None] * 2)
data.append(row)
return data
"""
from ohsn.util import db_util as dbt
from ohsn.api import profiles_check
db = dbt.db_connect_no_auth('ed')
ed_poi = db['poi_ed']
# db = dbt.db_connect_no_auth('echelon')
ech_poi = db['poi_ed_all']
# Common users in Echelon and ED
# index = 1
# for poi in ed_poi.find({}):
# poi_id = poi['id']
# count = ech_poi.count({'id':poi_id})
# if count > 0:
# print index, poi_id, poi['screen_name']
# index += 1
# Target users in Echelon but not in ED
index = 1
for poi in ech_poi.find({}):
poi_id = poi['id']
if profiles_check.check_ed(poi):
count = ed_poi.count({'id':poi_id})
if count < 1:
print index, poi_id, poi['screen_name']
index += 1
"""
from ohsn.util import db_util as dbt
from ohsn.api import profiles_check
db = dbt.db_connect_no_auth('ed')
ed_poi = db['poi_ed']
# db = dbt.db_connect_no_auth('echelon')
ech_poi = db['poi_ed_all']
# Common users in Echelon and ED
# index = 1
# for poi in ed_poi.find({}):
# poi_id = poi['id']
# count = ech_poi.count({'id':poi_id})
# if count > 0:
# print index, poi_id, poi['screen_name']
# index += 1
# Target users in Echelon but not in ED
index = 1
for poi in ech_poi.find({}):
poi_id = poi['id']
if profiles_check.check_ed(poi):
count = ed_poi.count({'id': poi_id})
if count < 1:
print index, poi_id, poi['screen_name']
index += 1
def eliminate_non_ed(poidb, netdb):
for user in poidb.find({'level':1}):
if not profiles_check.check_ed(user):
poidb.delete_one({'id': user['id']})
netdb.delete_many({'user': user['id']})
netdb.delete_many({'follower': user['id']})
