def trans(db1, db2): db2.create_index("id", unique=True) for user in db1.find({}): if profiles_check.check_ed(user) == True: try: db2.insert(user) except pymongo.errors.DuplicateKeyError: pass
def count_eds(poidb): print 'start count' count = 0 for user in poidb.find({}): if profiles_check.check_ed(user): print user['screen_name'] count += 1 print count
def states_change(dbname1, dbname2, comname1, comname2): db1 = dbt.db_connect_no_auth(dbname1) db2 = dbt.db_connect_no_auth(dbname2) com1 = db1[comname1] com2 = db2[comname2] count = 0 index = 0 for user1 in com1.find({'level': 1}): index += 1 user1_ed = profiles_check.check_ed(user1) user2 = com2.find_one({'id': user1['id']}) if user2: user2_ed = profiles_check.check_ed(user2) if user1_ed != user2_ed: print user1['id'] count += 1 print count print index
def output_net_user_data(dbname, comname, netname): ''' Output the social network (two-ground) and user's ED states into local files ''' g = gt.load_network(dbname, netname) gt.summary(g) com = dbt.db_connect_col(dbname, comname) for v in g.vs: user = com.find_one({'id': int(v['name'])}) v['l'] = user['level'] v['ed'] = profiles_check.check_ed(user) g.write_graphml(dbname+'-'+netname+'.graphml')
def check_users(stream, user_info): while True: count = stream.count({'user_extracted': {'$exists': False}}) if count == 0: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'no new stream, sleep 2 hours' time.sleep(2*60*60) continue else: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"), 'extract users from stream amount:', count for tweet in stream.find({'user_extracted':{'$exists': False}}, ['id_str', 'user']).limit(min(100, count)): user = tweet['user'] if profiles_check.check_ed(user) == True: try: user_info.insert(user) except pymongo.errors.DuplicateKeyError: pass stream.update({'id': int(tweet['id_str'])}, {'$set':{"user_extracted": True }}, upsert=False)
def ed_friend_num(dbname, comname, netname, flag): ''' Only one-round social network are used ''' filter_user = {'level': 1} users = iot.get_values_one_field(dbname, comname, 'id', filter_user) net = gt.load_network(dbname, netname) com = dbt.db_connect_col(dbname, comname) data = [] for uid in users: row = [uid, flag] exist = True try: v = net.vs.find(name=str(uid)) except ValueError: exist = False if exist: followees = set([int(net.vs[v]['name']) for v in net.successors(str(uid))]) followers = set([int(net.vs[v]['name']) for v in net.predecessors(str(uid))]) common = followees.intersection(followers) followees = followees - common followers = followers - common for fids in [followees, followers, common]: if len(fids) > 0: print uid in fids print len(fids) ed_num, noned_num = 0, 0 for fid in fids: ed_flag = profiles_check.check_ed(com.find_one({'id': fid})) if ed_flag: ed_num += 1 else: noned_num += 1 row.extend([ed_num, noned_num]) else: row.extend([None] * 2) data.append(row) return data
""" from ohsn.util import db_util as dbt from ohsn.api import profiles_check db = dbt.db_connect_no_auth('ed') ed_poi = db['poi_ed'] # db = dbt.db_connect_no_auth('echelon') ech_poi = db['poi_ed_all'] # Common users in Echelon and ED # index = 1 # for poi in ed_poi.find({}): # poi_id = poi['id'] # count = ech_poi.count({'id':poi_id}) # if count > 0: # print index, poi_id, poi['screen_name'] # index += 1 # Target users in Echelon but not in ED index = 1 for poi in ech_poi.find({}): poi_id = poi['id'] if profiles_check.check_ed(poi): count = ed_poi.count({'id':poi_id}) if count < 1: print index, poi_id, poi['screen_name'] index += 1
""" from ohsn.util import db_util as dbt from ohsn.api import profiles_check db = dbt.db_connect_no_auth('ed') ed_poi = db['poi_ed'] # db = dbt.db_connect_no_auth('echelon') ech_poi = db['poi_ed_all'] # Common users in Echelon and ED # index = 1 # for poi in ed_poi.find({}): # poi_id = poi['id'] # count = ech_poi.count({'id':poi_id}) # if count > 0: # print index, poi_id, poi['screen_name'] # index += 1 # Target users in Echelon but not in ED index = 1 for poi in ech_poi.find({}): poi_id = poi['id'] if profiles_check.check_ed(poi): count = ed_poi.count({'id': poi_id}) if count < 1: print index, poi_id, poi['screen_name'] index += 1
def eliminate_non_ed(poidb, netdb): for user in poidb.find({'level':1}): if not profiles_check.check_ed(user): poidb.delete_one({'id': user['id']}) netdb.delete_many({'user': user['id']}) netdb.delete_many({'follower': user['id']})