def getSourceList(users,typ,sampleSize,filterN): tw={} twc={} twDetails={} print users #we can look up a max of 100 users... #TO DO / HACK just sample 100 for now, if required...? if len(users)>100: users=random.sample(users, 100) print 'HACK FUDGE, only using 100 users:',users twd=api.lookup_users(screen_names=users) for u in twd: if type(u) is newt.tweepy.models.User: twc[u.screen_name]=filterN twDetails[u.screen_name]=u if sampleSize==0: sampleSize='all' for user in users: print "Getting ",typ," of ",user if typ=="followers": tmp=newt.getTwitterFollowersDetailsByIDs(api,user,sampleSize) else: tmp=newt.getTwitterFriendsDetailsByIDs(api,user,sampleSize) print "Grabbed ",str(len(tmp)),typ,' for ',user #tw.extend(tmp) for t in tmp: if t in twc: twc[t]=twc[t]+1 else: twc[t]=1 twDetails[t]=tmp[t] #deDupeList=list(set(origList)) #deDupeList=filter(lambda e: e not in origList,origList) for t in twc: if twc[t]>=filterN: tw[t]=twDetails[t] return tw
def handleUser(user): #get details of followers of user twd=newt.getTwitterFollowersDetailsByIDs(api,user,sampleSize) twc={} twDetails={} for t in twd: if t in twc: twc[t]=twc[t]+1 else: twc[t]=1 twDetails[t]=twd[t] outputter(projname+'/followers.csv',twd) getFriendsProjection(twDetails) fn=projname+'/friends__outerfriends.gdf' DG=nwx.directedNetworkFromGDF(fn) print DG.order(),DG.size() fn=projname+'/big_netstats.csv' f=open(fn,'wb+') writer=csv.writer(f,quoting=csv.QUOTE_ALL) writer.writerow(['user','indegree','outdegree']) for n in DG: writer.writerow( [n, DG.in_degree(n), DG.out_degree(n) ] ) f.close() fn=projname+'/followers_netstats.csv' f=open(fn,'wb+') writer=csv.writer(f,quoting=csv.QUOTE_ALL) writer.writerow(['user','indegree','outdegree','fullindegree','fulloutdegree']) filter= [n for n in DG if DG.out_degree(n)>0] L=DG.subgraph(filter) for n in L: writer.writerow( [n, L.in_degree(n), L.out_degree(n),DG.in_degree(n), DG.out_degree(n) ] ) f.close() fn=projname+'/followersfriends_netstats.csv' f=open(fn,'wb+') writer=csv.writer(f,quoting=csv.QUOTE_ALL) writer.writerow(['user','indegree','outdegree','fullindegree','fulloutdegree']) filter= [n for n in DG if DG.in_degree(n)>0] L=DG.subgraph(filter) for n in L: writer.writerow( [n,L.in_degree(n), L.out_degree(n),DG.in_degree(n), DG.out_degree(n)] ) f.close() fn=projname+'/followersfriends10_netstats.csv' f=open(fn,'wb+') writer=csv.writer(f,quoting=csv.QUOTE_ALL) writer.writerow(['user','indegree','outdegree','fullindegree','fulloutdegree']) filter= [n for n in DG if DG.in_degree(n)>10] L=DG.subgraph(filter) for n in L: writer.writerow( [n, L.in_degree(n), L.out_degree(n),DG.in_degree(n), DG.out_degree(n)] ) f.close() filterNet(DG)
typ="friends" try: extra=sys.argv[3] except: extra=-1 try: sampleSize=int(sys.argv[4]) except: sampleSize='all' #---------------------------------------------------------------- tw={} if typ=="followers": tw=newt.getTwitterFollowersDetailsByIDs(api,user,sampleSize) else: tw=newt.getTwitterFriendsDetailsByIDs(api,user,sampleSize) f=newt.openTimestampedFile(user+'/'+typ,'sample'+str(sampleSize)+'tweeps.txt') for tweep in tw: f.write(str(tweep)+'\n') f.close() report("List members:") for i in tw: report(tw[i].screen_name) ''' report("List members:") for i in tw:
newt.gephiOutputFile(api, 'hashtag-' + tag, tw, 'outerfriends') newt.gephiOutputFile(api, 'hashtag-' + tag, tw, 'extrafriends') newt.gephiOutputFile(api, 'hashtag-' + tag, tw, 'outerfollowers') if followerView: typ = 'followers' sampleSize = 195 filterN = 3 twc = {} twDetails = {} for tweep in tw: user = tw[tweep].screen_name print "Getting followers of ", user if typ == "followers": tmp = newt.getTwitterFollowersDetailsByIDs(api, user, sampleSize) else: tmp = newt.getTwitterFriendsDetailsByIDs(api, user, sampleSize) print "Grabbed ", str(len(tmp)), typ, ' for ', user #tw.extend(tmp) for t in tmp: if t in twc: twc[t] = twc[t] + 1 else: twc[t] = 1 twDetails[t] = tmp[t] tw = {} for t in twc: if twc[t] >= filterN: tw[t] = twDetails[t]
def getSourceList(users, typ, sampleSize, filterN): tw = {} twc = {} twDetails = {} print users #we can look up a max of 100 users... #TO DO / HACK just sample 100 for now, if required...? #if len(users)>100: # users=random.sample(users, 500) # print 'HACK FUDGE, only using 500 users:',users if len(users) > 100: twd = [] #print 'users',users for l in chunks(users, 100): #print 'partial',l tmp = api.lookup_users(screen_names=l) twd.append(tmp) else: twd = api.lookup_users(screen_names=users) for u in twd: if type(u) is newt.tweepy.models.User: twc[u.screen_name] = filterN twDetails[u.screen_name] = u fn = projname + '/userdetails.csv' writer = csv.writer(open(fn, 'wb+'), quoting=csv.QUOTE_ALL) k = [ 'source', 'screen_name', 'name', 'description', 'location', 'time_zone', 'created_at', 'contributors_enabled', 'url', 'listed_count', 'friends_count', 'followers_count', 'statuses_count', 'favourites_count', 'id_str', 'id', 'verified', 'utc_offset', 'profile_image_url', 'protected' ] writer.writerow(k) for uu in twDetails: u = twDetails[uu] ux = [''] for x in [ u.screen_name, u.name, u.description, u.location, u.time_zone ]: if x != None: ux.append( unicodedata.normalize('NFKD', unicode(x)).encode( 'ascii', 'ignore')) else: ux.append('') for x in [ u.created_at, u.contributors_enabled, u.url, u.listed_count, u.friends_count, u.followers_count, u.statuses_count, u.favourites_count, u.id_str, u.id, u.verified, u.utc_offset, u.profile_image_url, u.protected ]: ux.append(x) try: writer.writerow(ux) except: pass if sampleSize == 0: sampleSize = 'all' for user in users: print "Getting ", typ, " of ", user if typ == "followers": tmp = newt.getTwitterFollowersDetailsByIDs(api, user, sampleSize) else: tmp = newt.getTwitterFriendsDetailsByIDs(api, user, sampleSize) print "Grabbed ", str(len(tmp)), typ, ' for ', user #tw.extend(tmp) for t in tmp: if t in twc: twc[t] = twc[t] + 1 else: twc[t] = 1 twDetails[t] = tmp[t] #deDupeList=list(set(origList)) #deDupeList=filter(lambda e: e not in origList,origList) for t in twc: if twc[t] >= filterN: tw[t] = twDetails[t] return tw