def getSourceList(users,typ,sampleSize,filterN):
tw={}
twc={}
twDetails={}
print users
#we can look up a max of 100 users...
#TO DO / HACK just sample 100 for now, if required...?
if len(users)>100:
users=random.sample(users, 100)
print 'HACK FUDGE, only using 100 users:',users
twd=api.lookup_users(screen_names=users)
for u in twd:
if type(u) is newt.tweepy.models.User:
twc[u.screen_name]=filterN
twDetails[u.screen_name]=u
if sampleSize==0: sampleSize='all'
for user in users:
print "Getting ",typ," of ",user
if typ=="followers":
tmp=newt.getTwitterFollowersDetailsByIDs(api,user,sampleSize)
else:
tmp=newt.getTwitterFriendsDetailsByIDs(api,user,sampleSize)
print "Grabbed ",str(len(tmp)),typ,' for ',user
#tw.extend(tmp)
for t in tmp:
if t in twc:
twc[t]=twc[t]+1
else:
twc[t]=1
twDetails[t]=tmp[t]
#deDupeList=list(set(origList))
#deDupeList=filter(lambda e: e not in origList,origList)
for t in twc:
if twc[t]>=filterN: tw[t]=twDetails[t]
return tw
def handleUser(user):
#get details of followers of user
twd=newt.getTwitterFollowersDetailsByIDs(api,user,sampleSize)
twc={}
twDetails={}
for t in twd:
if t in twc:
twc[t]=twc[t]+1
else:
twc[t]=1
twDetails[t]=twd[t]
outputter(projname+'/followers.csv',twd)
getFriendsProjection(twDetails)
fn=projname+'/friends__outerfriends.gdf'
DG=nwx.directedNetworkFromGDF(fn)
print DG.order(),DG.size()
fn=projname+'/big_netstats.csv'
f=open(fn,'wb+')
writer=csv.writer(f,quoting=csv.QUOTE_ALL)
writer.writerow(['user','indegree','outdegree'])
for n in DG:
writer.writerow( [n, DG.in_degree(n), DG.out_degree(n) ] )
f.close()
fn=projname+'/followers_netstats.csv'
f=open(fn,'wb+')
writer=csv.writer(f,quoting=csv.QUOTE_ALL)
writer.writerow(['user','indegree','outdegree','fullindegree','fulloutdegree'])
filter= [n for n in DG if DG.out_degree(n)>0]
L=DG.subgraph(filter)
for n in L:
writer.writerow( [n, L.in_degree(n), L.out_degree(n),DG.in_degree(n), DG.out_degree(n) ] )
f.close()
fn=projname+'/followersfriends_netstats.csv'
f=open(fn,'wb+')
writer=csv.writer(f,quoting=csv.QUOTE_ALL)
writer.writerow(['user','indegree','outdegree','fullindegree','fulloutdegree'])
filter= [n for n in DG if DG.in_degree(n)>0]
L=DG.subgraph(filter)
for n in L:
writer.writerow( [n,L.in_degree(n), L.out_degree(n),DG.in_degree(n), DG.out_degree(n)] )
f.close()
fn=projname+'/followersfriends10_netstats.csv'
f=open(fn,'wb+')
writer=csv.writer(f,quoting=csv.QUOTE_ALL)
writer.writerow(['user','indegree','outdegree','fullindegree','fulloutdegree'])
filter= [n for n in DG if DG.in_degree(n)>10]
L=DG.subgraph(filter)
for n in L:
writer.writerow( [n, L.in_degree(n), L.out_degree(n),DG.in_degree(n), DG.out_degree(n)] )
f.close()
filterNet(DG)
typ="friends"
try:
extra=sys.argv[3]
except:
extra=-1
try:
sampleSize=int(sys.argv[4])
except:
sampleSize='all'
#----------------------------------------------------------------
tw={}
if typ=="followers":
tw=newt.getTwitterFollowersDetailsByIDs(api,user,sampleSize)
else:
tw=newt.getTwitterFriendsDetailsByIDs(api,user,sampleSize)
f=newt.openTimestampedFile(user+'/'+typ,'sample'+str(sampleSize)+'tweeps.txt')
for tweep in tw:
f.write(str(tweep)+'\n')
f.close()
report("List members:")
for i in tw:
report(tw[i].screen_name)
'''
report("List members:")
for i in tw:
newt.gephiOutputFile(api, 'hashtag-' + tag, tw, 'outerfriends')
newt.gephiOutputFile(api, 'hashtag-' + tag, tw, 'extrafriends')
newt.gephiOutputFile(api, 'hashtag-' + tag, tw, 'outerfollowers')
if followerView:
typ = 'followers'
sampleSize = 195
filterN = 3
twc = {}
twDetails = {}
for tweep in tw:
user = tw[tweep].screen_name
print "Getting followers of ", user
if typ == "followers":
tmp = newt.getTwitterFollowersDetailsByIDs(api, user, sampleSize)
else:
tmp = newt.getTwitterFriendsDetailsByIDs(api, user, sampleSize)
print "Grabbed ", str(len(tmp)), typ, ' for ', user
#tw.extend(tmp)
for t in tmp:
if t in twc:
twc[t] = twc[t] + 1
else:
twc[t] = 1
twDetails[t] = tmp[t]
tw = {}
for t in twc:
if twc[t] >= filterN: tw[t] = twDetails[t]
def getSourceList(users, typ, sampleSize, filterN):
tw = {}
twc = {}
twDetails = {}
print users
#we can look up a max of 100 users...
#TO DO / HACK just sample 100 for now, if required...?
#if len(users)>100:
# users=random.sample(users, 500)
# print 'HACK FUDGE, only using 500 users:',users
if len(users) > 100:
twd = []
#print 'users',users
for l in chunks(users, 100):
#print 'partial',l
tmp = api.lookup_users(screen_names=l)
twd.append(tmp)
else:
twd = api.lookup_users(screen_names=users)
for u in twd:
if type(u) is newt.tweepy.models.User:
twc[u.screen_name] = filterN
twDetails[u.screen_name] = u
fn = projname + '/userdetails.csv'
writer = csv.writer(open(fn, 'wb+'), quoting=csv.QUOTE_ALL)
k = [
'source', 'screen_name', 'name', 'description', 'location',
'time_zone', 'created_at', 'contributors_enabled', 'url',
'listed_count', 'friends_count', 'followers_count', 'statuses_count',
'favourites_count', 'id_str', 'id', 'verified', 'utc_offset',
'profile_image_url', 'protected'
]
writer.writerow(k)
for uu in twDetails:
u = twDetails[uu]
ux = ['']
for x in [
u.screen_name, u.name, u.description, u.location, u.time_zone
]:
if x != None:
ux.append(
unicodedata.normalize('NFKD', unicode(x)).encode(
'ascii', 'ignore'))
else:
ux.append('')
for x in [
u.created_at, u.contributors_enabled, u.url, u.listed_count,
u.friends_count, u.followers_count, u.statuses_count,
u.favourites_count, u.id_str, u.id, u.verified, u.utc_offset,
u.profile_image_url, u.protected
]:
ux.append(x)
try:
writer.writerow(ux)
except:
pass
if sampleSize == 0: sampleSize = 'all'
for user in users:
print "Getting ", typ, " of ", user
if typ == "followers":
tmp = newt.getTwitterFollowersDetailsByIDs(api, user, sampleSize)
else:
tmp = newt.getTwitterFriendsDetailsByIDs(api, user, sampleSize)
print "Grabbed ", str(len(tmp)), typ, ' for ', user
#tw.extend(tmp)
for t in tmp:
if t in twc:
twc[t] = twc[t] + 1
else:
twc[t] = 1
twDetails[t] = tmp[t]
#deDupeList=list(set(origList))
#deDupeList=filter(lambda e: e not in origList,origList)
for t in twc:
if twc[t] >= filterN: tw[t] = twDetails[t]
return tw
