def getLayers(api, numLayers, options, userlist, olduserlist=set()):
for layer in range(0, numLayers):
log.log(log.info, "Beginning data collection for layer " + str(layer))
if (layer > 0):
oldRTs = loadUserList(options.workdir,
"layer" + str(layer - 1) + "retweetedUsers")
oldMentions = loadUserList(
options.workdir, "layer" + str(layer - 1) + "mentionedUsers")
rtUsernames = flattenUserDictionary(oldRTs)
mentionUsernames = flattenUserDictionary(oldMentions)
userlist = list(rtUsernames.union(mentionUsernames))
nextLayerRTs = dict()
nextLayerMentions = dict()
for username in userlist:
if (not userTweetsPresent(username, options.tweetdir)):
getUserTweets(api, username, options.tweetdir,
options.numtweets, options.compress)
if (not username in olduserlist):
olduserlist.add(username)
mentions, rts = getUserReferences(username, options.tweetdir)
if (len(rts) > 0):
nextLayerRTs[username] = list(rts)
if (len(mentions) > 0):
nextLayerMentions[username] = list(mentions)
log.log(
log.info, "Layer " + str(layer) +
" data collection complete, saving user lists...")
saveUserList(options.workdir, "layer" + str(layer) + "mentionedUsers",
nextLayerMentions)
saveUserList(options.workdir, "layer" + str(layer) + "retweetedUsers",
nextLayerRTs)
log.log(log.info, "Saving network to disk...")
analyze.saveNetwork(options.mapdir, layer, userlist, nextLayerRTs,
nextLayerMentions)
tweetDir = sys.argv[2]
numLayers = int(sys.argv[3])
outFileName = sys.argv[4]
if (numLayers < 1):
print("ERROR: Map must include at least one layer")
sys.exit(1)
# Work in a random temp directory so we can run multiple instances of
# this script at once
workDir = tempfile.TemporaryDirectory()
workDirName = workDir.name
# Setup is done, let's read all the tweets we've downloaded
userlist = origUserlist
for layer in range(0, numLayers):
layerMentioned = dict()
print("Layer %d: Reading data on %d users" % (layer, len(userlist)))
for username in userlist:
if (acquire.userTweetsPresent(username, tweetDir)):
mentions, rts = acquire.getUserReferences(username, tweetDir)
layerMentioned[username] = list(mentions)
print("Layer %d: Saving map" % layer)
userlist = acquire.flattenUserDictionary(layerMentioned)
analyze.saveNetwork(workDirName, layer, origUserlist, dict(),
layerMentioned)
# Now move the final map over to the user-requested location
origFileName = workDirName + "/layer" + str(numLayers) + ".gml"
shutil.move(origFileName, outFileName)
origUserlist = socmap.getUsernames(sys.argv[1])
tweetDir = sys.argv[2]
numLayers = int(sys.argv[3])
outFileName = sys.argv[4]
if (numLayers < 1):
print("ERROR: Map must include at least one layer")
sys.exit(1)
# Work in a random temp directory so we can run multiple instances of
# this script at once
workDir = tempfile.TemporaryDirectory()
workDirName = workDir.name
# Setup is done, let's read all the tweets we've downloaded
userlist = origUserlist
for layer in range(0, numLayers):
layerRTs = dict()
print("Layer %d: Reading data on %d users" % (layer, len(userlist)))
for username in userlist:
if (acquire.userTweetsPresent(username, tweetDir)):
mentions, rts = acquire.getUserReferences(username, tweetDir)
layerRTs[username] = list(rts)
print("Layer %d: Saving map" % layer)
userlist = acquire.flattenUserDictionary(layerRTs)
analyze.saveNetwork(workDirName, layer, origUserlist, layerRTs, dict())
# Now move the final map over to the user-requested location
origFileName = workDirName + "/layer" + str(numLayers) + ".gml"
shutil.move(origFileName, outFileName)