def getLayers(api, numLayers, options, userlist, olduserlist=set()): for layer in range(0, numLayers): log.log(log.info, "Beginning data collection for layer " + str(layer)) if (layer > 0): oldRTs = loadUserList(options.workdir, "layer" + str(layer - 1) + "retweetedUsers") oldMentions = loadUserList( options.workdir, "layer" + str(layer - 1) + "mentionedUsers") rtUsernames = flattenUserDictionary(oldRTs) mentionUsernames = flattenUserDictionary(oldMentions) userlist = list(rtUsernames.union(mentionUsernames)) nextLayerRTs = dict() nextLayerMentions = dict() for username in userlist: if (not userTweetsPresent(username, options.tweetdir)): getUserTweets(api, username, options.tweetdir, options.numtweets, options.compress) if (not username in olduserlist): olduserlist.add(username) mentions, rts = getUserReferences(username, options.tweetdir) if (len(rts) > 0): nextLayerRTs[username] = list(rts) if (len(mentions) > 0): nextLayerMentions[username] = list(mentions) log.log( log.info, "Layer " + str(layer) + " data collection complete, saving user lists...") saveUserList(options.workdir, "layer" + str(layer) + "mentionedUsers", nextLayerMentions) saveUserList(options.workdir, "layer" + str(layer) + "retweetedUsers", nextLayerRTs) log.log(log.info, "Saving network to disk...") analyze.saveNetwork(options.mapdir, layer, userlist, nextLayerRTs, nextLayerMentions)
tweetDir = sys.argv[2] numLayers = int(sys.argv[3]) outFileName = sys.argv[4] if (numLayers < 1): print("ERROR: Map must include at least one layer") sys.exit(1) # Work in a random temp directory so we can run multiple instances of # this script at once workDir = tempfile.TemporaryDirectory() workDirName = workDir.name # Setup is done, let's read all the tweets we've downloaded userlist = origUserlist for layer in range(0, numLayers): layerMentioned = dict() print("Layer %d: Reading data on %d users" % (layer, len(userlist))) for username in userlist: if (acquire.userTweetsPresent(username, tweetDir)): mentions, rts = acquire.getUserReferences(username, tweetDir) layerMentioned[username] = list(mentions) print("Layer %d: Saving map" % layer) userlist = acquire.flattenUserDictionary(layerMentioned) analyze.saveNetwork(workDirName, layer, origUserlist, dict(), layerMentioned) # Now move the final map over to the user-requested location origFileName = workDirName + "/layer" + str(numLayers) + ".gml" shutil.move(origFileName, outFileName)
origUserlist = socmap.getUsernames(sys.argv[1]) tweetDir = sys.argv[2] numLayers = int(sys.argv[3]) outFileName = sys.argv[4] if (numLayers < 1): print("ERROR: Map must include at least one layer") sys.exit(1) # Work in a random temp directory so we can run multiple instances of # this script at once workDir = tempfile.TemporaryDirectory() workDirName = workDir.name # Setup is done, let's read all the tweets we've downloaded userlist = origUserlist for layer in range(0, numLayers): layerRTs = dict() print("Layer %d: Reading data on %d users" % (layer, len(userlist))) for username in userlist: if (acquire.userTweetsPresent(username, tweetDir)): mentions, rts = acquire.getUserReferences(username, tweetDir) layerRTs[username] = list(rts) print("Layer %d: Saving map" % layer) userlist = acquire.flattenUserDictionary(layerRTs) analyze.saveNetwork(workDirName, layer, origUserlist, layerRTs, dict()) # Now move the final map over to the user-requested location origFileName = workDirName + "/layer" + str(numLayers) + ".gml" shutil.move(origFileName, outFileName)