def collectMentions(self): geo = Geo() tweetCount = 0 tweetErrorCount = 0 totalFiles = len(os.listdir(self.inputFolder)) fileCount = 1 for _file in sorted(os.listdir(self.inputFolder)): fileDate = datetime.strptime(_file[17:27], "%Y-%m-%d") if(fileDate > self.toDate or fileDate < self.fromDate): continue log.debug("processing file %d/%d-->%s" % (fileCount, totalFiles, _file)) fileCount += 1 try: with open(self.inputFolder + "/" + _file, "r") as FILE: for line in FILE: try: jsonTweet = json.loads(line.strip()) geoList = geo.geo_normalize(jsonTweet) city = geoList[0] country = geoList[1] state = geoList[2] if ((self.city == '-' and self.state == '-' and country and country.lower() == self.country) or (country and country.lower() == self.country and state and state.lower() == self.state) or (country and country.lower() == self.country and state and state.lower() == self.state and city and city.lower() == self.city)): tweetCount += 1 # use [5:25] if need HH:MM:SS datestr = jsonTweet["interaction"]["created_at"][5:16] klout, sentiment = getKloutSentiment(jsonTweet) tweeterId = jsonTweet["interaction"]["author"]["id"] candidatesFound = self.processTweet(jsonTweet) self.updateScoreCard(candidatesFound, tweeterId, klout, sentiment, datestr) except Exception, f: log.exception("error processing tweets %s", f) tweetErrorCount += 1 except Exception, e: log.exception("error processfing file %s", e)
def execute(arg): logs.init(arg) fromDate = datetime.strptime(arg.fromDate, "%d %b %Y") toDate = datetime.strptime(arg.toDate, "%d %b %Y") tweetFolder = arg.tweetFolder country = arg.country hashTagCounts = {} uids = {} # loading twitter handles from a file with open(arg.seedFile, 'r') as _file: for line in _file: handle, candidate = line.strip().split(',') if candidate not in uids: uids[candidate] = [] hashTagCounts[candidate] = {} uids[candidate].append(handle.lower()) else: uids[candidate].append(handle.lower()) # for geolocation geo = Geo() for _file in sorted(os.listdir(tweetFolder)): fileDate = datetime.strptime(_file[17:27], '%Y-%m-%d') if (fileDate >= fromDate and fileDate < toDate): log.info("processing file %s" % (_file)) try: with open(tweetFolder + "/" + _file, "r") as FILE: for line in FILE: try: jsonTweet = json.loads(line.strip()) dateStr = jsonTweet['interaction']['created_at'][5:16] tweetDate = datetime.strptime(dateStr, '%d %b %Y') geoList = geo.geo_normalize(jsonTweet) city, ctry, state = geoList[:3] if ctry and (ctry.lower() == country) and (tweetDate >= fromDate) and (tweetDate <= toDate): userId, realName = None, None if 'twiiter' in jsonTweet: if 'user' in jsonTweet['twitter']: if 'screen_name' in jsonTweet['twitter']['user']: userId = jsonTweet['twitter']['user']['screen_name'].lower() if 'name' in jsonTweet['twitter']['user']: realName = jsonTweet['twitter']['user']['name'].lower() if userId is None and realName is None: continue log.debug('userId or realName is not None') candidate = getCandidate(userId, realName, uids) if candidate is not None: log.debug('found candidate--> ' + candidate) # prereProcess the tweet text = jsonTweet["interaction"]["content"] text = re.sub(URL_REGEX, ' ', text) # remove urls text = re.sub('[^A-Za-z_@#0-9]', ' ', normalize_str(text, lower=True)) # allow only alphaNumerics and twitter tags text = re.sub(' +', ' ', text) # remove multiple spaces hashTags = extract_hash_tags(text) hashTags = [hashTag for hashTag in hashTags if len(hashTag) > 3] for hashTag in hashTags: if hashTag.startswith('#'): hashTag = hashTag[1:] if hashTag in hashTagCounts[candidate]: hashTagCounts[candidate][hashTag] += 1 else: hashTagCounts[candidate][hashTag] = 1 except Exception, e: log.exception('error processing tweet %s' % e) except Exception, f: log.exception('error processing file %s' % f) else: log.debug('skipping file %s ' % _file)
def preProcess(tweetFolder, outputFolder, keywordList, fromDate, toDate, country): log.info("inside preProcess") log.debug("fromDate-->" + fromDate.strftime("%d %b %Y")) log.debug("toDate-->" + toDate.strftime("%d %b %Y")) tweets = {} # output files tweetedFile = open(outputFolder + '/tweeted.csv', 'w') mentionFile = open(outputFolder + '/mentioned.csv', 'w') # retweetFile = open(outputFolder + '/retweet.csv', 'w') wordsFile = open(outputFolder + '/containsWord.csv', 'w') sentimentFile = open(outputFolder + '/sentiment.csv', 'w') tweetsFile = open(outputFolder + '/tweets.json', 'w') # build stop word list # englishStopWords = [normalize_str(w).lower() for w in stopwords.words('english')] # spanishStopWords = [normalize_str(w).lower() for w in stopwords.words('spanish')] # stopWordList = [] # stopWordList.extend(englishStopWords) # stopWordList.extend(spanishStopWords) log.info("# of keywords: " + str(len(keywordList))) log.info("tracking--> " + str(keywordList)) # build regular expression for keyword keywordRegex = re.compile(r'\b%s\b' % '\\b|\\b'.join(keywordList), flags=re.IGNORECASE) # for geocoding tweets geo = Geo() tweetCount, tweetErrorCount = 0, 0 for _file in sorted(os.listdir(tweetFolder)): fileDate = datetime.strptime(_file[17:27], '%Y-%m-%d') if (fileDate >= fromDate and fileDate < toDate): log.info("processing file %s" % (_file)) try: with open(tweetFolder + "/" + _file, "r") as FILE: for line in FILE: try: jsonTweet = json.loads(line.strip()) dateStr = jsonTweet['interaction']['created_at'][5:16] tweetDate = datetime.strptime(dateStr, '%d %b %Y') geoList = geo.geo_normalize(jsonTweet) city, ctry, state = geoList[:3] if ctry and (ctry.lower() == country) and (tweetDate >= fromDate) and (tweetDate <= toDate): # prereProcess the tweet text = jsonTweet["interaction"]["content"] text = re.sub(URL_REGEX, ' ', text) # remove urls text = re.sub('[^A-Za-z_@#0-9]', ' ', normalize_str(text, lower=True)) # allow only alphaNumerics and twitter tags text = re.sub(' +', ' ', text) # remove multiple spaces keywordsPresent = re.findall(keywordRegex, text) keywordsPresent = list(set(keywordsPresent)) if len(keywordsPresent) > 0: tweetId = jsonTweet["twitter"]["id"] tweeterId = str(jsonTweet["interaction"]["author"]["id"]) mentions = getInteractions(jsonTweet) sentiment = getSentiment(jsonTweet) hashTags = extract_hash_tags(text) hashTags = [hashTag for hashTag in hashTags if len(hashTag) > 3] #hashTags.extend(keywordsPresent) if len(hashTags) == 0: continue hashTags = list(set(hashTags).union(set(keywordsPresent))) tweetedFile.write(tweeterId + ',' + tweetId + '\n') sentimentFile.write(tweetId + ',' + str(sentiment) + '\n') for userId in mentions: mentionFile.write(tweetId + ',' + userId + '\n') # for userId in retweets: # retweetFile.write(tweetId + ',' + userId + '\n') for hashTag in hashTags: if hashTag.startswith('#'): hashTag = hashTag[1:] wordsFile.write(tweetId + ',' + hashTag + '\n') # tracking the tweets for checks. if tweeterId in tweets: tweets[tweeterId][tweetId] = jsonTweet["interaction"]["content"] else: tweets[tweeterId] = {} tweets[tweeterId][tweetId] = jsonTweet["interaction"]["content"] tweetCount += 1 except Exception, f: log.debug("error processing tweet %s", f) tweetErrorCount += 1 except Exception, e: log.exception("error processfing file %s", e) else: log.debug("skipping file %s" % (_file))
def trackTweets(tweetFolder, vocab, fromDate, toDate, country, threshold): counts = {} regex = {} totalWords = 0 # building regex for each group for group in vocab: counts[group] = {} sorted_tuples = sorted(vocab[group].iteritems(), key=operator.itemgetter(1), reverse=True) words = [] if len(sorted_tuples) <= 20: threshold = len(sorted_tuples) else: threshold = int(len(sorted_tuples) * threshold // 100) for (word, weight) in sorted_tuples[:threshold]: words.append(word) totalWords += 1 regex[group] = re.compile(r"\b%s\b" % "\\b|\\b".join(words), flags=re.IGNORECASE) log.info("tracking total of %d words" % totalWords) # for geoCoding tweets geo = Geo() tweetCount, tweetErrorCount = 0, 0 for _file in sorted(os.listdir(tweetFolder)): fileDate = datetime.strptime(_file[17:27], "%Y-%m-%d") if fileDate >= fromDate and fileDate < toDate: log.info("processing file %s" % (_file)) try: with open(tweetFolder + "/" + _file, "r") as FILE: for line in FILE: try: jsonTweet = json.loads(line.strip()) dateStr = jsonTweet["interaction"]["created_at"][5:16] tweetDate = datetime.strptime(dateStr, "%d %b %Y") geoList = geo.geo_normalize(jsonTweet) city, ctry, state = geoList[:3] if ctry and (ctry.lower() == country) and (tweetDate >= fromDate) and (tweetDate <= toDate): # prereProcess the tweet text = jsonTweet["interaction"]["content"] text = re.sub(URL_REGEX, " ", text) # remove urls text = re.sub( "[^A-Za-z_@#0-9]", " ", normalize_str(text, lower=True) ) # allow only alphaNumerics and twitter tags text = re.sub(" +", " ", text) # remove multiple spaces for group in regex: keywordsPresent = re.findall(regex[group], text) if len(keywordsPresent) > 0: keywordsPresent = list(set(keywordsPresent)) hashTags = extract_hash_tags(text) hashTags = [hashTag for hashTag in hashTags if len(hashTag) > 3] hashTags.extend(keywordsPresent) for hashTag in hashTags: if hashTag.startswith("#"): hashTag = hashTag[1:] if hashTag in counts[group]: counts[group][hashTag] += 1 else: counts[group][hashTag] = 1 tweetCount += 1 except Exception, f: log.debug("error processing tweet %s", f) tweetErrorCount += 1 except Exception, e: log.exception("error processfing file %s", e) else: log.debug("skipping file %s" % (_file))
def preProcess(tweetFolder, outputFolder, keywordList, fromDate, toDate, country, filesProcessed): log.info("inside preProcess") log.debug("fromDate-->" + fromDate.strftime("%d %b %Y")) log.debug("toDate-->" + toDate.strftime("%d %b %Y")) tweetCount, tweetErrorCount = 0, 0 tweets = {} # output files tweetedFile = open(outputFolder + '/tweeted.csv', 'w') mentionFile = open(outputFolder + '/mentioned.csv', 'w') retweetFile = open(outputFolder + '/retweet.csv', 'w') wordsFile = open(outputFolder + '/containsWord.csv', 'w') sentimentFile = open(outputFolder + '/sentiment.csv', 'w') tweetsFile = open(outputFolder + '/tweets.json', 'w') # build stop word list englishStopWords = [normalize_str(w).lower() for w in stopwords.words('english')] spanishStopWords = [normalize_str(w).lower() for w in stopwords.words('spanish')] stopWordList = [] stopWordList.extend(englishStopWords) stopWordList.extend(spanishStopWords) log.info("# of keywords: " + str(len(keywordList))) log.info("tracking--> " + str(keywordList)) # build regular expression for keyword keywordRegex = re.compile(r'\b%s\b' % '\\b|\\b'.join(keywordList), flags=re.IGNORECASE) # for geocoding tweets geo = Geo() log.info("filesProcessed-->" + str(filesProcessed)) for _file in sorted(os.listdir(tweetFolder)): fileDate = datetime.strptime(_file[17:27], '%Y-%m-%d') if (_file not in filesProcessed and fileDate >= fromDate and fileDate < toDate): log.info("processing file %s" % (_file)) try: with open(tweetFolder + "/" + _file, "r") as FILE: tweetCount, tweetErrorCount = 0, 0 for line in FILE: try: jsonTweet = json.loads(line.strip()) dateStr = jsonTweet['interaction']['created_at'][5:16] tweetDate = datetime.strptime(dateStr, '%d %b %Y') sentiment = getSentiment(jsonTweet) if sentiment == 0: continue geoList = geo.geo_normalize(jsonTweet) ctry, a1, a2, a3 = geoList[1:5] if ctry and (ctry.lower() == country) and (tweetDate >= fromDate) and (tweetDate <= toDate): text = jsonTweet["interaction"]["content"] text = re.sub(URL_REGEX, ' ', text) # remove urls text = re.sub('[^A-Za-z_@#0-9]', ' ', normalize_str(text, lower=True)) # allow only alphaNumerics and twitter tags text = re.sub(' +', ' ', text) # remove multiple spaces keywordsPresent = keywordRegex.search(text) if keywordsPresent is not None: words = text.split(" ") words = [w for w in words if len(w) > 2 and w not in stopWordList] words2 = [] for word in words: for w in word: if (word not in keywordList) and (w.isdigit() or w == '@'): break else: if word[0] == '#': word = word[1:] words2.append(word) tweetId = jsonTweet["twitter"]["id"] tweeterId = str(jsonTweet["interaction"]["author"]["id"]) mentions, retweets = getInteractions(jsonTweet) tweetedFile.write(tweeterId + ',' + tweetId + '\n') sentimentFile.write(tweetId + ',' + str(sentiment) + '\n') for userId in mentions: mentionFile.write(tweetId + ',' + userId + '\n') for userId in retweets: retweetFile.write(tweetId + ',' + userId + '\n') for word in words2: wordsFile.write(tweetId + ',' + word + '\n') # tracking the tweets for checks. if tweeterId in tweets: tweets[tweeterId][tweetId] = jsonTweet["interaction"]["content"] else: tweets[tweeterId] = {} tweets[tweeterId][tweetId] = jsonTweet["interaction"]["content"] tweetCount += 1 except Exception, f: log.exception("error processing tweet %s", f) tweetErrorCount += 1 except Exception, e: log.exception("error processfing file %s", e) log.info("tweets used: %s" % str(tweetCount)) log.debug("tweetErrorCount : %s" % str(tweetErrorCount)) filesProcessed.append(_file) break else: log.debug("skipping file %s" % (_file))
ap.add_argument('-t', '--tweetFolder', type=str, help='inputFolder pointing to PSLs output', default='/hdd/tweets/2012/oct') ap.add_argument('-c', '--country', type=str) ap.add_argument('-m', '--month', type=str) arg = ap.parse_args() logs.init(arg) geo = Geo() tweetCount = 0 date = datetime.strptime(arg.month, "%b %Y") for _file in os.listdir(arg.tweetFolder): try: with open(arg.tweetFolder + "/" + _file, "r") as FILE: for line in FILE: try: jsonTweet = json.loads(line.strip()) dateStr = jsonTweet['interaction']['created_at'][5:16] tweetDate = datetime.strptime(dateStr, '%d %b %Y') geoList = geo.geo_normalize(jsonTweet) city, ctry, state = geoList[:3] if ctry and ctry.lower() == arg.country.lower() and date.month == tweetDate.month and date.year == tweetDate.year: tweetCount += 1 except Exception, f: _traceback = sys.exc_info()[2] log.exception("error processing tweet %s @line %s" % (f, traceback.tb_lineno(_traceback))) log.info("tweet Count -->" + str(tweetCount)) except: log.exception('error opening file') print ('# tweets for %s in %s --> %d' % (arg.country, arg.month, tweetCount))