def collectMentions(self):
geo = Geo()
tweetCount = 0
tweetErrorCount = 0
totalFiles = len(os.listdir(self.inputFolder))
fileCount = 1
for _file in sorted(os.listdir(self.inputFolder)):
fileDate = datetime.strptime(_file[17:27], "%Y-%m-%d")
if(fileDate > self.toDate or fileDate < self.fromDate):
continue
log.debug("processing file %d/%d-->%s" % (fileCount, totalFiles, _file))
fileCount += 1
try:
with open(self.inputFolder + "/" + _file, "r") as FILE:
for line in FILE:
try:
jsonTweet = json.loads(line.strip())
geoList = geo.geo_normalize(jsonTweet)
city = geoList[0]
country = geoList[1]
state = geoList[2]
if ((self.city == '-' and self.state == '-' and country and country.lower() == self.country) or
(country and country.lower() == self.country and state and state.lower() == self.state) or
(country and country.lower() == self.country and state and state.lower() == self.state and city and city.lower() == self.city)):
tweetCount += 1
# use [5:25] if need HH:MM:SS
datestr = jsonTweet["interaction"]["created_at"][5:16]
klout, sentiment = getKloutSentiment(jsonTweet)
tweeterId = jsonTweet["interaction"]["author"]["id"]
candidatesFound = self.processTweet(jsonTweet)
self.updateScoreCard(candidatesFound,
tweeterId, klout,
sentiment, datestr)
except Exception, f:
log.exception("error processing tweets %s", f)
tweetErrorCount += 1
except Exception, e:
log.exception("error processfing file %s", e)
def execute(arg):
logs.init(arg)
fromDate = datetime.strptime(arg.fromDate, "%d %b %Y")
toDate = datetime.strptime(arg.toDate, "%d %b %Y")
tweetFolder = arg.tweetFolder
country = arg.country
hashTagCounts = {}
uids = {}
# loading twitter handles from a file
with open(arg.seedFile, 'r') as _file:
for line in _file:
handle, candidate = line.strip().split(',')
if candidate not in uids:
uids[candidate] = []
hashTagCounts[candidate] = {}
uids[candidate].append(handle.lower())
else:
uids[candidate].append(handle.lower())
# for geolocation
geo = Geo()
for _file in sorted(os.listdir(tweetFolder)):
fileDate = datetime.strptime(_file[17:27], '%Y-%m-%d')
if (fileDate >= fromDate and fileDate < toDate):
log.info("processing file %s" % (_file))
try:
with open(tweetFolder + "/" + _file, "r") as FILE:
for line in FILE:
try:
jsonTweet = json.loads(line.strip())
dateStr = jsonTweet['interaction']['created_at'][5:16]
tweetDate = datetime.strptime(dateStr, '%d %b %Y')
geoList = geo.geo_normalize(jsonTweet)
city, ctry, state = geoList[:3]
if ctry and (ctry.lower() == country) and (tweetDate >= fromDate) and (tweetDate <= toDate):
userId, realName = None, None
if 'twiiter' in jsonTweet:
if 'user' in jsonTweet['twitter']:
if 'screen_name' in jsonTweet['twitter']['user']:
userId = jsonTweet['twitter']['user']['screen_name'].lower()
if 'name' in jsonTweet['twitter']['user']:
realName = jsonTweet['twitter']['user']['name'].lower()
if userId is None and realName is None:
continue
log.debug('userId or realName is not None')
candidate = getCandidate(userId, realName, uids)
if candidate is not None:
log.debug('found candidate--> ' + candidate)
# prereProcess the tweet
text = jsonTweet["interaction"]["content"]
text = re.sub(URL_REGEX, ' ', text) # remove urls
text = re.sub('[^A-Za-z_@#0-9]', ' ', normalize_str(text, lower=True)) # allow only alphaNumerics and twitter tags
text = re.sub(' +', ' ', text) # remove multiple spaces
hashTags = extract_hash_tags(text)
hashTags = [hashTag for hashTag in hashTags if len(hashTag) > 3]
for hashTag in hashTags:
if hashTag.startswith('#'):
hashTag = hashTag[1:]
if hashTag in hashTagCounts[candidate]:
hashTagCounts[candidate][hashTag] += 1
else:
hashTagCounts[candidate][hashTag] = 1
except Exception, e:
log.exception('error processing tweet %s' % e)
except Exception, f:
log.exception('error processing file %s' % f)
else:
log.debug('skipping file %s ' % _file)
def preProcess(tweetFolder, outputFolder, keywordList, fromDate, toDate, country):
log.info("inside preProcess")
log.debug("fromDate-->" + fromDate.strftime("%d %b %Y"))
log.debug("toDate-->" + toDate.strftime("%d %b %Y"))
tweets = {}
# output files
tweetedFile = open(outputFolder + '/tweeted.csv', 'w')
mentionFile = open(outputFolder + '/mentioned.csv', 'w')
# retweetFile = open(outputFolder + '/retweet.csv', 'w')
wordsFile = open(outputFolder + '/containsWord.csv', 'w')
sentimentFile = open(outputFolder + '/sentiment.csv', 'w')
tweetsFile = open(outputFolder + '/tweets.json', 'w')
# build stop word list
# englishStopWords = [normalize_str(w).lower() for w in stopwords.words('english')]
# spanishStopWords = [normalize_str(w).lower() for w in stopwords.words('spanish')]
# stopWordList = []
# stopWordList.extend(englishStopWords)
# stopWordList.extend(spanishStopWords)
log.info("# of keywords: " + str(len(keywordList)))
log.info("tracking--> " + str(keywordList))
# build regular expression for keyword
keywordRegex = re.compile(r'\b%s\b' % '\\b|\\b'.join(keywordList),
flags=re.IGNORECASE)
# for geocoding tweets
geo = Geo()
tweetCount, tweetErrorCount = 0, 0
for _file in sorted(os.listdir(tweetFolder)):
fileDate = datetime.strptime(_file[17:27], '%Y-%m-%d')
if (fileDate >= fromDate and fileDate < toDate):
log.info("processing file %s" % (_file))
try:
with open(tweetFolder + "/" + _file, "r") as FILE:
for line in FILE:
try:
jsonTweet = json.loads(line.strip())
dateStr = jsonTweet['interaction']['created_at'][5:16]
tweetDate = datetime.strptime(dateStr, '%d %b %Y')
geoList = geo.geo_normalize(jsonTweet)
city, ctry, state = geoList[:3]
if ctry and (ctry.lower() == country) and (tweetDate >= fromDate) and (tweetDate <= toDate):
# prereProcess the tweet
text = jsonTweet["interaction"]["content"]
text = re.sub(URL_REGEX, ' ', text) # remove urls
text = re.sub('[^A-Za-z_@#0-9]', ' ', normalize_str(text, lower=True)) # allow only alphaNumerics and twitter tags
text = re.sub(' +', ' ', text) # remove multiple spaces
keywordsPresent = re.findall(keywordRegex, text)
keywordsPresent = list(set(keywordsPresent))
if len(keywordsPresent) > 0:
tweetId = jsonTweet["twitter"]["id"]
tweeterId = str(jsonTweet["interaction"]["author"]["id"])
mentions = getInteractions(jsonTweet)
sentiment = getSentiment(jsonTweet)
hashTags = extract_hash_tags(text)
hashTags = [hashTag for hashTag in hashTags if len(hashTag) > 3]
#hashTags.extend(keywordsPresent)
if len(hashTags) == 0:
continue
hashTags = list(set(hashTags).union(set(keywordsPresent)))
tweetedFile.write(tweeterId + ',' + tweetId + '\n')
sentimentFile.write(tweetId + ',' + str(sentiment) + '\n')
for userId in mentions:
mentionFile.write(tweetId + ',' + userId + '\n')
# for userId in retweets:
# retweetFile.write(tweetId + ',' + userId + '\n')
for hashTag in hashTags:
if hashTag.startswith('#'):
hashTag = hashTag[1:]
wordsFile.write(tweetId + ',' + hashTag + '\n')
# tracking the tweets for checks.
if tweeterId in tweets:
tweets[tweeterId][tweetId] = jsonTweet["interaction"]["content"]
else:
tweets[tweeterId] = {}
tweets[tweeterId][tweetId] = jsonTweet["interaction"]["content"]
tweetCount += 1
except Exception, f:
log.debug("error processing tweet %s", f)
tweetErrorCount += 1
except Exception, e:
log.exception("error processfing file %s", e)
else:
log.debug("skipping file %s" % (_file))
def trackTweets(tweetFolder, vocab, fromDate, toDate, country, threshold):
counts = {}
regex = {}
totalWords = 0
# building regex for each group
for group in vocab:
counts[group] = {}
sorted_tuples = sorted(vocab[group].iteritems(), key=operator.itemgetter(1), reverse=True)
words = []
if len(sorted_tuples) <= 20:
threshold = len(sorted_tuples)
else:
threshold = int(len(sorted_tuples) * threshold // 100)
for (word, weight) in sorted_tuples[:threshold]:
words.append(word)
totalWords += 1
regex[group] = re.compile(r"\b%s\b" % "\\b|\\b".join(words), flags=re.IGNORECASE)
log.info("tracking total of %d words" % totalWords)
# for geoCoding tweets
geo = Geo()
tweetCount, tweetErrorCount = 0, 0
for _file in sorted(os.listdir(tweetFolder)):
fileDate = datetime.strptime(_file[17:27], "%Y-%m-%d")
if fileDate >= fromDate and fileDate < toDate:
log.info("processing file %s" % (_file))
try:
with open(tweetFolder + "/" + _file, "r") as FILE:
for line in FILE:
try:
jsonTweet = json.loads(line.strip())
dateStr = jsonTweet["interaction"]["created_at"][5:16]
tweetDate = datetime.strptime(dateStr, "%d %b %Y")
geoList = geo.geo_normalize(jsonTweet)
city, ctry, state = geoList[:3]
if ctry and (ctry.lower() == country) and (tweetDate >= fromDate) and (tweetDate <= toDate):
# prereProcess the tweet
text = jsonTweet["interaction"]["content"]
text = re.sub(URL_REGEX, " ", text) # remove urls
text = re.sub(
"[^A-Za-z_@#0-9]", " ", normalize_str(text, lower=True)
) # allow only alphaNumerics and twitter tags
text = re.sub(" +", " ", text) # remove multiple spaces
for group in regex:
keywordsPresent = re.findall(regex[group], text)
if len(keywordsPresent) > 0:
keywordsPresent = list(set(keywordsPresent))
hashTags = extract_hash_tags(text)
hashTags = [hashTag for hashTag in hashTags if len(hashTag) > 3]
hashTags.extend(keywordsPresent)
for hashTag in hashTags:
if hashTag.startswith("#"):
hashTag = hashTag[1:]
if hashTag in counts[group]:
counts[group][hashTag] += 1
else:
counts[group][hashTag] = 1
tweetCount += 1
except Exception, f:
log.debug("error processing tweet %s", f)
tweetErrorCount += 1
except Exception, e:
log.exception("error processfing file %s", e)
else:
log.debug("skipping file %s" % (_file))
def preProcess(tweetFolder, outputFolder, keywordList, fromDate, toDate, country, filesProcessed):
log.info("inside preProcess")
log.debug("fromDate-->" + fromDate.strftime("%d %b %Y"))
log.debug("toDate-->" + toDate.strftime("%d %b %Y"))
tweetCount, tweetErrorCount = 0, 0
tweets = {}
# output files
tweetedFile = open(outputFolder + '/tweeted.csv', 'w')
mentionFile = open(outputFolder + '/mentioned.csv', 'w')
retweetFile = open(outputFolder + '/retweet.csv', 'w')
wordsFile = open(outputFolder + '/containsWord.csv', 'w')
sentimentFile = open(outputFolder + '/sentiment.csv', 'w')
tweetsFile = open(outputFolder + '/tweets.json', 'w')
# build stop word list
englishStopWords = [normalize_str(w).lower() for w in stopwords.words('english')]
spanishStopWords = [normalize_str(w).lower() for w in stopwords.words('spanish')]
stopWordList = []
stopWordList.extend(englishStopWords)
stopWordList.extend(spanishStopWords)
log.info("# of keywords: " + str(len(keywordList)))
log.info("tracking--> " + str(keywordList))
# build regular expression for keyword
keywordRegex = re.compile(r'\b%s\b' % '\\b|\\b'.join(keywordList),
flags=re.IGNORECASE)
# for geocoding tweets
geo = Geo()
log.info("filesProcessed-->" + str(filesProcessed))
for _file in sorted(os.listdir(tweetFolder)):
fileDate = datetime.strptime(_file[17:27], '%Y-%m-%d')
if (_file not in filesProcessed and fileDate >= fromDate and fileDate < toDate):
log.info("processing file %s" % (_file))
try:
with open(tweetFolder + "/" + _file, "r") as FILE:
tweetCount, tweetErrorCount = 0, 0
for line in FILE:
try:
jsonTweet = json.loads(line.strip())
dateStr = jsonTweet['interaction']['created_at'][5:16]
tweetDate = datetime.strptime(dateStr, '%d %b %Y')
sentiment = getSentiment(jsonTweet)
if sentiment == 0:
continue
geoList = geo.geo_normalize(jsonTweet)
ctry, a1, a2, a3 = geoList[1:5]
if ctry and (ctry.lower() == country) and (tweetDate >= fromDate) and (tweetDate <= toDate):
text = jsonTweet["interaction"]["content"]
text = re.sub(URL_REGEX, ' ', text) # remove urls
text = re.sub('[^A-Za-z_@#0-9]', ' ', normalize_str(text, lower=True)) # allow only alphaNumerics and twitter tags
text = re.sub(' +', ' ', text) # remove multiple spaces
keywordsPresent = keywordRegex.search(text)
if keywordsPresent is not None:
words = text.split(" ")
words = [w for w in words if len(w) > 2 and w not in stopWordList]
words2 = []
for word in words:
for w in word:
if (word not in keywordList) and (w.isdigit() or w == '@'):
break
else:
if word[0] == '#':
word = word[1:]
words2.append(word)
tweetId = jsonTweet["twitter"]["id"]
tweeterId = str(jsonTweet["interaction"]["author"]["id"])
mentions, retweets = getInteractions(jsonTweet)
tweetedFile.write(tweeterId + ',' + tweetId + '\n')
sentimentFile.write(tweetId + ',' + str(sentiment) + '\n')
for userId in mentions:
mentionFile.write(tweetId + ',' + userId + '\n')
for userId in retweets:
retweetFile.write(tweetId + ',' + userId + '\n')
for word in words2:
wordsFile.write(tweetId + ',' + word + '\n')
# tracking the tweets for checks.
if tweeterId in tweets:
tweets[tweeterId][tweetId] = jsonTweet["interaction"]["content"]
else:
tweets[tweeterId] = {}
tweets[tweeterId][tweetId] = jsonTweet["interaction"]["content"]
tweetCount += 1
except Exception, f:
log.exception("error processing tweet %s", f)
tweetErrorCount += 1
except Exception, e:
log.exception("error processfing file %s", e)
log.info("tweets used: %s" % str(tweetCount))
log.debug("tweetErrorCount : %s" % str(tweetErrorCount))
filesProcessed.append(_file)
break
else:
log.debug("skipping file %s" % (_file))
ap.add_argument('-t', '--tweetFolder', type=str,
help='inputFolder pointing to PSLs output',
default='/hdd/tweets/2012/oct')
ap.add_argument('-c', '--country', type=str)
ap.add_argument('-m', '--month', type=str)
arg = ap.parse_args()
logs.init(arg)
geo = Geo()
tweetCount = 0
date = datetime.strptime(arg.month, "%b %Y")
for _file in os.listdir(arg.tweetFolder):
try:
with open(arg.tweetFolder + "/" + _file, "r") as FILE:
for line in FILE:
try:
jsonTweet = json.loads(line.strip())
dateStr = jsonTweet['interaction']['created_at'][5:16]
tweetDate = datetime.strptime(dateStr, '%d %b %Y')
geoList = geo.geo_normalize(jsonTweet)
city, ctry, state = geoList[:3]
if ctry and ctry.lower() == arg.country.lower() and date.month == tweetDate.month and date.year == tweetDate.year:
tweetCount += 1
except Exception, f:
_traceback = sys.exc_info()[2]
log.exception("error processing tweet %s @line %s" % (f, traceback.tb_lineno(_traceback)))
log.info("tweet Count -->" + str(tweetCount))
except:
log.exception('error opening file')
print ('# tweets for %s in %s --> %d' % (arg.country, arg.month, tweetCount))