def genRandomWebLogs(self):
"""
Method for generating random web log data.
"""
self.count += 1
if self.randomWebLogsWindowOpenedFlag == False:
self.randomWebLogsWindowOpenedFlag = True # set window opened
global RandomWebLogsWindow
def toggleFlag():
self.randomWebLogsWindowOpenedFlag = False # set window closed
RandomWebLogsWindow.destroy()
RandomWebLogsWindow = tk.Toplevel(self)
RandomWebLogsWindow.minsize(300, 500)
RandomWebLogsWindow.geometry("300x500+100+100")
RandomWebLogsWindow.title("Random web log data")
RandomWebLogsWindow.config(bd=5)
RandomWebLogsWindow.protocol("WM_DELETE_WINDOW", toggleFlag)
x = sp.arange(1, 31 * 24) # 1 month of traffic data
y = sp.array(200 * (sp.sin(2 * sp.pi * x / (7 * 24))), dtype=int)
y += gamma.rvs(15, loc=0, scale=100, size=len(x))
y += 2 * sp.exp(x / 100.0)
y = sp.ma.array(y, mask=[y < 0])
sp.savetxt(os.path.join("sample_data", "sample_web_traffic.tsv"), list(zip(x, y)), delimiter="\t", fmt="%s")
model = TableModel() # create a new TableModel for table data
table = TableCanvas(RandomWebLogsWindow, model=model, editable=False) # create a new TableCanvas for showing the table
table.createTableFrame()
tableData = {} # dictionary for storing table data
for k, v in list(zip(x,y)):
tableData[uuid.uuid4()] = {'Hour': str(k), 'Hits': str(v)}
model.importDict(tableData)
table.resizeColumn(0, 100)
table.resizeColumn(1, 100)
table.sortTable(columnName='Hour')
table.redrawTable()
else:
RandomWebLogsWindow.deiconify()
def findTermFrequencies(self):
"""
Method for calculating the frequencies of each term in the tweets.
"""
if not os.path.isfile(self.TwitterKeysFile):
tkMessageBox.showerror("ERROR", "Twitter API credentials not filled. Use the Set/Update Twitter Credentials button to do so.", parent = self.parent)
return
self.count += 1
if self.termFrequenciesOpenedFlag == False:
# set window opened
self.termFrequenciesOpenedFlag = True
# initialize window
global TermFrequenciesWindow
def toggleFlag():
self.termFrequenciesOpenedFlag = False
TermFrequenciesWindow.destroy()
TermFrequenciesWindow = tk.Toplevel(self)
TermFrequenciesWindow.minsize(500, 500)
#TwitterKeysWindow.overrideredirect(True)
TermFrequenciesWindow.geometry("500x500+100+100")
TermFrequenciesWindow.title("Term Frequencies (only > 0.5%)")
TermFrequenciesWindow.config(bd=5)
TermFrequenciesWindow.protocol("WM_DELETE_WINDOW", toggleFlag)
# create a new TableModel for table data
model = TableModel()
# create a new TableCanvas for showing the table
table = TableCanvas(TermFrequenciesWindow, model=model,
editable=False)
table.createTableFrame()
# dictionary for storing table data
tableData = {}
# calculate frequencies
freqs = {}
total = 0
outfile = open(self.TwitterStreamFile)
for line in outfile:
json_obj = json.loads(line)
try:
text = json_obj['text'].decode('utf-8')
# clean the text
text = ' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)","",text).split())
text = re.sub(r'^https?:\/\/.*[\r\n]*', '', text, flags=re.MULTILINE)
text = re.sub(r'RT', '', text, flags=re.MULTILINE)
#print text
text_list = text.split(' ')
for char in text_list:
if char.isalnum():
if char not in freqs:
freqs[char] = 1
else:
freqs[char] += 1
total += 1
except:
#print "passed"
pass
for key in freqs.keys():
if freqs[key]/float(total) > 0.005:
# insert frequencies into the table dictionary
tableData[uuid.uuid4()] = {'Term': key, 'Frequency (%)': str(round((freqs[key]/float(total))*100, 2))}
#print key + " " + str(freqs[key]/float(total))
# insert and sort data in the table
model.importDict(tableData)
#sort in descending order
model.setSortOrder(columnIndex = 1, reverse = 1)
table.adjustColumnWidths()
table.resizeColumn(0, 200)
table.resizeColumn(1, 200)
table.redrawTable()
def findTweetSentiment(self):
"""
Method for calculating the sentiment of each tweet.
"""
if not os.path.isfile(self.TwitterKeysFile):
tkMessageBox.showerror("ERROR", "Twitter API credentials not filled. Use the Set/Update Twitter Credentials button to do so.", parent = self.parent)
return
self.count += 1
if self.tweetSentimentOpenedFlag == False:
# set window opened
self.tweetSentimentOpenedFlag = True
# initialize window
global TweetSentimentWindow
def toggleFlag():
self.tweetSentimentOpenedFlag = False
TweetSentimentWindow.destroy()
TweetSentimentWindow = tk.Toplevel(self)
TweetSentimentWindow.minsize(600, 500)
#TwitterKeysWindow.overrideredirect(True)
TweetSentimentWindow.geometry("1000x500+100+100")
TweetSentimentWindow.title("Tweet Sentiments (Zero values omitted)")
TweetSentimentWindow.config(bd=5)
TweetSentimentWindow.protocol("WM_DELETE_WINDOW", toggleFlag)
# create a new TableModel for table data
model = TableModel()
# create a new TableCanvas for showing the table
table = TableCanvas(TweetSentimentWindow, model=model,
editable=False)
table.createTableFrame()
# dictionary for storing table data
tableData = {}
# calculate sentiments
afinnfile = open(self.AFINNFile)
scores = {}
for line in afinnfile:
term, score = line.split("\t")
scores[term] = int(score)
#print scores.items()
positive = 0.0
negative = 0.0
outfile = open(self.TwitterStreamFile)
for line in outfile:
json_obj = json.loads(line)
sentiment = 0
try:
text = json_obj['text'].decode('utf-8')
# clean the text
text = ' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)","",text).split())
text = re.sub(r'^https?:\/\/.*[\r\n]*', '', text, flags=re.MULTILINE)
text = re.sub(r'RT', '', text, flags=re.MULTILINE)
#print text
text_list = text.split(' ')
for char in text_list:
if char in scores:
sentiment += scores[char]
if sentiment != 0:
# add items to table data dictionary
tableData[uuid.uuid4()] = {'Tweet': text, 'Score': str(sentiment)}
if sentiment > 0:
positive += 1
elif sentiment < 0:
negative += 1
#print text + " " + str(sentiment) + "\n\n"
except:
#print "passed"
pass
# calculate ratio
if positive > 0 and negative > 0:
ratio = round(float(positive) / float(negative), 2)
# insert and sort data in the table
model.importDict(tableData)
table.resizeColumn(0, 850)
table.resizeColumn(1, 50)
table.sortTable(columnName='Score')
table.redrawTable()
if positive > negative:
extra = "The overall sentiment is POSITIVE."
else:
extra = "The overall sentiment is NEGATIVE."
# show info box about the overall result
tkMessageBox.showinfo("Score Ratio", "The ratio of positive vs. negative tweets is " + str(ratio) + ". " + extra, parent = TweetSentimentWindow)
def findTermFrequencies(self):
"""
Method for calculating the frequencies of each term in the tweets.
"""
if not os.path.isfile(self.TwitterKeysFile):
tkMessageBox.showerror(
"ERROR",
"Twitter API credentials not filled. Use the Set/Update Twitter Credentials button to do so.",
parent=self.parent)
return
self.count += 1
if self.termFrequenciesOpenedFlag == False:
# set window opened
self.termFrequenciesOpenedFlag = True
# initialize window
global TermFrequenciesWindow
def toggleFlag():
self.termFrequenciesOpenedFlag = False
TermFrequenciesWindow.destroy()
TermFrequenciesWindow = tk.Toplevel(self)
TermFrequenciesWindow.minsize(500, 500)
#TwitterKeysWindow.overrideredirect(True)
TermFrequenciesWindow.geometry("500x500+100+100")
TermFrequenciesWindow.title("Term Frequencies (only > 0.5%)")
TermFrequenciesWindow.config(bd=5)
TermFrequenciesWindow.protocol("WM_DELETE_WINDOW", toggleFlag)
# create a new TableModel for table data
model = TableModel()
# create a new TableCanvas for showing the table
table = TableCanvas(TermFrequenciesWindow,
model=model,
editable=False)
table.createTableFrame()
# dictionary for storing table data
tableData = {}
# calculate frequencies
freqs = {}
total = 0
outfile = open(self.TwitterStreamFile)
for line in outfile:
json_obj = json.loads(line)
try:
text = json_obj['text'].decode('utf-8')
# clean the text
text = ' '.join(
re.sub(
"(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)",
"", text).split())
text = re.sub(r'^https?:\/\/.*[\r\n]*',
'',
text,
flags=re.MULTILINE)
text = re.sub(r'RT', '', text, flags=re.MULTILINE)
#print text
text_list = text.split(' ')
for char in text_list:
if char.isalnum():
if char not in freqs:
freqs[char] = 1
else:
freqs[char] += 1
total += 1
except:
#print "passed"
pass
for key in freqs.keys():
if freqs[key] / float(total) > 0.005:
# insert frequencies into the table dictionary
tableData[uuid.uuid4()] = {
'Term':
key,
'Frequency (%)':
str(round((freqs[key] / float(total)) * 100, 2))
}
#print key + " " + str(freqs[key]/float(total))
# insert and sort data in the table
model.importDict(tableData)
#sort in descending order
model.setSortOrder(columnIndex=1, reverse=1)
table.adjustColumnWidths()
table.resizeColumn(0, 200)
table.resizeColumn(1, 200)
table.redrawTable()
def findTweetSentiment(self):
"""
Method for calculating the sentiment of each tweet.
"""
if not os.path.isfile(self.TwitterKeysFile):
tkMessageBox.showerror(
"ERROR",
"Twitter API credentials not filled. Use the Set/Update Twitter Credentials button to do so.",
parent=self.parent)
return
self.count += 1
if self.tweetSentimentOpenedFlag == False:
# set window opened
self.tweetSentimentOpenedFlag = True
# initialize window
global TweetSentimentWindow
def toggleFlag():
self.tweetSentimentOpenedFlag = False
TweetSentimentWindow.destroy()
TweetSentimentWindow = tk.Toplevel(self)
TweetSentimentWindow.minsize(600, 500)
#TwitterKeysWindow.overrideredirect(True)
TweetSentimentWindow.geometry("1000x500+100+100")
TweetSentimentWindow.title(
"Tweet Sentiments (Zero values omitted)")
TweetSentimentWindow.config(bd=5)
TweetSentimentWindow.protocol("WM_DELETE_WINDOW", toggleFlag)
# create a new TableModel for table data
model = TableModel()
# create a new TableCanvas for showing the table
table = TableCanvas(TweetSentimentWindow,
model=model,
editable=False)
table.createTableFrame()
# dictionary for storing table data
tableData = {}
# calculate sentiments
afinnfile = open(self.AFINNFile)
scores = {}
for line in afinnfile:
term, score = line.split("\t")
scores[term] = int(score)
#print scores.items()
positive = 0.0
negative = 0.0
outfile = open(self.TwitterStreamFile)
for line in outfile:
json_obj = json.loads(line)
sentiment = 0
try:
text = json_obj['text'].decode('utf-8')
# clean the text
text = ' '.join(
re.sub(
"(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)",
"", text).split())
text = re.sub(r'^https?:\/\/.*[\r\n]*',
'',
text,
flags=re.MULTILINE)
text = re.sub(r'RT', '', text, flags=re.MULTILINE)
#print text
text_list = text.split(' ')
for char in text_list:
if char in scores:
sentiment += scores[char]
if sentiment != 0:
# add items to table data dictionary
tableData[uuid.uuid4()] = {
'Tweet': text,
'Score': str(sentiment)
}
if sentiment > 0:
positive += 1
elif sentiment < 0:
negative += 1
#print text + " " + str(sentiment) + "\n\n"
except:
#print "passed"
pass
# calculate ratio
if positive > 0 and negative > 0:
ratio = round(float(positive) / float(negative), 2)
# insert and sort data in the table
model.importDict(tableData)
table.resizeColumn(0, 850)
table.resizeColumn(1, 50)
table.sortTable(columnName='Score')
table.redrawTable()
if positive > negative:
extra = "The overall sentiment is POSITIVE."
else:
extra = "The overall sentiment is NEGATIVE."
# show info box about the overall result
tkMessageBox.showinfo(
"Score Ratio",
"The ratio of positive vs. negative tweets is " + str(ratio) +
". " + extra,
parent=TweetSentimentWindow)
