def senti(csv_file=CSV_FILE):
"""graph of positive and negative tweets over time (mandatory)
:param:
:return:
"""
polar_pattern = re.compile(r'[-]?\d+\.\d+')
sentiment_list = list()
polar_list = list()
dates_list = list()
with open(csv_file) as file_object:
reader_object = csv.reader(file_object)
next(reader_object, None) # Skip headers
sentiment_list = [i[-1] for i in reader_object] # get sentiment column
for i in sentiment_list:
polarity = re.findall(polar_pattern, i) # Just the numbers
polarity = polarity[0] # Only the polarity part
polar_list.append(polarity) # In a list.
polar_float = [float(i) for i in polar_list] # Make if float.
# for i in polar_float: # For testing
# print(i)
# print(polar_float)
with open(csv_file) as file_object:
reader_object = csv.reader(file_object)
next(reader_object, None) # Skip headers
created_list = [i[2] for i in reader_object]
for i in created_list:
# this is dt object, not subscriptable
dates_tweets_1 = datetime.datetime.strptime(
i, '%a %b %d %H:%M:%S %z %Y')
# Convert into string of dates and times + GMT
# dates_tweets = str(dates_tweets_1)
# Adds tuples together in a list [('date', 'time'), ('date', 'time')]
dates_list.append(dates_tweets_1)
try:
# dates = mdates.date2num(dates_list)
# yearsFmt = mdates.DateFormatter('%a %b %d %H:%M:%S %z %Y')
fig, ax = plt.subplots()
ax.scatter(dates_list, polar_float)
# print(polar_float)
plt.title("Sentiment analysis over Time")
plt.xlabel("Time and Date")
plt.ylabel("Sentiment Polarity")
# The following lines does not seem to have any effect
# ax.format_xdata = mdates.DateFormatter('%a %b %d %H:%M:%S %z %Y')
# ax.format_ydata = polar_float
# ax.yaxis.set_major_formatter(FormatStrFormatter('%f'))
ax.grid(True) # Cute grid.
ui.info_1("Saving...")
plt.savefig("Q4_sentiment.png")
plt.show()
except TypeError as e:
print("TypeError")
exit()
def main(args):
workspace_path = args.workspace_path or os.getcwd()
workspace = tsrc.workspace.Workspace(path.Path(workspace_path))
ui.info_1("Creating new workspace in", ui.bold, workspace_path)
workspace.init_manifest(args.manifest_url, branch=args.branch)
workspace.load_manifest()
workspace.clone_missing()
workspace.set_remotes()
workspace.copy_files()
ui.info("Done", ui.check)
def help(args, name=None):
if(name == '-list_users'):
print(list_users.__doc__)
elif(name == '-create_project'):
print(create_project.__doc__)
elif(name == '-create_user'):
print(create_user.__doc__)
elif(name == '-update_user'):
print(update_user.__doc__)
elif(name == '-list_users'):
print(list_users.__doc__)
elif(name == '-list_projects'):
print(list_projects.__doc__)
elif(name == '-list_domains'):
print(list_domains.__doc__)
elif(name == '-grant_role'):
print(grant_role.__doc__)
elif(name == '-grant_role2'):
print(grant_role2.__doc__)
elif(name == '-create_network'):
print(create_network.__doc__)
elif(name == '-list_networks'):
print(list_networks.__doc__)
elif(name == '-create_instance'):
print(create_instance.__doc__)
elif(name == '-create_route'):
print(create_router.__doc__)
elif(name == '-create_floatingip'):
print(create_floatingip.__doc__)
elif(name == '-delete_users'):
print(delete_users.__doc__)
elif(name == '-delete_network'):
print(delete_network.__doc__)
elif(name == '-delete_project'):
print(delete_project.__doc__)
elif(name == '-revoke_role'):
print(revoke_role.__doc__)
elif(name == '-update_user'):
print(update_user.__doc__)
elif(name == '-delete_users'):
print(update_users.__doc__)
elif(name == '-update_project'):
print(update_project.__doc__)
elif name is None:
print('')
ui.info_1(
'Untuk informasi command Anda bisa mengetik help <spasi> -cmd: \n')
print(' -create_project \n -create_user \n -create_network \n -create_instance \n -create_route \n -create_floatingip')
print(' -list_users \n -list_networks \n -list_projects \n -list_domains \n -list_images')
print(' -grant_role \n -grant_role2 \n -delete_project \n -delete_network \n -delete_user \n -delete_users')
print(' -update_user \n -update_project \n')
else:
print('command yang anda masukkan tidak tersedia')
def main(args):
workspace_path = args.workspace_path or os.getcwd()
workspace = tsrc.workspace.Workspace(path.Path(workspace_path))
ui.info_1("Configuring workspace in", ui.bold, workspace_path)
workspace.configure_manifest(url=args.manifest_url,
branch=args.branch,
groups=args.groups)
workspace.load_manifest()
workspace.clone_missing()
workspace.set_remotes()
workspace.copy_files()
ui.info("Done", ui.check)
def main():
ui.info_1("Starting CI")
checks = init_checks()
for check in checks:
check.run()
failed_checks = [check for check in checks if not check.ok]
if not failed_checks:
ui.info(ui.green, "CI passed")
return
for check in failed_checks:
ui.error(check.name, "failed")
sys.exit(1)
def process(self, items):
if not items:
return True
ui.info_1(self.task.description())
self.errors = list()
num_items = len(items)
for i, item in enumerate(items):
if not self.task.quiet():
ui.info_count(i, num_items, end="")
self.process_one(item)
if self.errors:
self.handle_errors()
def change_filter():
"""Changes the search filter for myStream.filter(track=[trump])
:param:
:return trump: New keyword or DEFAULT keyword 'trump'"""
trump = 'trump' # A variable named trump.
ui.info_1(ui.yellow, "SEARCH:", ui.blue, ui.standout, "'trump'")
ans_filter = ui.ask_yes_no("Change search keyword? [Ctrl-C Quits] >>>",
default=False)
if ans_filter == False:
return trump
else:
ui.info_1(ui.red, "Really change?")
ans_really = ui.ask_yes_no("[ENTER] for No >>>", default=False)
if ans_really == False:
return trump
else:
trump = ui.ask_string("Enter new search keyword")
ui.info_3("Starting search for new keyword:", ui.blue, trump)
return trump
def tweets_table_create(
conn=create_conection(), sql_file=TWEETS_DB, table_name=TABLE_NAME):
"""Create tweets table.
:param:
:return:
This code was take from SQLite Tutorial,
http://www.sqlitetutorial.net/sqlite-python/create-tables/ (2018), accessed on 22-Nov-2018
It was modified to have {tn} instead of the table name, and added ui interface.
"""
tweet_id = "tweets_id"
tweet_text = "tweet_text"
created_at = "created_at"
location = "location"
coordinates = "geocoordinates"
user_followers = "user_followers"
friends_no = "number_of_friends"
senti = "sentiment_analysis"
create_tweets_database = """CREATE TABLE IF NOT EXISTS {tn} (
{ti} interger NOT NULL,
{tt} text NOT NULL,
{ct} text NOT NULL,
{loc} text,
{coor} text,
{uf} interger,
{fn} interger,
{sn} text)""".format(tn=table_name,
ti=tweet_id,
tt=tweet_text,
ct=created_at,
loc=location,
coor=coordinates,
uf=user_followers,
fn=friends_no,
sn=senti)
if conn is not None:
create_table(conn, create_tweets_database)
ui.info_1("New table created:", ui.blue, table_name)
else:
ui.error("Cannot create connection to DATABASE")
def user_ans_tweets(table_name=TABLE_NAME):
"""Asks user how many tweets to append to table, and does so.
:param table_name: name of table
"""
ui.info_1(ui.bold, "Add Tweets to", ui.blue, MyStreamListener.table_name)
ans_int = input(" How many tweets? (interger) >>> ")
MyStreamListener.tweet_stop = 0
try:
ans_int = int(ans_int)
if ans_int > 0:
MyStreamListener.tweet_stop = ans_int
fetch_tweets()
else:
print("0 tweets gathered")
return
except ValueError:
ui.info_1(ui.red, "Please write an interger")
user_ans_tweets()
except ConnectionError:
ui.error(ui.cross, "Connection Error")
ui.info(ui.bold, "Program Finished")
exit()
def test_record_message(message_recorder):
ui.info_1("This is foo")
assert message_recorder.find("foo")
message_recorder.reset()
ui.info_1("This is bar")
assert not message_recorder.find("foo")
def ask_table(db_file=TWEETS_DB,
conn=create_conection(),
table_name=TABLE_NAME):
"""Default table or new table. Checks if table exists,
creates one if not with name from user and calls user_ans_tweets().
:param db_file: DEFAULT database
:param conn: creates connection()
:param table_name: name of table
"""
def ploting(tweets_db=TWEETS_DB, csv_file=CSV_FILE):
table_name = MyStreamListener.table_name
plot_question = ui.ask_yes_no("Plot?", default=True)
if plot_question:
ui.info(ui.green, "Populating csv file with {}".format(table_name))
q_four.db_into_csv(TWEETS_DB, CSV_FILE, table_name)
ui.info_3("Ploting...")
q_four.frecuency()
q_four.senti()
else:
ui.info(ui.turquoise, "Program Finished")
exit()
tables_list = list()
conn = create_conection()
cur = conn.cursor()
cur.execute("SELECT name FROM sqlite_master")
for table in cur:
tables = str(table).strip('(').strip(')').strip(',').strip("'")
tables_list.append(tables)
if "trump" in tables_list:
ui.info_1(ui.yellow, ui.bold, "DEFAULT TABLE FOUND! =>", ui.blue,
ui.standout, "'trump'")
new_db_ans = ui.ask_yes_no("Add tweets to table 'trump'?",
default=True)
if new_db_ans:
ui.warning("Creating database (If it doesn't exist)")
create_conection()
ui.info_2(ui.green, "Accessing Database")
MyStreamListener.table_name = TABLE_NAME
user_ans_tweets()
ploting()
else:
choices = ["Create New Table", "Load table", "Plot", "Quit"]
new = ui.ask_choice("Select: ", choices)
if new == "Create New Table":
ui.warning("Tables with the same name will not be created")
new_table_name = ui.ask_string("Enter new table name:")
new_table_name = new_table_name.lower()
new_table_name = new_table_name.replace(" ", "_")
ui.info("Table name with format:", ui.blue, new_table_name)
create_ans = ui.ask_yes_no("Create?")
if create_ans:
tweets_table_create(create_conection(), TWEETS_DB,
new_table_name)
insert_new_tbl = ui.ask_yes_no(
"Insert new tweets into {}?".format(new_table_name))
if insert_new_tbl:
MyStreamListener.table_name = new_table_name
user_ans_tweets()
ploting()
else:
ui.info(ui.bold, ("Program Finished"))
else:
ui.info(ui.bols, "Program Finished")
elif new == "Load table":
new_table_name = ui.ask_choice("Select Table to load:",
tables_list)
MyStreamListener.table_name = new_table_name
user_ans_tweets()
ploting()
elif new == "Plot":
new_table_name = ui.ask_choice("Select Table to plot:",
tables_list)
MyStreamListener.table_name = new_table_name
ploting()
elif new == "Quit":
ui.warning("Program Finished")
exit()
def frecuency(csv_file=CSV_FILE):
"""Frecuency of the tweets over time (mandatory)
:param csv_file: the csv file containing the data to be graphed.
:return:
# line of code bellow is from the Python Documentation, last updated 04-12-2018 by the Python Software Foundation, accessed on 4-12-2018: https://docs.python.org/3/library/datetime.html#strafed-strptime-behavior
**************************************************
I believe a scatter plot is better for representing this kind of data. With pandas is trivial to group the times of the day in hour intervarls, but I could not find a way to do it with just lists. My attempts at this task are kept in the comments bellow.
"""
created_list = list()
dates_list = list()
dates_tweets = list()
datetimex = list()
x_list = list()
x = list()
y = list()
with open(csv_file) as file_object:
reader_object = csv.reader(file_object)
next(reader_object, None) # Skip headers
created_list = [i[2] for i in reader_object]
for i in created_list:
# this is dt object, not subscriptable
dates_tweets = datetime.datetime.strptime(
i, '%a %b %d %H:%M:%S %z %Y') # Thu Apr 23 13:38:19 +0000 2009
# datetime objects
# Convert into string of dates and times + GMT
# dates_tweets = str(dates_tweets_1)
# Remove +GMT
# dates_tweets = dates_tweets[:-6]
# Make a ['date','time'] list.
# dates_tweets = dates_tweets.split('')
# Make a tuple ('date', 'time')
# dates_tweets = tuple(dates_tweets)
# Adds tuples together in a list [('date', 'time'), ('date', 'time')]
dates_list.append(dates_tweets)
# Make test list(100 entries) comment this lines for production
# jig = list()
# for x, i in dates_list[3100:3200]:
# jig.append(i)
# dates_list = jig
# End of test list
# This sorts the list:
dates_list = sorted(dates_list)
dates_list = [(key, len(list(group)))
for key, group in groupby(dates_list)]
# print(dates_list) = (datetime.datetime(2018, 12, 6, 18, 2, 25, tzinfo=datetime.timezone.utc), 6)]
for i in dates_list:
x.append(i[0])
# for i in x:
# print(i)
# for i in x:
# i = str(i)
# datetime_x = datetime.datetime.strptime(i, '%Y-%m-%d %H:%M:%S%z')
# x_list.append(datetime_x)
dates = mdates.date2num(x)
# datetime_x = datetime.datetime.strptime(i, '%b %d %Y %I:%M%p')
# return dates_list
# for i in dates_list:
# x.append(i[1])
# x = list(range(9))
# jig = [(len(list(group)), key) for key, group in groupby(jig)]
# below doesnt work properly
# x = [datetime.datetime.now() + datetime.timedelta(hours=i) for i in range(9)]
# dates_tweets_3 = dates_tweets_2[1100:1200]
# print(dates_tweets_3)
# dates_tweets_3 = sorted(dates_tweets_3, reverse=True)
# dates_tweets_3 = [(len(list(key)), group) for key, group in groupby(dates_tweets_3)]
# x = mdates.date2num(dates_tweets_3)
for i in dates_list:
y.append(i[1])
fig, ax = plt.subplots(1)
fig.autofmt_xdate()
plt.title("Frequency of tweets over time")
plt.xlabel('Date and Time')
plt.ylabel('No of tweets')
plt.plot_date(dates, y)
# plt.plot(dates, y)
# xmft = mdates.DateFormatter('%Y-%m-%d %H:%M:%S%z')
# ax.xaxis.set_major_formatter(xmft)
# print(x)
# print(y)
ui.info_1("Saving...")
plt.savefig('Q4_Frequency.png')
plt.show()