def graph_02(self):
legend = {"title": "Número de interações por tempo do vídeo"}
if (self._language == "en"):
legend = {"title": "Number of interaction by video time"}
df = pd.DataFrame(columns=self.DATASET.columns[1:].tolist())
for i in range(0, self.VIDEO_SIZE + 1):
df_aux = self.DATASET[self.DATASET.columns[2:]].loc[self.DATASET[
self.DATASET.columns[1]] == i].apply(np.sum)
df.loc[i] = df_aux.append(
pd.Series([i], index=[self.DATASET.columns[1]]))
trace = [
Table(header=dict(values=list(df.columns[:len(df.columns)]),
fill=dict(color='#C2D4FF'),
align='center'),
cells=dict(values=[
df[i].tolist() for i in df.columns[:len(df.columns)]
],
fill=dict(color='#F5F8FF'),
align=['center', 'center']))
]
data = trace
layout = Layout(title=legend["title"])
fig = Figure(data=data, layout=layout)
if self._type_result == "jupyter-notebook":
iplot(data, filename='pandas_table')
elif self._type_result == "dash":
return dcc.Graph(id='V009@2', figure={"data": data})
elif self._type_result == "flask":
modeBarButtonsToRemove = [
'toImage', 'sendDataToCloud', 'hoverCompareCartesian',
'lasso2d', 'hoverClosestCartesian', 'toggleHover',
'hoverClosest3d', 'hoverClosestGeo', 'hoverClosestGl2d',
'hoverClosestPie'
]
config = {
"displaylogo": False,
"responsive": True,
"displayModeBar": True,
"modeBarButtonsToRemove": modeBarButtonsToRemove
}
return {
"id":
"V009@2",
"layout":
json.dumps({
"data": data,
"layout": layout,
"config": config
},
cls=PlotlyJSONEncoder)
}
def graph_03(self):
legend = {"title": "Cronologia de interações dos alunos"}
if (self._language == "en"):
legend = {"title": "Chronology of students' interaction"}
trace = [
Table(header=dict(values=list(
self._df_chronology.columns[:len(self._df_chronology.columns
)]),
fill=dict(color='#C2D4FF'),
align='center'),
cells=dict(values=[
self._df_chronology[i].tolist() for i in self.
_df_chronology.columns[:len(self._df_chronology.columns)]
],
fill=dict(color='#F5F8FF'),
align=['left', 'center']))
]
data = trace
layout = Layout(title=legend["title"])
fig = Figure(data=data, layout=layout)
if self._type_result == "jupyter-notebook":
iplot(data, filename='pandas_table')
elif self._type_result == "dash":
return dcc.Graph(id='V009@2', figure={"data": data})
elif self._type_result == "flask":
modeBarButtonsToRemove = [
'toImage', 'sendDataToCloud', 'hoverCompareCartesian',
'lasso2d', 'hoverClosestCartesian', 'toggleHover',
'hoverClosest3d', 'hoverClosestGeo', 'hoverClosestGl2d',
'hoverClosestPie'
]
config = {
"displaylogo": False,
"responsive": True,
"displayModeBar": True,
"modeBarButtonsToRemove": modeBarButtonsToRemove
}
return {
"id":
"V009@3",
"layout":
json.dumps({
"data": data,
"layout": layout,
"config": config
},
cls=PlotlyJSONEncoder)
}
def test_table_plot(self):
test_db = {}
db_list = os.listdir(self._cfg['db_dir'])
db_list = [x for x in db_list if x[:4] == 'test']
db_list.sort(key=self.natural_sort)
for idx in range(len(db_list)):
sts_file = '/'.join((self._cfg['db_dir'], db_list[idx]))
with open(sts_file, 'r') as fh:
db = json.load(fh)
header = dict(values=[
'<b>ID<b>', '<b>TestName<b>', '<b>Status<b>', '<b>Tag<b>',
'<b>Syndrome<b>', '<b>ErrorInfo<b>', '<b>BugID<b>'
],
line=dict(color='#7D7F80'),
fill=dict(color='#A1C3D1'),
font=dict(size=13),
align=[['middle'] * 7])
id_list = []
name_list = []
status_list = []
tag_list = []
syndrome_list = []
err_list = []
bug_list = []
for cnt in range(len(db)):
tid = str(cnt)
tinfo = db[tid]
id_list.append(tid)
name_list.append(tinfo['name'])
status_list.append(tinfo['status'])
tag_list.append('<br>'.join(tinfo['tags']))
syndrome_list.append(tinfo['syndrome'])
err_list.append(tinfo['errinfo'])
bug_list.append('TBD')
cell_val = dict(values=[
id_list, name_list, status_list, tag_list, syndrome_list,
err_list, bug_list
],
line=dict(color='#7D7F80'),
fill=dict(color='#EDFAFF'),
font=dict(size=13),
align=[['middle'] * 7])
trace = Table(header=header, cells=cell_val)
data = [trace]
layout = dict(paper_bgcolor='rgba(240,240,240,0.85)',
plot_bgcolor='rgba(240,255,255,0.85)')
fig = dict(data=data, layout=layout)
fn = db_list[idx][:-5] + '.html'
off.plot(fig, filename=fn, auto_open=False)
def update(interval):
trades = pd.DataFrame(app.trades_, columns=['Time', 'Size', 'Price', 'Side'])
trades['color'] = 'rgb(255,0,0)'
trades.loc[trades.Side == BUY, 'color'] = 'rgb(0,255,0)'
trace = Table(
columnwidth=[50, 20, 30, 15],
header=dict(values=['Time', 'Size', 'Price', 'Side'],
line=dict(color='#7D7F80'),
fill=dict(color='#a1c3d1'),
align='left'),
cells=dict(values=[pd.to_datetime(trades.Time), trades.Size.round(4), trades.Price, trades.Side],
line=dict(color='#7D7F80'),
fill=dict(color=[trades.color]),
align='left')
)
layout = dict(
title='Trades',
height=1100
)
return {'data': [trace], 'layout': layout}
def regr_table_plot(self):
header = dict(values=[
'<b>Name<b>', '<b>Date<b>', '<b>Seed<b>', '<b>CommitID<b>',
'<b>Status<b>'
],
line=dict(color='#7D7F80'),
fill=dict(color='#A1C3D1'),
font=dict(size=13),
align=[['middle'] * 5])
name_list = []
date_list = []
seed_list = []
id_list = []
sts_list = []
regr_db = self._regr_db
regr_db.reverse()
for idx in range(len(regr_db)):
db = regr_db[idx]
name_list.append("<a href='test_status_" + db['start_time'] +
".html'>" + self._cfg['regression_name'] + '</a>')
date_list.append(db['start_time'])
seed_list.append(db['plan_seed'])
id_list.append(db['unique_id'])
sts_list.append(db['status'])
cell_val = dict(
values=[name_list, date_list, seed_list, id_list, sts_list],
line=dict(color='#7D7F80'),
fill=dict(color='#EDFAFF'),
font=dict(size=13),
align=[['middle'] * 5])
trace = Table(header=header, cells=cell_val)
data = [trace]
layout = dict(paper_bgcolor='rgba(240,240,240,0.85)',
plot_bgcolor='rgba(240,255,255,0.85)')
fig = dict(data=data, layout=layout)
off.plot(fig,
filename=self._cfg['regression_name'] + "_table.html",
auto_open=False)
def main():
allFiles = open("diffFiles.txt")
fileList = allFiles.read().splitlines()
if (len(fileList) % 4 != 0):
print("diffFiles must be multiple of four")
return
i = 0
accsList = []
precsList = []
timeList = []
comList = []
while i < len(fileList) - 3:
ourFile = open(fileList[i])
expFile = open(fileList[i + 1])
rawOurOutput = ourFile.read().splitlines()
outSet = set(rawOurOutput)
rawExpOutput = expFile.read().splitlines()
expSet = set(rawExpOutput)
accSet = expSet - outSet
precSet = outSet - expSet
accuracy = float(len(expSet) - len(accSet)) / float(len(expSet))
precision = float(len(outSet) - len(precSet)) / float(len(outSet))
accsList.append(accuracy)
precsList.append(precision)
comList.append(fileList[i + 2])
timeList.append(fileList[i + 3])
i += 4
#producing bar graphs for accuracy, precision, and time
plotly.offline.plot(
{
"data": [Bar(x=comList, y=accsList, width=1)],
"layout":
Layout(title="Bug number vs Accuracy Graph",
xaxis=dict(title='Bug number'),
yaxis=dict(title='percent of bug fix lines found',
range=[0, 1.05]))
},
filename='accuracyGraph.html',
auto_open=False)
plotly.offline.plot(
{
"data": [Bar(x=comList, y=precsList, width=1)],
"layout":
Layout(
title="Bug number vs Precision Graph",
xaxis=dict(title='Bug number'),
yaxis=dict(
title=
'percent of tools bug fix lines valid(valid means line in expected)',
range=[0, 1.05]))
},
filename='precisionGraph.html',
auto_open=False)
plotly.offline.plot(
{
"data": [Bar(x=comList, y=timeList, width=1)],
"layout":
Layout(title="Bug number vs Time Graph",
xaxis=dict(title='Bug number'),
yaxis=dict(title='time for tool to run(seconds)'))
},
filename='timeGraph.html',
auto_open=False)
#producing tables for accuracy, precision, and time
plotly.offline.plot(
{
"data": [
Table(header=dict(
values=['Bug number', 'percent of bug fix lines found']),
cells=dict(values=[comList, accsList]))
],
"layout":
Layout(title="Bug number vs accuracy")
},
filename='accuracyTable.html',
auto_open=False)
plotly.offline.plot(
{
"data": [
Table(header=dict(values=[
'Bug number',
'percent of tools bug fix lines valid(valid means line in expected)'
]),
cells=dict(values=[comList, precsList]))
],
"layout":
Layout(title="Bug number vs precision")
},
filename='precisionTable.html',
auto_open=False)
plotly.offline.plot(
{
"data": [
Table(header=dict(
values=['Bug number', 'time for tool to run(seconds)']),
cells=dict(values=[comList, timeList]))
],
"layout":
Layout(title="Bug number vs Time Graph")
},
filename='timeTable.html',
auto_open=False)
def index():
# extract data needed for visuals
# visualization 1- count by genre
genre_counts = df.groupby('genre').count()['message']
genre_names = list(genre_counts.index)
# visualization 2- results from the classification report
result_df = report_df
# visualization 3- Sankey Diagram illustrating links between genres and categories
transformed_df = transform_df(df)
node_dict = create_node_dict(transformed_df)
link_df = create_link_df(transformed_df, node_dict)
nodes_df = create_nodes_df(node_dict)
# create visuals
graphs = [{
'data': [
Table(header=dict(values=list(result_df.columns),
fill_color='paleturquoise',
align='left'),
cells=dict(values=[
result_df.categories[:-4], result_df.f1[:-4],
result_df.precision[:-4], result_df.recall[:-4],
result_df.support[:-4]
],
fill_color='lavender',
align='left'))
],
'layout': {
'title': 'Classification Report Table',
}
}, {
'data': [Bar(x=genre_names, y=genre_counts)],
'layout': {
'title': 'Distribution of Message Genres',
'yaxis': {
'title': "Count"
},
'xaxis': {
'title': "Genre"
}
}
}, {
'data': [
dict(type='sankey',
orientation="h",
valueformat=".0f",
node=dict(pad=10,
thickness=30,
line=dict(color="black", width=0.5),
label=nodes_df['Label']),
link=dict(source=link_df['Source'].dropna(axis=0, how='any'),
target=link_df['Target'].dropna(axis=0, how='any'),
value=link_df['Value'].dropna(axis=0, how='any')))
],
'layout': {
'title': 'Message Genre to Category Classification',
'yaxis': {
'title': "Count"
},
'xaxis': {
'title': "Genre"
}
}
}]
# encode plotly graphs in JSON
ids = ["graph-{}".format(i) for i, _ in enumerate(graphs)]
graphJSON = json.dumps(graphs, cls=plotly.utils.PlotlyJSONEncoder)
# render web page with plotly graphs
return render_template('master.html', ids=ids, graphJSON=graphJSON)
def graph_02(self):
df = self.PROCDATASET
trace = Table(
header=dict(
values=list(df.columns[:len(df.columns)]),
fill = dict(color='#C2D4FF'),
align = 'center'
),
cells=dict(
values=[df[i].tolist() for i in df.columns[:len(df.columns)]],
fill = dict(color='#F5F8FF'),
align = ['left','center']
)
)
data = [trace]
if self._type_result == "jupyter-notebook":
iplot(data, filename = 'pandas_table')
elif self._type_result == "dash":
return dcc.Graph(
id='V009@1',
figure={"data": data}
)
def graph_03(self):
legend = {"title":"adicionar titulo",
"xaxis":"",
"yaxis":"",
}
if (self._language == "en"):
legend = {"title":"adicionar titulo", #bacalhau
"xaxis":"",
"yaxis":"",
}
color = ["rgb(127,0,0)","rgb(255,0,0)","rgb(127,0,127)","rgb(0,0,255)","rgb(0,127,127)","rgb(0,255,0)"]
dft = self.PROCDATASET
df = dft.set_index("Time")
sizeref = 0.01
trace = []
for i in range(0, len(df.columns[1:])):
trace.append(
Scatter(
x=df.index.get_level_values(0).values, #time
y=df.iloc[:,i].tolist(), #actions
hoverinfo='x+y',
mode='lines',
#stackgroup='one'
)
)
layout = Layout(
)
data = trace
fig=Figure(data=data, layout=layout)
if self._type_result == "jupyter-notebook":
iplot(fig, filename='Scatter')
elif self._type_result == "dash":
return dcc.Graph(
id='V009@3',
figure=fig
)
def index():
'''# extract data needed for visuals
# TODO: Below is an example - modify to extract data for your own visuals
genre_counts = df.groupby('genre').count()['message']
genre_names = list(genre_counts.index)
# find the 10 most common categories
#get df_cat with category cols only
df_cat= df.drop(['id', 'message', 'original', 'genre'], axis=1)
# Sum all cats, sort and get largest 10 vals
ranks=df_cat.sum().sort_values(ascending= False)[:10]
rank_vals= ranks.values
col_names= list(ranks.index)'''
# list of columns of interest in dataset
cols = list(m.mobil_df_new.columns[4:-1])
#graphs to graph
graphs = []
# TODO: show period at which countries reached a min mobility
# get timeline values for countries (using min week method)
X, Y = m.show_timeline(cols=cols, dates=m.dates_df_week.dropna())
# prepare figure for a scatter plot
fig = Figure()
# to remove long form of column
clean_str = lambda s: s.replace('_percent_change_from_baseline', '')
# iterate through everly location type
for i, col in enumerate(cols):
# add location type to figure
fig.add_trace(
Scatter(x=X[i], y=Y[i], mode='markers', name=clean_str(col)))
# add title and labels
fig.update_layout(
title="Dates at which countries reached slowest mobility",
xaxis_title="dates",
yaxis_title="col seperator (this is a 1-d graph)")
# add fig to graphs
graphs.append(fig)
'''Now to show dates of min for continents (average)'''
# TODO: show period at which continents reached a min mobility
continent_dates = m.get_continent_mean_date(
df=m.dates_df_week, continent_col=m.min_week_df.continent)
# get timeline values for continents
X, Y = m.show_timeline(cols=cols, dates=continent_dates)
# prepare figure for a scatter plot
fig = Figure()
# iterate through everly location type
for i, col in enumerate(cols):
# add location type to figure
fig.add_trace(
Scatter(x=X[i], y=Y[i], mode='markers', name=clean_str(col)))
# add title and labels
fig.update_layout(
title="Mean Dates at which continents reached slowest mobility",
xaxis_title="dates",
yaxis_title="col seperator (this is a 1-d graph)")
# add fig to graphs
graphs.append(fig)
# TODO: graph mean of slowest mobility drop for each continent
# get the means for each col
continent_mean_df = m.get_continent_mean(df=m.min_week_df)
# prepare a new fig
fig = Figure()
# y axis is constant
y = list(continent_mean_df.index)
for col in continent_mean_df:
# get col vals
x = continent_mean_df[col].values
fig.add_trace(Bar(x=x, y=y, orientation='h', name=clean_str(col)))
# add title and labels
fig.update_layout(title="Mean of min mobility for each continent",
xaxis_title="percent change from baseline",
yaxis_title="continent")
# add to graph
graphs.append(fig)
# TODO: show countries with earliest and latest min mobility
# get dates col with columns in short form
dates_df_week_clean = m.clean_col_name(m.dates_df_week)
# countries with earliest mobility slow-down (starts with earliest country)
ranks_earliest = dates_df_week_clean.apply(m.earliest_countries,
axis=0).transpose()
# create table
fig = Figure(data=[
Table(header=dict(values=list(ranks_earliest.index)),
cells=dict(values=ranks_earliest.values))
])
# add title
fig.update_layout(title="Countries with earliest min mobility")
# add table to graphs
graphs.append(fig)
# countries with latest mobility slow-down (starts with earliest country)
ranks_latest = dates_df_week_clean.apply(m.latest_countries,
axis=0).transpose()
#create table
fig = Figure(data=[
Table(header=dict(values=list(ranks_latest.index)),
cells=dict(values=ranks_latest.values))
])
# add title
fig.update_layout(title="Countries with latest min mobility")
# add table to graphs
graphs.append(fig)
# TODO: get a table of countries with most and least mobility drop
# modify min_week_df to have only baseline cols, and to have residential be consistent with other lambda functions
min_week_df_mod = m.min_week_df.copy()
# flip sign of values
min_week_df_mod.residential_percent_change_from_baseline = -m.min_week_df.residential_percent_change_from_baseline
# select columns of interest
min_week_df_mod = min_week_df_mod[min_week_df_mod.columns[3:]]
# 10 countries with greatest mobility drop
rank_highest = m.clean_col_name(min_week_df_mod).apply(
m.get_first_n).transpose()
#create table
fig = Figure(data=[
Table(header=dict(values=list(rank_highest.index)),
cells=dict(values=rank_highest.values))
])
# add title
fig.update_layout(
title="Countries that experienced greatest mobility drop")
# add table to graphs
graphs.append(fig)
# 10 countries with least mobility drop
rank_least = m.clean_col_name(min_week_df_mod).apply(
m.get_last_n).transpose()
#create table
fig = Figure(data=[
Table(header=dict(values=list(rank_least.index)),
cells=dict(values=rank_least.values))
])
# add title
fig.update_layout(title="Countries that experienced least mobility drop")
# add table to graphs
graphs.append(fig)
# TODO: graph countries with most and least mobility drop
# iterate through every location type col
cols = list(m.mobil_df_new.columns[4:-1])
for col in cols:
# get a sorted Series largest countries
sort = m.sort(col, df=m.min_week_df.dropna())
# get x values
x_1 = sort.head(10).values
x_2 = sort.tail(10).values
title_1 = '{}: countries with greatest drop in mobility'.format(
clean_str(col))
# get y values
y_1 = sort.head(10).index
y_2 = sort.tail(10).index
title_2 = '{}: countries with least drop in mobility'.format(
clean_str(col))
if col != 'residential_percent_change_from_baseline':
graph_1 = {
'data': [Bar(x=x_1, y=y_1, orientation='h')],
'layout': {
'title': title_1,
'yaxis': {
'title': "Country"
},
'xaxis': {
'title': "Percent change from baseline"
}
}
}
graph_2 = {
'data': [Bar(x=x_2, y=y_2, orientation='h')],
'layout': {
'title': title_2,
'yaxis': {
'title': "Country"
},
'xaxis': {
'title': "Percent change from baseline"
}
}
}
else:
# flip title for 'residential'
graph_1 = {
'data': [Bar(x=x_1, y=y_1, orientation='h')],
'layout': {
'title': title_2,
'yaxis': {
'title': "Country"
},
'xaxis': {
'title': "Percent change from baseline"
}
}
}
graph_2 = {
'data': [Bar(x=x_2, y=y_2, orientation='h')],
'layout': {
'title': title_1,
'yaxis': {
'title': "Country"
},
'xaxis': {
'title': "Percent_change_from baseline"
}
}
}
# add graph to graphs
graphs.append(graph_1)
graphs.append(graph_2)
# spearman correlation
corr = m.mobil_df_new.corrwith(m.mobil_df_new.corona_cases,
method='spearman')
cols = list(map(clean_str, corr.index))
# get a table of correlations with coronavirus cases
fig = Figure(
data=[Table(header=dict(values=cols), cells=dict(values=corr.values))])
# add title
fig.update_layout(title="correlations with coronavirus cases")
# add to graphs
graphs.append(fig)
# encode plotly graphs in JSON
ids = ["graph-{}".format(i) for i, _ in enumerate(graphs)]
graphJSON = json.dumps(graphs, cls=plotly.utils.PlotlyJSONEncoder)
# render web page with plotly graphs
return render_template('master_1.html', ids=ids, graphJSON=graphJSON)
def index():
#First plot
graph_1 = df_codes.violation_category.value_counts().sort_values(ascending=False)
#Second plot
graph_2 = df_codes[['violation_category', 'violation_date']]
categories = pd.unique(graph_2.violation_category.values)
df_list = []
for category in categories:
df_temp = graph_2.query('violation_category == @category').groupby(
graph_2.violation_date.dt.to_period('M').rename('date')).count()['violation_date']
df_temp = pd.DataFrame(df_temp.T) # .reset_index(inplace=True)
df_temp.reset_index(inplace=True)
df_temp.columns = ['validation_date', 'counts']
df_temp.validation_date = df_temp.validation_date.astype('str')
df_list.append(df_temp)
# Table
df_dates = df_codes.groupby('violation_category').aggregate(['min', 'max'])['violation_date'].reset_index()
# create visuals
graphs = [
{
'data': [
Bar(
x = graph_1.index.tolist(),
y = graph_1.values,
#name = 'news',
width = 0.8,
marker = dict(
color='rgb(0,128,128)'
)
#orientation = 'h'
)
],
'layout': {
'title': 'Number of violations by category',
'yaxis': {
'title': 'Count'
},
'xaxis': {
'title': 'Category'
},
#'legend': {
# 'orientation': 'h',
# 'x': 0.33,
# 'y': 1.1
#},
'margin': {
#'l': 1
}
#'width': 740
}
},
{
'data': [
Scatter(
x=df_list[0].validation_date,
y=df_list[0].counts,
name = categories[0],
#marker= dict(
# color ='LightSkyBlue',
# size = 10,
# opacity=0.7,
# line=dict(
# color='MediumPurple',
# width=2
# )
#color = ['rgb(255, 127, 14)']
#),
mode = 'lines',
stackgroup='one' # define stack group
),
Scatter(
x=df_list[1].validation_date,
y=df_list[1].counts,
name = categories[1],
mode = 'lines',
stackgroup='one'
),
Scatter(
x=df_list[2].validation_date,
y=df_list[2].counts,
name=categories[2],
mode='lines',
stackgroup='one'
),
Scatter(
x=df_list[3].validation_date,
y=df_list[3].counts,
name=categories[3],
mode='lines',
stackgroup='one'
),
Scatter(
x=df_list[4].validation_date,
y=df_list[4].counts,
name=categories[4],
mode='lines',
stackgroup='one'
),
Scatter(
x=df_list[5].validation_date,
y=df_list[5].counts,
name=categories[5],
mode='lines',
stackgroup='one'
),
Scatter(
x=df_list[6].validation_date,
y=df_list[6].counts,
name=categories[6],
mode='lines',
stackgroup='one'
),
Scatter(
x=df_list[7].validation_date,
y=df_list[7].counts,
name=categories[7],
mode='lines',
stackgroup='one'
),
Scatter(
x=df_list[8].validation_date,
y=df_list[8].counts,
name=categories[8],
mode='lines',
stackgroup='one'
)
],
'layout': {
'title': 'Violation dates per category',
'yaxis': {
'title': "Count",
'showgrid': True,
'nticks': 20
},
'xaxis': {
'title': "Dates",
'nticks': 20
},
'legend': {
'orientation': "h",
'x':0.33,
'y':1.18
},
'automargin': True
},
},
{
'data': [
Table(
header= {
'values': ["Type","Earliest<br>date", "Latest<br>date"]
},
cells= {
'values': [df_dates[k].tolist() for k in df_dates.columns]
}
)
],
'layout': {
'title': "Violation Dates",
#'height': 800
}
}
]
# encode plotly graphs in JSON
ids = ["graph-{}".format(i) for i, _ in enumerate(graphs)]
graphJSON = json.dumps(graphs, cls=plotly.utils.PlotlyJSONEncoder)
# render web page with plotly graphs
return render_template('master.html', ids=ids, graphJSON=graphJSON)
def draw_graph(self, df, row, col):
table_trace = Table(header=dict(values=df.columns),
cells=dict(values=[df[col] for col in df.columns]))
return table_trace