def generate_graphs():
global consumption_list
global total_power_consumption
global cummulative_consumption_list
time_list = []
name_list = []
power_list = []
html = ""
count = 0
for item in consumption_list:
if count == 0:
time_list.append(time(0, 0, 0))
cummulative_consumption_list[count] = item
else:
cummulative_consumption_list[
count] = item + cummulative_consumption_list[count - 1]
time_list.append(
(datetime.combine(date(1, 1, 1), time_list[count - 1]) +
timedelta(minutes=1)).time())
#print time_list[count]
count += 1
plot_data = [go.Scatter(x=time_list, y=consumption_list, name='Watts')]
layout = go.Layout(title=("Simulated Total Power Consumption"),
xaxis=dict(title="Time"),
yaxis=dict(title="Power consumption in Watts"))
py_fig = dict(data=plot_data, layout=layout)
plot_html, plotdivid, width, height = _plot_html(py_fig, False, "", True,
'100%', 525, False)
#html = "<div class=\"row\"><h3>Overall Power Consumption</h3>"
html += "<div class=\"row\">%s</div>" % plot_html
for appliance in appliance_list:
name_list.append(appliance.get_name())
power_list.append(appliance.get_total_watts())
plot_data = [go.Bar(x=name_list, y=power_list, name='Power Consumption')]
layout = go.Layout(title=("Simulated Appliance Power Consumption"),
xaxis=dict(title="Appliance"),
yaxis=dict(title="Power consumption in Watts"))
py_fig = dict(data=plot_data, layout=layout)
plot_html, plotdivid, width, height = _plot_html(py_fig, False, "", True,
'100%', 525, False)
#html += "<div class=\"row\"><h3>Appliance Consumption</h3>"
html += "\n<div class=\"row\">%s</div>" % plot_html
plot_data = [
go.Scatter(x=time_list, y=cummulative_consumption_list, name='Watts')
]
layout = go.Layout(title=("Cummulative Total Power Consumption"),
xaxis=dict(title="Time"),
yaxis=dict(title="Power consumption in Watts"))
py_fig = dict(data=plot_data, layout=layout)
plot_html, plotdivid, width, height = _plot_html(py_fig, False, "", True,
'100%', 525, False)
html += "\n<div class=\"row\">%s</div>" % plot_html
return (html, total_power_consumption, consumption_list)
def report_category(number):
category = num2category[number]['name']
with pd.HDFStore(args.report, 'r') as store:
plotly_data = []
correct, wrong, count, accuracy, top_level_accuracy = category_stats(number)
plotly_data.append(go.Scatter(
x=np.linspace(0,100, num=len(correct)),
y=correct.score,
mode='lines',
name='Correct',
hoverinfo='name+y',
text=[ json.dumps({ 'path': path, 'prediction': category })
for path in correct.index ]))
wrong_categories = [ num2category[c]['name'] for c in wrong.category ]
plotly_data.append(go.Scatter(
x=np.linspace(0,100, num=len(wrong)),
y=wrong.score,
mode='lines',
name='Wrong',
hoverinfo='name+y',
text=[ json.dumps({ 'path': path, 'prediction': prediction })
for path, prediction in zip(wrong.index, wrong_categories)]))
layout = go.Layout(hovermode='closest', title='Performance of correct vs wrong classified pictures', xaxis={'title': '%'}, yaxis={'title': 'score'})
figure = go.Figure(data=plotly_data, layout=layout)
performance_plot, performance_id, _,_ = _plot_html(figure, False, '', True, '50%', '100%', False)
categories_counter = Counter(wrong_categories)
labels, values = zip(*categories_counter.items())
pie = go.Pie(labels=labels, values=values, showlegend=False, textinfo='text', text=[None]*len(values))
layout= go.Layout(hovermode='closest', title='Wrongly classified pictures ({}%) labelled {}'.format(np.round(100-accuracy, 1), category))
figure = go.Figure(data=[pie], layout=layout)
pie, _, _, _ = _plot_html(figure, False, '', True, '100%', '100%', False)
wrong_as_this = store['{}/wrong/in'.format(number)]
wrong_out = sorted(zip(wrong.index, [ num2category[c]['name'] for c in wrong.category], wrong.score), key=lambda x: x[2], reverse=True)
wrong_in = sorted(zip(wrong_as_this.index, [ num2category[c]['name'] for c in wrong_as_this.category], wrong_as_this.score), key=lambda x: x[2], reverse=True)
return flask.render_template('report.html', accuracy=accuracy, category=category, performance_plot=performance_plot, performance_id=performance_id,
pie=pie, wrong_out=wrong_out, wrong_in = wrong_in, count=count, top_level_accuracy=top_level_accuracy)
def _ipython_display_(self):
disable_ipython_scrollbar()
xaxis, yaxis = self.config['x'], self.config.get('y', None)
yaxes = [yaxis] if yaxis is not None else \
[col for col in self.table.columns if col != xaxis]
xvals = self.table[xaxis].tolist()
traces = [
dict(x=xvals, y=self.table[axis].tolist(), name=axis)
for axis in yaxes
]
layout = dict(
xaxis=dict(title=xaxis),
yaxis=dict(title=self.config['table']),
legend=dict(x=0.7, y=1),
margin=dict(r=0, t=40),
)
fig = dict(data=traces, layout=layout)
display(
HTML(
_plot_html(fig,
False,
'',
True,
'100%',
525,
global_requirejs=True)[0]))
def plot_to_html(self, fig):
plotly_html_div, plotdivid, width, height = _plot_html(
figure_or_data=fig,
config="",
validate=True,
default_width='90%',
default_height="100%",
global_requirejs=False)
return plotly_html_div
def plot_to_html(self, fig):
plotly_html_div, plotdivid, width, height = _plot_html(
figure_or_data=fig,
config="",
validate=True,
default_width='90%',
default_height="100%",
global_requirejs=False)
return plotly_html_div
def plot2(mini, maxi, len): # plotly
try:
data = [{
'x': list(range(len)),
'y': np.random.randint(mini, maxi, len)
}]
plot_html, plotdivid, width, height = _plot_html(
data, '', False, '80%', 525, False, False)
return plot_html
except:
return '<h1><font color="red">plot failed!</font></h1>'
def create_climate_chart(
cls, df: pd.DataFrame, var_climate, year_week,
climate_crit, climate_title
) -> 'Plotly_HTML':
"""
:param df:
:param var_climate:
:param year_week:
:param climate_crit:
:param climate_title:
:return:
"""
k = var_climate.replace('_', '.')
df_climate = df.reset_index()[['SE', k]]
df_climate = df_climate[
df_climate.SE >= year_week - 200
]
df_climate['SE'] = df_climate.SE.map(
lambda v: '%s/%s' % (str(v)[:4], str(v)[-2:])
)
df_climate['Limiar favorável transmissão'] = climate_crit
figure = df_climate.iplot(
asFigure=True, x=['SE'], y=[k, 'Limiar favorável transmissão'],
showlegend=True,
yTitle=climate_title, xTitle='Período (Ano/Semana)'
)
figure['layout']['xaxis1'].update(
tickangle=-60, nticks=len(df_climate)//4
)
figure['layout']['legend'].update(
x=-.1, y=1.2,
traceorder='normal',
font=dict(
family='sans-serif',
size=12,
color='#000'
),
bgcolor='#FFFFFF',
bordercolor='#E2E2E2',
borderwidth=1
)
return _plot_html(
figure_or_data=figure, config={}, validate=True,
default_width='100%', default_height=500, global_requirejs=''
)[0]
def create_tweet_chart(
cls, df: pd.DataFrame, year_week
) -> 'Plotly_HTML':
"""
:param df:
:param var_climate:
:param year_week:
:param climate_crit:
:param climate_title:
:return:
"""
df_tweet = df.reset_index()[['SE', 'tweets']]
df_tweet = df_tweet[
df_tweet.SE >= year_week - 200
]
df_tweet['SE'] = df_tweet.SE.map(
lambda v: '%s/%s' % (str(v)[:4], str(v)[-2:])
)
df_tweet.rename(columns={'tweets': 'menções'}, inplace=True)
figure = df_tweet.iplot(
x=['SE'],
y=['menções'], asFigure=True,
showlegend=True, xTitle='Período (Ano/Semana)'
)
figure['layout']['xaxis1'].update(
tickangle=-60, nticks=len(df_tweet)//4
)
figure['layout']['yaxis1'].update(title='Tweets')
figure['layout']['legend'].update(
x=-.1, y=1.2,
traceorder='normal',
font=dict(
family='sans-serif',
size=12,
color='#000'
),
bgcolor='#FFFFFF',
bordercolor='#E2E2E2',
borderwidth=1
)
return _plot_html(
figure_or_data=figure, config={}, validate=True,
default_width='100%', default_height=500, global_requirejs=''
)[0]
def create_climate_chart(cls, df: pd.DataFrame, var_climate, year_week,
climate_crit, climate_title) -> 'Plotly_HTML':
"""
:param df:
:param var_climate:
:param year_week:
:param climate_crit:
:param climate_title:
:return:
"""
k = var_climate.replace('_', '.')
df_climate = df.reset_index()[['SE', k]]
df_climate = df_climate[df_climate.SE >= year_week - 200]
df_climate['SE'] = df_climate.SE.map(lambda v: '%s/%s' %
(str(v)[:4], str(v)[-2:]))
df_climate['Limiar favorável transmissão'] = climate_crit
figure = df_climate.iplot(asFigure=True,
x=['SE'],
y=[k, 'Limiar favorável transmissão'],
showlegend=True,
yTitle=climate_title,
xTitle='Período (Ano/Semana)')
figure['layout']['xaxis1'].update(tickangle=-60,
nticks=len(df_climate) // 4)
figure['layout']['legend'].update(x=-.1,
y=1.2,
traceorder='normal',
font=dict(family='sans-serif',
size=12,
color='#000'),
bgcolor='#FFFFFF',
bordercolor='#E2E2E2',
borderwidth=1)
return _plot_html(figure_or_data=figure,
config={},
validate=True,
default_width='100%',
default_height=500,
global_requirejs='')[0]
def html(fig):
config = dict(showLink=False, displaylogo=False)
plot_html, plotdivid, w, h = pl_offline._plot_html(
fig,
config,
validate=True,
default_width='100%',
default_height='100%',
global_requirejs=False)
script_split = plot_html.find('<script ')
plot_content = {
'div': plot_html[:script_split],
'script': plot_html[script_split:],
'id': plotdivid
}
return plot_content
def __init__(self,
figure,
click_callback=None,
relayout_callback=None,
**kwargs):
super().__init__(**kwargs)
html, uid, _, _ = _plot_html(figure, False, "", True, None, None,
True) #{1}
self.uid = str(uid)
jscode = JsTemplate.render(uid=self.uid,
onclick=click_callback is not None,
onrelayout=relayout_callback is not None)
html += jscode #{2}
self.value = html #{3}
self.click_callback = click_callback
self.relayout_callback = relayout_callback
self.comm_manager.register_target(self.uid, self._on_open) #{4}
def plotly_hack(fig):
r"""Work around to get plotly on html.
Parameters
----------
fig : plotly figure
Returns
-------
html : string
the html string
"""
from plotly.offline.offline import _plot_html
plot_html, plotdivid, width, height = _plot_html(
fig, False, "", True, '100%', 525, False)
return plot_html
def getplotly(dataseries, layout):
config = {}
config['showLink'] = False
figure_or_data = {"data": dataseries, "layout": layout}
plot_html = plot_html, plotdivid, width, height = _plot_html(
figure_or_data, config, False, '100%', 600, False)
resize_script = ''
if width == '100%' or height == '100%':
resize_script = (
''
'<script type="text/javascript">'
'window.removeEventListener("resize");'
'window.addEventListener("resize", function(){{'
'Plotly.Plots.resize(document.getElementById("{id}"));}});'
'</script>').format(id=plotdivid)
graph = ''.join([plot_html, resize_script])
return graph
def opportunities_boxplot(df: pd.DataFrame, week: int=None):
"""
:param df:
:param week:
:return:
"""
title_param = {
'name': df.territory_name.unique()[0],
'week': ''
}
if week not in [0, None]:
title_param['week'] = 'até a semana epidemiológica %s ' % week
df_plot = df.iloc[:, :-1].dropna(how='all', axis=1)
# Set ymax to higher upper fence:
q1 = df_plot.quantile(.25, axis=0)
q3 = df_plot.quantile(.75, axis=0)
ymax = (q3 + 1 * (q3 - q1)).max()
# Set bottom and right margin to avoid xaxis labels beeing cut off
figure = df_plot.iplot(
kind='box',
boxpoints=False,
margin={'b': 130,
't': 50,
'r': 120,
'l': 50
},
showlegend=False,
yTitle='Dias',
asFigure=True
)
figure['layout']['yaxis1'].update(range=[0, ymax])
figure['layout']['xaxis1'].update(tickangle=30)
return _plot_html(
figure_or_data=figure, config={}, validate=True,
default_width='100%', default_height=450, global_requirejs=''
)[0]
def create_tweet_chart(cls, df: pd.DataFrame, year_week) -> 'Plotly_HTML':
"""
:param df:
:param var_climate:
:param year_week:
:param climate_crit:
:param climate_title:
:return:
"""
df_tweet = df.reset_index()[['SE', 'tweets']]
df_tweet = df_tweet[df_tweet.SE >= year_week - 200]
df_tweet['SE'] = df_tweet.SE.map(lambda v: '%s/%s' %
(str(v)[:4], str(v)[-2:]))
df_tweet.rename(columns={'tweets': 'menções'}, inplace=True)
figure = df_tweet.iplot(x=['SE'],
y=['menções'],
asFigure=True,
showlegend=True,
xTitle='Período (Ano/Semana)')
figure['layout']['xaxis1'].update(tickangle=-60,
nticks=len(df_tweet) // 4)
figure['layout']['yaxis1'].update(title='Tweets')
figure['layout']['legend'].update(x=-.1,
y=1.2,
traceorder='normal',
font=dict(family='sans-serif',
size=12,
color='#000'),
bgcolor='#FFFFFF',
bordercolor='#E2E2E2',
borderwidth=1)
return _plot_html(figure_or_data=figure,
config={},
validate=True,
default_width='100%',
default_height=500,
global_requirejs='')[0]
def plot_plotly(chart, width='100%', height=525):
# produce the html in Ipython compatible format
plot_html, plotdivid, width, height = _plot_html(chart, {'showLink': False}, True, width, height, True)
# define the plotly js library source url
head = '<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>'
# extract the div element from the ipython html
div = plot_html[0:plot_html.index('<script')]
# extract the script element from the ipython html
script = plot_html[plot_html.index('Plotly.newPlot'):plot_html.index('});</script>')] + ';'
# combine div and script to build the body contents
body = '<body>{div}<script>{script}</script></body>'.format(div=div, script=script)
# instantiate a figure object
figure = Figure()
# add the head
figure.header.add_child(Element(head))
# add the body
figure.html.add_child(Element(body))
return figure
def offline_plotly_data(data,
filename='plotly.html',
config=None,
validate=True,
default_width='100%',
default_height=525,
global_requirejs=False):
""" Write a plotly scatter plot to HTML file that doesn't require server
>>> import pandas as pd
>>> from plotly.offline.offline import _plot_html
>>> from plotly.graph_objs import Scatter, Marker, Layout, YAxis, XAxis
>>> df = pd.read_csv('https://plot.ly/~etpinard/191.csv')
>>> df.columns = [eval(c) if c[0] in '"\'' else str(c) for c in df.columns]
>>> data = {'data': [
>>> Scatter(x=df[continent+', x'],
>>> y=df[continent+', y'],
>>> text=df[continent+', text'],
>>> marker=Marker(size=df[continent+', size'], sizemode='area', sizeref=131868,),
>>> mode='markers',
>>> name=continent) for continent in ['Africa', 'Americas', 'Asia', 'Europe', 'Oceania']
>>> ],
>>> 'layout': Layout(xaxis=XAxis(title='Life Expectancy'), yaxis=YAxis(title='GDP per Capita', type='log'))
>>> }
>>> html = offline_plotly_data(data)
"""
config_default = dict(DEFAULT_PLOTLY_CONFIG)
if config is not None:
config_default.update(config)
with open(os.path.join(DATA_PATH, 'plotly.js.min'), 'rt') as f:
js = f.read()
html, divid, width, height = _plot_html(data,
config=config_default,
validate=validate,
default_width=default_width,
default_height=default_height,
global_requirejs=global_requirejs)
html = PLOTLY_HTML.format(plotlyjs=js, plotlyhtml=html)
if filename and isinstance(filename, str):
with open(filename, 'wt') as f:
f.write(html)
return html
def opportunities_boxplot(df: pd.DataFrame, week: int = None):
"""
:param df:
:param week:
:return:
"""
title_param = {'name': df.territory_name.unique()[0], 'week': ''}
if week not in [0, None]:
title_param['week'] = 'até a semana epidemiológica %s ' % week
df_plot = df.iloc[:, :-1].dropna(how='all', axis=1)
# Set ymax to higher upper fence:
q1 = df_plot.quantile(.25, axis=0)
q3 = df_plot.quantile(.75, axis=0)
ymax = (q3 + 1 * (q3 - q1)).max()
# Set bottom and right margin to avoid xaxis labels beeing cut off
figure = df_plot.iplot(kind='box',
boxpoints=False,
margin={
'b': 130,
't': 50,
'r': 120,
'l': 50
},
showlegend=False,
yTitle='Dias',
asFigure=True)
figure['layout']['yaxis1'].update(range=[0, ymax])
figure['layout']['xaxis1'].update(tickangle=30)
return _plot_html(figure_or_data=figure,
config={},
validate=True,
default_width='100%',
default_height=450,
global_requirejs='')[0]
def ethio_ts(df=pd.DataFrame, scale_id=int, year=int):
cols = [
'Testes positivos',
'Influenza A',
'SARS-CoV-2',
'Influenza B',
'VSR',
'Adenovirus',
'Parainfluenza 1',
'Parainfluenza 2',
'Parainfluenza 3',
]
trace = []
if scale_id == 2:
ytitle = 'Casos'
else:
ytitle = 'Incidência'
trace.append(
go.Scatter(
x=df['epiweek'],
y=df[cols[0]],
name=cols[0],
mode='lines',
))
for i, col in enumerate(cols[1:]):
trace.append(
go.Scatter(x=df['epiweek'],
y=df[col],
name=ytitle,
mode='lines',
showlegend=False))
nrows = len(cols)
fig = tools.make_subplots(
rows=nrows,
cols=1,
print_grid=False,
subplot_titles=('Situação dos exames', 'Influenza A', 'SARS-CoV-2',
'Influenza B', 'VSR', 'Adenovirus', 'Parainfluenza 1',
'Parainfluenza 2', 'Parainfluenza 3'))
for i in range(1, (nrows + 1)):
fig.append_trace(trace[i - 1], i, 1)
# Add extra lines to first subplot:
extra_cols = [
'Testes negativos', 'Casos sem teste laboratorial',
'Casos aguardando resultado', 'Casos sem informação laboratorial'
]
for i, col in enumerate(extra_cols):
fig.append_trace(
go.Scatter(x=df['epiweek'],
y=df[col],
name=col,
mode='lines',
line=dict(color=cl.scales['12']['qual']['Paired'][-i])),
1, 1)
# X-axis title and range:
lastepiweek = int(episem(lastepiday(year), out='W'))
extra_cols.extend(['Testes positivos'])
if scale_id == 2:
ymax = [
max(5 * np.ceil(df[extra_cols].max().max() / 5), 5),
max(5 * np.ceil(df[cols[1:]].max().max() / 5), 5)
]
else:
ymax = [df[extra_cols].max().max(), df[cols[1:]].max().max()]
for i in range(1, (nrows + 1)):
xaxis = 'xaxis%s' % i
yaxis = 'yaxis%s' % i
fig['layout'][xaxis].update(range=[1, lastepiweek])
fig['layout'][yaxis].update(range=[0, ymax[min(1, i - 1)]],
rangemode='nonnegative')
fig['layout']['xaxis%s' % nrows].update(title='Semana epidemiológica',
zeroline=True,
showline=True)
i = int(nrows / 2)
fig['layout']['yaxis%s' % i].update(title=ytitle)
fig['layout'].update(margin={'t': 50, 'l': 50})
return _plot_html(figure_or_data=fig,
config={},
validate=True,
default_width='100%',
default_height=1600,
global_requirejs='')[0]
</head>
<body>
<table>
'''
for model in args.models:
plotly_data, accuracy, top_k_accuracy, stats = evaluate(model, h5_files, args.top_k, categories)
layout= go.Layout(
showlegend=False,
hovermode='closest')
figure = go.Figure(data=plotly_data, layout=layout)
plot_html, plotdivid, width, height = _plot_html(figure, False, '', True, '100%', '100%', False)
html += '<tr><td colspan=2><h3>{}</h3>Overall accuracy: {} Overall top-{} accuracy: {}</td></tr><tr><td>{}</td>'.format(os.path.basename(model),
accuracy, args.top_k,
top_k_accuracy, plot_html)
html += '''
<td>
<img id="hover-image-'''+str(plotdivid)+'''" src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7">
<div id="hoverinfo-'''+str(plotdivid)+'''"></div>
</td>
</tr>
<script>
var myPlot = document.getElementById("''' + str(plotdivid) + '''"),
hoverInfo = document.getElementById("hoverinfo-'''+str(plotdivid)+'''"),
xaxis=dict(
title="False positive (hh100%={}, yt100%={})".format(
2649, 4254),
range=[0, 2000],
),
yaxis=dict(
title="True positive rate",
range=[0, 1],
))
# Plot and embed in ipython notebook!
fig = dict(data=data, layout=layout)
#df=iplot(fig)
# plot_html, plotdivid, width, height = _plot_html(fig, False, "", True, '100%', 525,'')
# _plot_html(figure_or_data, config, validate, default_width, default_height, global_requirejs)
plot_html, plotdivid, width, height = _plot_html(fig, "", True, '100%', 525,
'')
html_start = """
<html>
<head>
<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
</head>
<body>"""
html_end = """
</body>
</html>"""
html_final = html_start + plot_html + html_end
f = open("compare.html", 'w')
f.write(html_final)
def create_incidence_chart(cls, df: pd.DataFrame, year_week: int,
threshold_pre_epidemic: float,
threshold_pos_epidemic: float,
threshold_epidemic: float) -> 'Plotly_HTML':
"""
@see: https://stackoverflow.com/questions/45526734/
hide-legend-entries-in-a-plotly-figure
:param df:
:param year_week:
:param threshold_pre_epidemic: float,
:param threshold_pos_epidemic: float
:param threshold_epidemic: float
:return:
"""
df = df.reset_index()[[
'SE', 'incidência', 'casos notif.', 'level_code'
]]
# 200 = 2 years
df = df[df.SE >= year_week - 200]
df['SE'] = df.SE.map(lambda v: '%s/%s' % (str(v)[:4], str(v)[-2:]))
k = 'incidência'
df['alerta verde'] = df[df.level_code == 1][k]
df['alerta amarelo'] = df[df.level_code == 2][k]
df['alerta laranja'] = df[df.level_code == 3][k]
df['alerta vermelho'] = df[df.level_code == 4][k]
df['limiar epidêmico'] = threshold_epidemic
df['limiar pós epidêmico'] = threshold_pos_epidemic
df['limiar pré epidêmico'] = threshold_pre_epidemic
figure_bar = df.iplot(asFigure=True,
kind='bar',
x=['SE'],
y=[
'alerta verde', 'alerta amarelo',
'alerta laranja', 'alerta vermelho'
],
legend=True,
showlegend=True,
yTitle='Incidência',
xTitle='Período (Ano/Semana)',
color=[
'rgb(0,255,0)', 'rgb(255,255,0)',
'rgb(255,150,0)', 'rgb(255,0,0)'
],
hoverinfo='x+y+name')
figure_threshold = df.iplot(
asFigure=True,
x=['SE'],
y=[
'limiar pré epidêmico', 'limiar pós epidêmico',
'limiar epidêmico'
],
legend=False,
showlegend=False,
color=['rgb(0,255,0)', 'rgb(255,150,0)', 'rgb(255,0,0)'],
hoverinfo='none')
figure_line = df.iplot(asFigure=True,
x=['SE'],
y=['casos notif.'],
legend=False,
showlegend=False,
secondary_y=['casos notif.'],
secondary_y_title='Casos',
hoverinfo='x+y+name',
color=['rgb(33,33,33)'])
figure_line['layout']['legend'].update(
x=-.27,
y=0.5,
traceorder='normal',
font=dict(family='sans-serif', size=12, color='#000'),
bgcolor='#FFFFFF',
bordercolor='#E2E2E2',
borderwidth=1,
orientation='l',
)
figure_line['layout']['yaxis1'].update(title='Incidência')
figure_line['layout'].update(
title=('Limiares de incidência:: ' + 'pré epidêmico=%s; ' +
'pós epidêmico=%s; ' + 'epidêmico=%s;') %
('{:.1f}'.format(threshold_pre_epidemic),
'{:.1f}'.format(threshold_pos_epidemic),
'{:.1f}'.format(threshold_epidemic)),
showlegend=True)
figure_threshold.data.extend(figure_bar.data)
figure_line.data.extend(figure_threshold.data)
figure_line['layout']['yaxis2'].update(
showgrid=False, range=[0, df['casos notif.'].max()])
figure_line['layout']['xaxis1'].update(tickangle=-60,
nticks=len(df) // 4,
title='Período (Ano/Semana)')
for trace in figure_line['data']:
if trace['name'] == 'casos notif.':
trace['visible'] = 'legendonly'
return _plot_html(figure_or_data=figure_line,
config={},
validate=True,
default_width='100%',
default_height=500,
global_requirejs='')[0]
def draw_chart(self):
import plotly.graph_objs as go
from plotly.offline.offline import _plot_html
import pandas as pd
diarios = []
bancos = self.line_ids.filtered(lambda x: x.liquidity)
for item in bancos:
diarios.append((item.amount, item.name))
movimentacoes = []
for item in self.line_ids.filtered(lambda x: not x.liquidity):
movimentacoes.append((item.amount, item.date, item.line_type))
diarios = pd.DataFrame(diarios, columns=['total', 'name'])
moves = pd.DataFrame(
movimentacoes, columns=['total', 'date_maturity', 'type'])
moves['total'] = moves["total"].astype(float)
moves['date_maturity'] = pd.to_datetime(moves["date_maturity"])
moves['receitas'] = moves["total"]
moves['despesas'] = moves["total"]
moves.ix[moves.type == 'payable', 'receitas'] = 0.0
moves.ix[moves.type == 'receivable', 'despesas'] = 0.0
moves = moves.sort_values(by="date_maturity")
moves["acumulado"] = moves["total"].cumsum()
moves["acumulado"] += diarios["total"].sum()
saldo = []
saldo_inicial = 0.0
for index, row in diarios.iterrows():
saldo.append(go.Bar(
x=["Saldo"],
y=[row["total"]],
name=row["name"]
))
saldo_inicial += row["total"]
acumulado_x = pd.Series(["Saldo"])
acumulado_y = pd.Series([saldo_inicial])
trace3 = go.Bar(
x=moves['date_maturity'],
y=moves['receitas'],
name='Receitas',
)
trace4 = go.Bar(
x=moves['date_maturity'],
y=moves['despesas'],
name='Despesas',
)
moves.drop_duplicates(
subset='date_maturity', keep='last', inplace=True)
x = acumulado_x.append(moves["date_maturity"])
y = acumulado_y.append(moves["acumulado"])
trace5 = go.Scatter(
x=x,
y=y,
mode='lines+markers',
name="Saldo",
line=dict(
shape='spline'
),
)
data = [trace3, trace4, trace5]
layout = go.Layout(
barmode='stack',
xaxis=dict(
tickformat="%d-%m-%Y"
),
)
fig = go.Figure(data=data, layout=layout)
plot_html, plotdivid, width, height = _plot_html(
fig, {}, True, '100%', 525, False)
return plot_html
def create_tweet_chart(
cls, df: pd.DataFrame, year_week, disease: str
) -> 'Plotly_HTML':
"""
:param df:
:param year_week:
:param disease:
:return:
"""
ks_cases = [
'casos notif. {}'.format(disease),
]
df_tweet = df.reset_index()[['SE', 'tweets'] + ks_cases]
df_tweet = df_tweet[
df_tweet.SE >= year_week - 200
]
df_tweet.rename(columns={'tweets': 'menções'}, inplace=True)
df_grp = df_tweet.groupby(df.index)[
['menções'] + ks_cases
].sum().reset_index()
df_grp['SE'] = df_grp.SE.map(
lambda v: '%s/%s' % (str(v)[:4], str(v)[-2:])
)
fig_tweet = df_grp.iplot(
x=['SE'],
y=['menções'], asFigure=True,
showlegend=True, xTitle='Período (Ano/Semana)',
color=['rgb(128,128,128)']
)
fig_cases = df_grp.iplot(
x=['SE'], y=ks_cases, asFigure=True,
secondary_y=ks_cases, secondary_y_title='Casos',
showlegend=True, xTitle='Período (Ano/Semana)',
color=['rgb(0,0,255)']
)
fig_cases.data.extend(fig_tweet.data)
fig_cases['layout']['xaxis1'].update(
tickangle=-60,
nticks=len(df_grp) // 24
)
fig_cases['layout']['yaxis1'].update(
title='Tweets',
range=[0, df_grp['menções'].max()]
)
fig_cases['layout']['yaxis2'].update(
range=[0, df_grp[ks_cases].max().max()]
)
fig_cases['layout'].update(
title='Casos {} / Menções mídia social'.format(disease)
)
fig_cases['layout']['legend'].update(
x=-.1, y=1.2,
traceorder='normal',
font=dict(
family='sans-serif',
size=12,
color='#000'
),
bgcolor='#FFFFFF',
bordercolor='#E2E2E2',
borderwidth=1
)
return _plot_html(
figure_or_data=fig_cases, config={}, validate=True,
default_width='100%', default_height=300, global_requirejs=''
)[0]
#fig = tls.make_subplots(rows=2, cols=1) errorfig_vcc= go.Figure(data=error_vcc, layout = layout1) errorfig_vdbl= go.Figure(data=error_vdbl, layout = layout1) errorfig_vcc['layout'].update(title=device+' ICC Ratio_PMV')#height=600, width=600, errorfig_vdbl['layout'].update(title=device+' ICC Ratio_VDBL') #fig = go.Figure(data=databoth, layout=layout1) #fig.append_trace(error_vcc, 1, 1) #fig.append_trace(error_vdbl, 2, 1) #plot(errorfig) from plotly.offline.offline import _plot_html plot_html, plotdivid, width, height = _plot_html(errorfig_vcc, False, "", True, '100%', 525,global_requirejs=False) plot_html_1, plotdivid, width, height = _plot_html(errorfig_vdbl, False, "", True, '100%', 525,global_requirejs=False) file="<html><head><script src='https://cdn.plot.ly/plotly-latest.min.js'></script></head><body>" + plot_html+plot_html_1+ "</body></html>" Html_file= open(outfolder1+device+'_Die_Intercept.html',"w") Html_file.write(file) Html_file.close() print(device + " Die ICC Done")
combtest100 = scatter100 + bar100
figtest0 = go.Figure(data=combtest0, layout=layout)
figtest0['layout'].update(barmode='stack')
figtest0['layout'].update(yaxis=dict(title='Count'))
figtest0['layout'].update(title=device + ' Lot Level Test Program Plot: 0C')
figtest100 = go.Figure(data=combtest100, layout=layout)
figtest100['layout'].update(barmode='stack')
figtest100['layout'].update(yaxis=dict(title='Count'))
figtest100['layout'].update(title=device +
' Lot Level Test Program Plot: 100C')
from plotly.offline.offline import _plot_html
plot_test_0, plotdivid, width, height = _plot_html(
figtest0, False, "", True, '100%', 525,
global_requirejs=False) #,global_requirejs=False
plot_test_100, plotdivid, width, height = _plot_html(
figtest100, False, "", True, '100%', 525,
global_requirejs=False) #,global_requirejs=False
file_test0 = "<html><head><script src='https://cdn.plot.ly/plotly-latest.min.js'></script></head><body>" + plot_test_0 + "</body></html>"
file_test100 = "<html><head><script src='https://cdn.plot.ly/plotly-latest.min.js'></script></head><body>" + plot_test_100 + "</body></html>"
html_test0 = open(outfolder2 + "/" + "Lot_WS_Rev_0.html", "w")
html_test0.write(file_test0)
html_test0.close()
html_test100 = open(outfolder2 + "/" + "Lot_WS_Rev_100.html", "w")
html_test100.write(file_test100)
def query(compares, targets, between, by, chartMode, barMode, pieMode, sortBy, jsonfile):
# compares = ['assigned', 'not_assigned', 'guest']
# target = {"m":"","h":"","s": "","p":""}
# between = 'masseur'|'helper'|'shop'|'date'
conn = pymysql.connect(host='localhost', port=3306, user='******', passwd='d4sg', db='d4sg_vim',charset='utf8')
cur = conn.cursor()
cur.execute("SELECT wid,mid,hid,sid,assigned,not_assigned,guest_num,log_date FROM worklog")
worklogs = list(cur.fetchall())
cur.execute("SELECT * FROM masseur")
masseurs = list(cur.fetchall())
cur.execute("SELECT * FROM helper")
helpers = list(cur.fetchall())
cur.execute("SELECT * FROM shop")
shops = list(cur.fetchall())
# find target
target_masseurs = masseurs
target_helpers = helpers
target_shops = shops
timeSeriesPlot = False
fromDate = None
toDate = None
if targets["m"] != "":
target_masseurs = filter(lambda x: x[1] == targets["m"].decode("utf-8"), target_masseurs)
for i in target_masseurs:
worklogs = filter(lambda x: x[1] == i[0], worklogs)
if targets["h"] != "":
target_helpers = filter(lambda x: x[1] == targets["h"].decode("utf-8"), target_helpers)
for i in target_helpers:
worklogs = filter(lambda x: x[2] == i[0], worklogs)
if targets["s"] != "":
target_shops = filter(lambda x: x[1] == targets["s"].decode("utf-8"), target_shops)
for i in target_shops:
worklogs = filter(lambda x: x[3] == i[0], worklogs)
if targets["p"] != None:
if chartMode == "line":
timeSeriesPlot = True
fromDate = targets["p"]["from"].date()
toDate = targets["p"]["to"].date()
worklogs = filter(lambda x: x[7] >= fromDate and x[7] <= toDate, worklogs)
# between condition
# dataList = [
# (TARGET_NAME,[ WORKLOGS ])
# ]
dataList = []
if between == "masseur":
for masseur in target_masseurs:
mid = masseur[0]
logs = filter(lambda x: x[1] == mid ,worklogs)
if len(logs) == 0:
continue;
else:
dataList.append((masseur[1],logs))
elif between == "helper":
for helper in target_helpers:
hid = helper[0]
logs = filter(lambda x: x[2] == hid ,worklogs)
if len(logs) == 0:
continue;
else:
dataList.append((helper[1],logs))
elif between == "shop":
for shop in target_shops:
sid = shop[0]
logs = filter(lambda x: x[3] == sid ,worklogs)
if len(logs) == 0:
continue;
else:
dataList.append((shop[1],logs))
else:
# for time period
# dataList = [
# (TIME_PERIOD,[ WORKLOGS ])
# ]
fromDate = between["from"].date()
toDate = between["to"].date()
if between["step"] == "week":
step = timedelta(days=7)
to = fromDate + step
while to <= toDate:
dateRange = fromDate.strftime("%Y-%m-%d")+'至'+(to-timedelta(days=1)).strftime("%Y-%m-%d")
logs = filter(lambda x: x[7] >= fromDate and x[7] < to, worklogs)
if len(logs) == 0:
fromDate = to
to = to + step
continue;
else:
dataList.append((dateRange,logs))
fromDate = to
to = to + step
elif between["step"] == "month":
while fromDate.month <= toDate.month:
month = fromDate.strftime("%Y年%m月")
logs = filter(lambda x: x[7].month == fromDate.month,worklogs)
fromDate = add_months(fromDate,1)
if len(logs) == 0:
continue;
else:
dataList.append((month,logs))
elif between["step"] == "season":
while fromDate.month <= toDate.month:
to = add_months(fromDate,2)
monthRange = fromDate.strftime("%Y年%m月") +'至'+to.strftime("%Y年%m月")
logs = filter(lambda x: x[7].month >= fromDate.month and x[7].month < to.month, worklogs)
fromDate = add_months(fromDate,3)
if len(logs) == 0:
continue;
else:
dataList.append((monthRange,logs))
# compare result and create plot
data = []
layout = None
if timeSeriesPlot:
# line chart
daynum = fromDate - toDate
x = [toDate + timedelta(days=num) for num in range(daynum.days, 1)]
for name, logs in dataList:
y = None
if by == "sum":
for compare in compares:
y = [(date,reduce(apply_lambda(compares), grp, 0)) for date, grp in itertools.groupby(logs, key=lambda x: x[7])]
elif by == "count":
y = [(date,len(list(grp))) for date, grp in itertools.groupby(logs, key=lambda x: x[7])]
elif by == "average":
y = [(date, list(grp))for date, grp in itertools.groupby(logs, key=lambda x: x[7])]
y = [(date,reduce(apply_lambda(compares), grp, 0)/float(len(grp))) if len(grp) != 0 else (date, 0) for date, grp in y]
for d in x:
if not d in [date for date,value in y]:
y.append((d, 0))
y.sort(key=lambda a: a[0], reverse=False)
trace = Scatter(
x = x,
y = [v for d,v in y],
name = name
)
data.append(trace)
layout = dict(
xaxis = dict(
tickformat = "%Y年%m月%d日"
),
yaxis=dict(
title='+'.join(map(lambda x: EnglishToChinese[x], compares)),
titlefont=dict(
family='Arial, sans-serif',
size=14,
color='black'
),
)
)
fig = Figure(data=data, layout=layout)
plot_html, plotdivid, width, height = _plot_html(fig, False, "", True, '100%', 525, False)
print plot_html
else:
# tmp = [(TARGET_NAME,{COMPARE:[VALUES]})]
tmp = []
for name, logs in dataList:
values = {}
for compare in compares:
if by == "sum":
values[compare] = reduce(apply_lambda([compare]),logs , 0)
elif by == "count":
values[compare] = len(logs)
elif by == "average":
if len(logs) != 0:
values[compare] = reduce(apply_lambda([compare]),logs , 0)/float(len(logs))
tmp.append((name, values))
# Sort List by SORTBY
if sortBy != None:
if sortBy in compares:
tmp = sorted(tmp, key=lambda a: a[1][sortBy], reverse=True)
elif sortBy == "assigned_not_assigned":
tmp = sorted(tmp, key= lambda a: a[1]["assigned"]+a[1]["not_assigned"], reverse=True)
if jsonfile:
jsonArray = []
fieldname = ""
if between == "masseur" or between == "helper":
fieldname = "人名"
elif between == "店名":
fieldname = "店名"
else:
fieldname = "時間"
for item in tmp:
obj = {}
obj[fieldname] = item[0]
for compare in compares:
obj[compare] = item[1][compare]
jsonArray.append(obj)
print json.dumps(jsonArray,sort_keys=True, indent=4, separators=(',', ': '))
else:
x = map(lambda x: x[0], tmp)
y = {}
for compare in compares:
y[compare] = map(lambda a: a[1][compare], tmp)
if chartMode == "pie":
if pieMode == "sum" and"assigned" in compares and "not_assigned" in compares:
length = len(y["assigned"])
values = []
for i in range(0, length):
values.append(y["assigned"][i]+y["not_assigned"][i])
data.append({
"labels":x,
"values":values,
"type":"pie",
"name":EnglishToChinese["assigned_not_assigned"],
"hoverinfo":"label+percent+name"
})
layout = {}
elif pieMode == "split":
iterator = 0
for compare in compares:
iterator += 1
xLeft = float(1)/len(compares) * (iterator-1)
xRight = float(1)/len(compares) * iterator
data.append({
"labels":x,
"values":y[compare],
"type":"pie",
"name":EnglishToChinese[compare],
"domain": {'x': [xLeft, xRight],
'y': [0,0.8]},
"hoverinfo":"label+percent+name"
})
layout = {}
elif chartMode == "bar":
for compare in compares:
trace = Bar(
x = x,
y = y[compare],
name = EnglishToChinese[compare]
)
data.append(trace)
layout = Layout(
barmode= barMode
)
fig = Figure(data=data, layout=layout)
plot_html, plotdivid, width, height = _plot_html(fig, False, "", True, '100%', 525, False)
# plotly.offline.plot(fig)
print plot_html
cur.close()
conn.close()
def get(self, request):
params=dict()
template='champs.html'
if request.method == 'GET':
if 'cid' in request.GET:
cid=int(request.GET['cid'])
events=MEvent.objects.select_related('p1','p2').filter(champ__id=cid).order_by('-dt')
params['champ']=MChamp.objects.get(id=cid)
params['events']=events
template='events.html'
if 'evid' in request.GET and 'sn' in request.GET:
import plotly
from plotly.offline.offline import _plot_html
from plotly.graph_objs import Scatter, Layout
setsCount=1
meid=int(request.GET['evid'])
sn=int(request.GET['sn'])
gid=0
if 'gid' in request.GET:
gid=int(request.GET['gid'])
currentgame=LSGame.objects.prefetch_related('lspoint_set').get(id=gid)
params['gid']=gid
mevent=MEvent.objects.select_related('p1','p2','champ').get(id=meid)
params['mevent']=mevent
setsCount=len(mevent.res.split(' '))
params['champ']=mevent.champ
params['player1']=mevent.p1
params['player2']=mevent.p2
#
# PlayerSetStats section
#
pss1=PlayerSetStats.objects.filter(meid=mevent.meid,pid=mevent.p1.mpid).order_by('sc')
pss2=PlayerSetStats.objects.filter(meid=mevent.meid,pid=mevent.p2.mpid).order_by('sc')
if (len(pss1)>0 and len(pss1)>0):
pssstr1='[%s]' % ','.join([str(x) for x in pss1])
pssstr2='[%s]' % ','.join([str(x) for x in pss2])
params['pss']='<script type="text/javascript">var pss = [%s,%s];StatInit();</script>' % (pssstr1,pssstr2)
else:
params['pss']=''
lsevent=LSEvent.objects.prefetch_related('lsgame_set').filter(meid=mevent.meid)
isls=len(lsevent)>0
bfevent=BFEvent.objects.prefetch_related('bfodds_set').filter(meid=mevent.meid)
isbf=len(bfevent)>0
if sn==0: # OP data for Not In-Play only
opevent=OPEvent.objects.prefetch_related('opodds_set').filter(meid=mevent.meid)
isop=len(opevent)>0
if isop:
opev=opevent[0]
if isbf:
try:
lastbf=BFOdds.objects.filter(eid=bfevent[0],ip=0).latest('dtc').dtc
opchanges=OPOdds.objects.filter(ev=opev, dtc__lte=lastbf).order_by('dtc')
except:
opchanges=OPOdds.objects.filter(ev=opev).order_by('dtc')
xrop=[i.dtc_tz for i in opchanges]
y1rop=[i.w1 for i in opchanges] if opev.reversed==0 else [i.w2 for i in opchanges]
y2rop=[i.w2 for i in opchanges] if opev.reversed==0 else [i.w1 for i in opchanges]
y1ropm=[i.w1max for i in opchanges] if opev.reversed==0 else [i.w2max for i in opchanges]
y2ropm=[i.w2max for i in opchanges] if opev.reversed==0 else [i.w1max for i in opchanges]
ts=None
te=None
gameshapes=[]
if isls:
linestylegames=dict(color='rgb(128, 0, 128)',width=3,dash='dot')
linestylepoints=dict(color='rgb(128, 128, 128)',width=0.75)
lsev=lsevent[0]
params['lsevent']=lsev
shapespoint=[]
annos=[]
annospoint=[]
games=[]
if gid>0:
isfirstgame=currentgame.sc1==0 and currentgame.sc2==0 and currentgame.setn==1
islastgame=LSGame.objects.filter(eid=lsev, dtc__gt=currentgame.dtc).count()==1
currentpoints=LSPoint.objects.filter(gid=currentgame).order_by('dtc')
if not isfirstgame:
prevgame=LSGame.objects.filter(eid=lsev, dtc__lt=currentgame.dtc).latest('dtc')
currentgamestart=currentgame.dtc
nextgame=LSGame.objects.filter(eid=lsev, dtc__gt=currentgame.dtc).order_by('dtc')[0]
currentgameend=LSPoint.objects.filter(gid=nextgame).order_by('dtc')[0].dtc_tz
pts=LSPoint.objects.filter(gid=prevgame).order_by('-dtc')[1].dtc if not isfirstgame else currentgamestart-timedelta(seconds=60)
pte=LSPoint.objects.filter(gid=nextgame).order_by('dtc')[0].dtc if islastgame else LSPoint.objects.filter(gid=nextgame).order_by('dtc')[1].dtc
shapespoint.append(dict(type='rect',yref='paper',x0=timezone.make_naive(pts),y0=0,x1=currentgame.dtc_tz,y1=1,fillcolor='#ffccbc', opacity=0.3,line={'width': 0}))
shapespoint.append(dict(type='rect',yref='paper',x0=currentgameend,y0=0,x1=timezone.make_naive(pte),y1=1,fillcolor='#ffccbc', opacity=0.3,line={'width': 0}))
if lsev.reversed==0:
prevgametxt='Not IP' if isfirstgame else '%s*-%s'%(prevgame.sc1,prevgame.sc2) if prevgame.serve==1 else '%s-%s*'%(prevgame.sc1,prevgame.sc2)
currentgametxt='%s*-%s'%(currentgame.sc1,currentgame.sc2) if currentgame.serve==1 else '%s-%s*'%(currentgame.sc1,currentgame.sc2)
nextgametxt='' if islastgame else '%s*-%s'%(nextgame.sc1,nextgame.sc2) if nextgame.serve==1 else '%s-%s*'%(nextgame.sc1,nextgame.sc2)
else:
prevgametxt='Not IP' if isfirstgame else '%s-%s*'%(prevgame.sc2,prevgame.sc1) if prevgame.serve==1 else '%s*-%s'%(prevgame.sc2,prevgame.sc1)
currentgametxt='%s-%s*'%(currentgame.sc2,currentgame.sc1) if currentgame.serve==1 else '%s*-%s'%(currentgame.sc2,currentgame.sc1)
nextgametxt='' if islastgame else '%s-%s*'%(nextgame.sc2,nextgame.sc1) if nextgame.serve==1 else '%s*-%s'%(nextgame.sc2,nextgame.sc1)
annospoint.append(dict(x=currentgame.dtc_tz-timedelta(seconds=10),y=1,xref='x',yref='paper', showarrow=False,text=prevgametxt, textangle=-90, font=dict(family='verdana', size=14, color='#1f77b4'),opacity=0.7 ))
annospoint.append(dict(x=currentgameend+timedelta(seconds=10),y=1,xref='x',yref='paper', showarrow=False,text=nextgametxt, textangle=-90, font=dict(family='verdana', size=14, color='#1f77b4'),opacity=0.7 ))
for point in currentpoints:
shapespoint.append(dict(type='line',yref='paper',x0=point.dtc_tz,y0=0,x1=point.dtc_tz,y1=1,line=linestylepoints))
annospoint.append(dict(x=point.dtc_tz+timedelta(seconds=3),y=1,xref='x',yref='paper', showarrow=False,text=self.getPointText(point.sc1, point.sc2,lsev.reversed), textangle=-90, font=dict(family='verdana', size=11, color='#1f77b4'),opacity=0.8 ))
shapespoint.append(dict(type='line',yref='paper',x0=currentgame.dtc_tz,y0=0,x1=currentgame.dtc_tz,y1=1,line=linestylegames))
shapespoint.append(dict(type='line',yref='paper',x0=currentgameend,y0=0,x1=currentgameend,y1=1,line=linestylegames))
gamelay=Layout(title=currentgametxt,shapes = shapespoint, annotations=annospoint,yaxis=dict(type='log', autorange=True),)
if(sn==0): # not IP
tmp=LSGame.objects.filter(eid=lsev,setn=1).order_by('dtc')
if len(tmp)==0:
ts=lsev.dtc if not isbf else BFOdds.objects.filter(eid=bfevent[0],ip=0).latest('dtc').dtc
te=ts
else:
ts=tmp[0].dtc
te=tmp[1].dtc
gameshapes.append(dict(type='line',yref='paper',x0=timezone.make_naive(ts),y0=0,x1=timezone.make_naive(ts),y1=1,line=linestylegames))
gameshapes.append(dict(type='rect',yref='paper',x0=timezone.make_naive(ts),y0=0,x1=timezone.make_naive(te),y1=1,fillcolor='#ffccbc', opacity=0.3,line={'width': 0}))
if isbf:
bfchanges=BFOdds.objects.filter(eid=bfevent[0], dtc__lte=te,b1odds__gt=0,b2odds__gt=0,l1odds__gt=0,l2odds__gt=0).order_by('dtc')
elif(sn<setsCount): # First or Middle set
#gs=LSGame.objects.filter(eid=lsev,setn=sn-1).order_by('-dtc')[0].dtc
curgame=LSGame.objects.filter(eid=lsev,setn=sn).order_by('dtc')[0]
curgames=LSGame.objects.filter(eid=lsev,setn=sn).order_by('dtc')[0].dtc
gameshapes.append(dict(type='line',yref='paper',x0=timezone.make_naive(curgames),y0=0,x1=timezone.make_naive(curgames),y1=1,line=linestylegames))
ts=curgames-timedelta(seconds=900) if sn==1 else curgames
#if sn==1:
gameshapes.append(dict(type='rect',yref='paper',x0=timezone.make_naive(ts),y0=0,x1=timezone.make_naive(curgames),y1=1,fillcolor='#ffccbc', opacity=0.3,line={'width': 0}))
#else:
# gameshapes.append(dict(type='rect',yref='paper',x0=ts,y0=0,x1=curgames,y1=1,fillcolor='#ffccbc', opacity=0.3,line={'width': 0}))
te=LSGame.objects.filter(eid=lsev,setn=sn+1).order_by('dtc')[1].dtc
ge=LSGame.objects.filter(eid=lsev,setn=sn+1).order_by('dtc')[0].dtc
games=LSGame.objects.prefetch_related('lspoint_set').filter(eid=lsev, setn__gt=0, dtc__gte=ts, dtc__lte=te).order_by('dtc')
if gid>0:
ts=pts
te=pte
if isbf:
bfchanges=BFOdds.objects.filter(eid=bfevent[0], dtc__gte=ts, dtc__lte=te,b1odds__gt=0,b2odds__gt=0,l1odds__gt=0,l2odds__gt=0).order_by('dtc')
gameshapes.append(dict(type='line',yref='paper',x0=timezone.make_naive(ge),y0=0,x1=timezone.make_naive(ge),y1=1,line=linestylegames))
gameshapes.append(dict(type='rect',yref='paper',x0=timezone.make_naive(ge),y0=0,x1=timezone.make_naive(te),y1=1,fillcolor='#ffccbc', opacity=0.3,line={'width': 0}))
else: # Last set
ts=LSGame.objects.filter(eid=lsev,setn=sn-1).order_by('-dtc')[0].dtc
endset=LSGame.objects.filter(eid=lsev,setn=6)
games=LSGame.objects.prefetch_related('lspoint_set').filter(eid=lsev, dtc__gte=ts, setn__lt=6).order_by('dtc')
te=games[1].dtc
if gid>0:
ts=pts
te=endset[0].dtc if islastgame else pte
else:
gameshapes.append(dict(type='line',yref='paper',x0=timezone.make_naive(te),y0=0,x1=timezone.make_naive(te),y1=1,line=linestylegames))
gameshapes.append(dict(type='rect',yref='paper',x0=timezone.make_naive(ts),y0=0,x1=timezone.make_naive(te),y1=1,fillcolor='#ffccbc', opacity=0.3,line={'width': 0}))
try:
te=endset[0].dtc
except:
te=BFOdds.objects.filter(eid=bfevent[0]).latest('dtc').dtc
if isbf:
bfchanges=BFOdds.objects.filter(eid=bfevent[0], dtc__gte=ts, dtc__lte=te,b1odds__gt=0,b2odds__gt=0,l1odds__gt=0,l2odds__gt=0).order_by('dtc') if len(endset)>0 or gid>0 else BFOdds.objects.filter(eid=bfevent[0], dtc__gte=ts,b1odds__gt=0,b2odds__gt=0,l1odds__gt=0,l2odds__gt=0).order_by('dtc')
params['games']=[]
if len(games)>0:
c=0
for g in games:
#points=LSPoint.objects.filter(gid=g).order_by('dtc')
params['games'].append(g)
if gid==0:
if lsev.reversed==0:
gametxt='%s*-%s'%(g.sc1,g.sc2) if g.serve==1 else '%s-%s*'%(g.sc1,g.sc2)
else:
gametxt='%s-%s*'%(g.sc2,g.sc1) if g.serve==1 else '%s*-%s'%(g.sc2,g.sc1)
if not (sn<setsCount and c==len(games)-1):
annos.append(dict(x=games[c].dtc_tz+timedelta(seconds=30),y=1,xref='x',yref='paper', showarrow=False,text=gametxt, textangle=-90, font=dict(family='verdana', size=12, color='#1f77b4'),opacity=0.8 ))
if g.serve==1 and c<len(games)-2 and c>0:
gameshapes.append(dict(type='rect',yref='paper',x0=g.dtc_tz,y0=0,x1=games[c+1].dtc_tz,y1=1,fillcolor='#d3d3d3', opacity=0.3,line={'width': 0}))
c+=1
if (sn==0):
title='Not In-Play'
else:
title='Set #%s' % sn
lay=Layout(title=title,shapes = gameshapes, annotations=annos,yaxis=dict(type='log', autorange=True),)
if isbf:
bfev=bfevent[0]
params['tscheduled']=bfev.dt
if mevent.res!='':
params['tcomplete']=bfev.dtc
if (ts==None and te==None):
bfchanges=BFOdds.objects.filter(eid=bfev,ip=0 if sn==0 else 1 ).order_by('dtc')
params=self.getSetButtons(params,sn,setsCount,isls)
try:
goip=BFOdds.objects.filter(eid=bfev,ip=False).latest('dtc').dtc_tz
except:
goip=''
params['goip']=goip
params['bfchanges']=bfchanges
plotly.tools.set_credentials_file(username='******', api_key='mYs3EJORl98E0vxkWvGr')
xr=[i.dtc_tz for i in bfchanges]
if bfev.reversed==0:
y1r=[i.b1odds for i in bfchanges]
y1rl=[i.l1odds for i in bfchanges]
y2r=[i.b2odds for i in bfchanges]
y2rl=[i.l2odds for i in bfchanges]
p1=bfev.pid1.name
p2=bfev.pid2.name
else:
y2r=[i.b1odds for i in bfchanges]
y2rl=[i.l1odds for i in bfchanges]
y1r=[i.b2odds for i in bfchanges]
y1rl=[i.l2odds for i in bfchanges]
p2=bfev.pid1.name
p1=bfev.pid2.name
dataseries=[]
dataseries.append(Scatter(x=xr, y=y1r,mode = 'lines',name = p1+' (Back)', line=dict( color='rgb(40, 53, 147)', )))
dataseries.append(Scatter(x=xr, y=y1rl,mode = 'lines',name = p1+' (Lay)', line=dict( color='rgb(121, 134, 203)', ),visible="legendonly"))
if sn==0 and isop:
dataseries.append(Scatter(x=xrop, y=y1rop,mode = 'lines',name = p1+' (Avg)', line=dict( color='rgb(46, 125, 50)', )))
dataseries.append(Scatter(x=xrop, y=y1ropm,mode = 'lines',name = p1+' (Max)', line=dict( color='rgb(142, 36, 170)', ),visible="legendonly"))
dataseries.append(Scatter(x=xr, y=y2r,mode = 'lines',name = p2+' (Back)', line=dict( color='rgb(216, 67, 21)', )))
dataseries.append(Scatter(x=xr, y=y2rl,mode = 'lines',name = p2+' (Lay)', line=dict( color='rgb(255, 138, 101)', ),visible="legendonly"))
if sn==0 and isop:
dataseries.append(Scatter(x=xrop, y=y2rop,mode = 'lines',name = p2+' (Avg)', line=dict( color='rgb(249, 168, 37)', )))
dataseries.append(Scatter(x=xrop, y=y2ropm,mode = 'lines',name = p2+' (Max)', line=dict( color='rgb(233, 30, 99)', ),visible="legendonly"))
if gid>0:
figure_or_data = { "data": dataseries, "layout":gamelay }
elif isls and sn>0:
figure_or_data = { "layout": lay, "data": dataseries }
else:
figure_or_data = { "data": dataseries, "layout":Layout(title='Non-IP',shapes = gameshapes,yaxis=dict(type='log', autorange=True),) }
config = {}
config['showLink'] = False
#config['layout'] = layout
plot_html = plot_html, plotdivid, width, height = _plot_html(
figure_or_data,config, False, '100%', 600, False)
resize_script = ''
if width == '100%' or height == '100%':
resize_script = (
''
'<script type="text/javascript">'
'window.removeEventListener("resize");'
'window.addEventListener("resize", function(){{'
'Plotly.Plots.resize(document.getElementById("{id}"));}});'
'</script>'
).format(id=plotdivid)
params['graph'] = ''.join([
plot_html,
resize_script])
template='selected.html'
else:
params['cid']=''
atp=MChamp.objects.filter(gender=1).order_by('name')
wta=MChamp.objects.filter(gender=0).order_by('name')
params['atp']=atp
params['wta']=wta
params['currentpage']='Tennis'
return render(request,template,params)
def process_data():
# Database details:
connection = MongoClient(MONGODB_HOST, MONGODB_PORT)
collection = connection[DBS_NAME][COLLECTION_NAME]
projects = collection.find(projection=fields)
# Count the number of times a something is seen:
def count_stuff(listofitems):
listofitems_counted = Counter()
for d in listofitems:
t = d.split(',')[0]
listofitems_counted[d] += 1
return(listofitems_counted)
# Get data from db and save results to a list
#############################
timeanddomain = [] # Hold the list of domains for counting
for project in projects:
dateandtime = project['time']
domain = project['domain']
line = ("{0},{1}").format(dateandtime, domain)
timeanddomain.append(line)
# Save the time of domain hits
all_times = [] # Hold the times
for item in timeanddomain:
#print item
time = item.split(',')[0]
all_times.append(time)
# Separate IP addresses from Domains
#############################
temp1 = [] # Hold the domains that are not IPV4
unique_domains = [] # Hold the domains that are not IPV4 and IPV6. domain_count(domain) will use this to count the domains
ip = [] # Hold the IP addresses (IPV4 and IPV6)
empty_items = 0
for item in timeanddomain:
domain = item.split(',')[1]
if valid_ipv4(domain) == True:
ip.append(domain)
if valid_ipv6(domain) == True:
ip.append(domain)
# This next one takes all non-ipv4 items (domains) and adds them to the temp1 list:
if valid_ipv4(domain) == False:
temp1.append(domain)
# This one reads the temp1 list that contains both domains and ipv6 addresses and takes only the non-ipv6 entries and adds them to the unique_domains list:
for domain in temp1:
if valid_ipv6(domain) == False:
unique_domains.append(domain)
count_of_domains = count_stuff(unique_domains) # Count the domains and save as count_of_domains
count_of_times = count_stuff(all_times) # Count the domains and save as count_of_domains
# Turn the count_of_domains into a dictionary
# Used to set a threshold and view domains contacted over or under a certain number
domainslist = []
temp = []
for key, value in count_of_domains.iteritems():
temp = [key,value]
domainslist.append(temp)
timeslist = []
temp = []
for key, value in count_of_times.iteritems():
temp = [key,value]
timeslist.append(temp)
#############################
# Determine normal in a loose manner:
#############################
normal_traffic = []
suspicious_traffic = []
# Print domains with a certain number of visits
for item in domainslist:
domain = item[0]
count = item[1]
# if count is greater than or equal to a number:
if count >= 2:
#print("{0}, {1}".format(domain,count))
normal_traffic.append(item)
# if count is equal to a number:
if count < 5:
#print("{0}, {1}".format(domain,count))
suspicious_traffic.append(item)
# if count is less than or equal to a number:
# if count <= 10:
# print("{0}, {1}".format(domain,count))
######################################################
# Take the suspicious domains and unique them, getting them ready to look at using third party tools:
domain_counts = [] # firstlevel.tld, count
domain_fullrequest_counts = [] # actual request, firstlevel.tld, count
temp = [] # Temporary holder to unique the firstlevel.tld, as used below
for line in suspicious_traffic:
fulldomain = line[0]
dom = tldextract.extract(fulldomain)
if dom.suffix != '':
domain = dom.domain + '.' + dom.suffix # Just the first level domain
# Create a newline, which has the firstlevel domain, the full domain with subdomains and the count
domain_fullrequest_count = "{0},{1},{2}".format(domain,fulldomain,line[1])
if domain in temp:
continue
else:
domain_count = "{0},{1}".format(domain,line[1])
domain_counts.append(domain_count)
domain_fullrequest_counts.append(domain_fullrequest_count)
temp.append(domain) # Unique the domains
#################################
# Check the suspicious traffic in
# OpenDNS Investigate:
token = ()
with open('investigate_token.txt') as API_KEY:
token = API_KEY.read()
token = token.rstrip()
inv = investigate.Investigate(token)
categories_list = []
security_categories_list = []
wl_domains = []
bl_domains = []
not_determined_domains = []
for line in domain_fullrequest_counts:
line = line.split(',')
domain = line[0]
domain_fullrequest_count = "{0},{1},{2}".format(line[0],line[1],line[2])
res = inv.categorization(domain, labels=True)
status = res[domain]['status']
if status == 0:
# Get domain categorization and add it to categories_list
content_category = res[domain]['content_categories']
if content_category == []:
continue
else:
for value in content_category:
categories_list.append(value)
##############
not_determined_domains.append(domain_fullrequest_count)
if status == 1:
# Get domain categorization and add it to categories_list
content_category = res[domain]['content_categories']
if content_category == []:
continue
else:
for value in content_category:
categories_list.append(value)
##############
wl_domains.append(domain_fullrequest_count)
#print domain_fullrequest_count
if status == -1:
bl_domains.append(domain_fullrequest_count)
security_category = res[domain]['security_categories']
if security_category == []:
continue
else:
for value in security_category:
security_categories_list.append(value)
categories_list.append(value)
# Get domain categorization and add it to categories_list
content_category = res[domain]['content_categories']
if content_category == []:
continue
else:
for value in content_category:
categories_list.append(value)
##############
#domain_and_cat = str(domain) + ":" + str(security_category)
#print (domain, security_category)
count_of_categories = count_stuff(categories_list) # Count the categories and save as count_of_categories
# Turn the count_of_categories into a dictionary
category_list = []
temp = []
for key, value in count_of_categories.iteritems():
temp = [key,value]
category_list.append(temp)
count_of_security_categories = count_stuff(security_categories_list) # Count the categories and save as count_of_security_categories
# Turn the count_of_security_categories into a dictionary
security_category_list = []
temp = []
for key, value in count_of_security_categories.iteritems():
temp = [key,value]
security_category_list.append(temp)
#print security_categories_list
#####################################
# CHART GENERATION
#####################################
# Time visit Barchart:
xtimedata = []
ytimedata = []
for item in timeanddomain:
item = item.split(',')
xtimedata.append(item[0])
ytimedata.append(item[1])
'''
timebarchart = discreteBarChart(name='discreteBarChart', height=600, width=1000)
timebarchart.add_serie(y=ytimedata, x=xtimedata)
timebarchart.buildhtml()
timebarchart = discreteBarChart(name='discreteBarChart', height=600, width=1000)
timebarchart.add_serie(y=df['date'], x=df['counts'])
#timebarchart.add_serie(y=df.index.values, x=df['domain'])
timebarchart.buildhtml()
writefile = open('../flask/templates/timebar.html','w')
writefile.write(timebarchart.htmlcontent)'''
#####################################
#alldomains Barchart:
xdomaindata = []
ydomaindata = []
for item in domainslist:
xdomaindata.append(item[0])
ydomaindata.append(item[1])
### Plotly does this better. All domains visit Barchart:
'''alldomains = discreteBarChart(name='discreteBarChart', height=300, width=1000)
alldomains.add_serie(y=ydomaindata, x=xdomaindata)
alldomains.buildhtml()
writefile = open('../flask/templates/alldomains.html','w')
writefile.write(alldomains.htmlcontent)'''
#####################################
# For a chart of suspicious domains:
xsuspicious = []
ysuspicious = []
for item in domain_counts:
item = item.split(',')
xsuspicious.append(item[0]) # Domains
ysuspicious.append(item[1]) # Count
### Plotly does this better. suspicious visit Barchart:
'''suspiciousbarchart = discreteBarChart(name='discreteBarChart', height=600, width=1000)
suspiciousbarchart.add_serie(y=ysuspicious, x=xsuspicious)
suspiciousbarchart.buildhtml()
writefile = open('../flask/templates/suspicious_domains.html','w')
writefile.write(suspiciousbarchart.htmlcontent)'''
#####################################
# For a chart of blacklisted domains:
xblacklisted = []
yblacklisted = []
for item in bl_domains:
item = item.split(',')
xblacklisted.append(item[0]) # Domains
yblacklisted.append(item[2]) # Count
### Plotly does this better. Blacklisted Barchart:
'''blacklistedbarchart = discreteBarChart(name='discreteBarChart', height=300, width=600)
blacklistedbarchart.add_serie(y=yblacklisted, x=xblacklisted)
blacklistedbarchart.buildhtml()
writefile = open('../flask/templates/blacklisted_domains.html','w')
writefile.write(blacklistedbarchart.htmlcontent)'''
#####################################
# For a chart of whitelisted domains:
xwhitelisted = []
ywhitelisted = []
for item in wl_domains:
item = item.split(',')
xwhitelisted.append(item[0]) # Domains
ywhitelisted.append(item[2]) # Count
### Plotly does this better. whitelisted Barchart:
'''whitelistedbarchart = discreteBarChart(name='discreteBarChart', height=300, width=600)
whitelistedbarchart.add_serie(y=ywhitelisted, x=xwhitelisted)
whitelistedbarchart.buildhtml()
writefile = open('../flask/templates/whitelisted_domains.html','w')
writefile.write(whitelistedbarchart.htmlcontent)'''
#####################################
# For a Pie Chart of categories:
xcategorylist = []
ycategorylist = []
for item in category_list:
xcategorylist.append(item[0]) # Categories
ycategorylist.append(item[1]) # Count
### Blacklisted Barchart:
type = 'pieChart'
categorypiechart = pieChart(name=type, color_category='category20c', height=1000, width=1000)
xdata = xcategorylist
ydata = ycategorylist
extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}}
categorypiechart.add_serie(y=ydata, x=xdata, extra=extra_serie)
categorypiechart.buildhtml()
writefile = open('../flask/templates/category_piechart.html','w')
writefile.write(categorypiechart.htmlcontent)
df_categories = pd.DataFrame(data=ycategorylist,index=xcategorylist)
#####################################
# For a Pie Chart of security categories:
xseccategorylist = []
yseccategorylist = []
for item in security_category_list:
xseccategorylist.append(item[0]) # Categories
yseccategorylist.append(item[1]) # Count
### security category piehart:
type = 'pieChart'
security_categorypiechart = pieChart(name=type, color_category='category20c', height=450, width=450)
xdata = xseccategorylist
ydata = yseccategorylist
extra_serie = {"tooltip": {"y_start": "", "y_end": ""}}
security_categorypiechart.add_serie(y=ydata, x=xdata, extra=extra_serie)
security_categorypiechart.buildhtml()
writefile = open('../flask/templates/security_category_piechart.html','w')
writefile.write(security_categorypiechart.htmlcontent)
#####################################
# For a chart of neutral listed domains:
xneutrallisted = []
yneutrallisted = []
for item in not_determined_domains:
item = item.split(',')
xneutrallisted.append(item[0]) # Domains
yneutrallisted.append(item[2]) # Count
### Plotly does this better. neutral listed Barchart:
'''neutrallistedbarchart = discreteBarChart(name='discreteBarChart', height=300, width=600)
neutrallistedbarchart.add_serie(y=yneutrallisted, x=xneutrallisted)
neutrallistedbarchart.buildhtml()
writefile = open('../flask/templates/neutral_domains.html','w')
writefile.write(neutrallistedbarchart.htmlcontent)'''
#################################
stats_number_of_u_domains = str(len(unique_domains)) + " unique domains seen<br>"
stats_number_of_t_domains = str(len(timeanddomain)) + " total domains<br>"
stats_number_of_ips = str(len(ip)) + " total IP addresses<br>"
stats_normaltraffic = ('Normal traffic (domains visited over 3 times): {0}<br>').format(len(normal_traffic))
stats_suspicioustraffic = ('Amount of suspicious traffic domains visited under 2 times): {0}<br>').format(len(suspicious_traffic))
stats_wl = ('<br>Of the suspicious domains (uniqued):<br>Whitelisted domains (OpenDNS): {0}<br>'.format(len(wl_domains)))
stats_bl = ('Blacklisted domains (OpenDNS): {0}<br>'.format(len(bl_domains)))
stats_neutral = ('Neutral domains (OpenDNS): {0}'.format(len(not_determined_domains)))
stats_categories_list = ('Number of categories seen: {0}'.format(len(categories_list)))
stats = stats_number_of_u_domains + stats_number_of_t_domains + stats_number_of_ips + stats_normaltraffic + stats_suspicioustraffic + stats_wl + stats_bl + stats_neutral
writefile = open('../flask/templates/stats.html','w')
writefile.write(stats)
# This one's for plotly:
xtime = []
ytime = []
for item in timeanddomain:
item = item.split(',')
xtime.append(item[0])
ytime.append(item[1])
d = {'date':xtime,'domain':ytime}
df = pd.DataFrame(data=d)
df['counts'] = df.groupby('domain').transform('count')
# For plotly:
df2 = pd.DataFrame(data=ytime,index=xtime)
df2.index = pd.to_datetime(df2.index)
#############################
# HTML for plotly plots:
#############################
html_start = """<html>
<head>
<script src="../static/plotly-latest.min.js"></script>
</head>
<body>"""
html_end = """
</body>
</html>"""
# Line plot:
'''plot_html, plotdivid, width, height = _plot_html(df2.iplot(asFigure=True, kind ='scatter', subplots=True, shared_xaxes=True, fill=True, title='Count by day',dimensions=(800,800)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('../flask/templates/plottest_scatter.html', 'w')
f.write(html_bar_chart)
f.close()'''
# Suspicious domains
df_suspicious = pd.DataFrame(ysuspicious, xsuspicious)
plot_html, plotdivid, width, height = _plot_html(df_suspicious.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Suspicious Traffic',dimensions=(600,600)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('../flask/templates/suspicious_traffic.html', 'w')
f.write(html_bar_chart)
f.close()
# All domains visited
df_alldomains = pd.DataFrame(ydomaindata, xdomaindata)
plot_html, plotdivid, width, height = _plot_html(df_alldomains.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='All Traffic',dimensions=(600,300)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('../flask/templates/all_traffic.html', 'w')
f.write(html_bar_chart)
f.close()
# Whitelisted
df_whitelisted = pd.DataFrame(ywhitelisted, xwhitelisted)
plot_html, plotdivid, width, height = _plot_html(df_whitelisted.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Whitelisted Traffic',dimensions=(600,300)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('../flask/templates/whitelisted_traffic.html', 'w')
f.write(html_bar_chart)
f.close()
# Not categorized
df_not_categorized = pd.DataFrame(yneutrallisted, xneutrallisted)
plot_html, plotdivid, width, height = _plot_html(df_not_categorized.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Non-categorized Traffic',dimensions=(600,300)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('../flask/templates/not_categorized_traffic.html', 'w')
f.write(html_bar_chart)
f.close()
# Blacklisted
df_blacklisted = pd.DataFrame(yblacklisted, xblacklisted)
plot_html, plotdivid, width, height = _plot_html(df_blacklisted.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Blacklisted Traffic',dimensions=(600,300)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('../flask/templates/blacklisted_traffic.html', 'w')
f.write(html_bar_chart)
f.close()
# Time Series
df_timeseries = pd.DataFrame(ytimedata,xtimedata)
plot_html, plotdivid, width, height = _plot_html(df_timeseries.iplot(asFigure=True, kind ='bar', subplots=False, shared_xaxes=True, fill=True, title='Time Series',dimensions=(800,450)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('../flask/templates/timeseries.html', 'w')
f.write(html_bar_chart)
f.close()
def _kanban_dashboard_graph(self):
import plotly.graph_objs as go
from plotly import tools
from plotly.graph_objs import Scatter
from plotly.offline.offline import _plot_html
import pandas as pd
import pandas.io.sql as psql
self.env.cr.execute("select sum(amount_total) as total, rp.name from \
account_invoice ac inner join res_partner rp on \
ac.partner_id = rp.id group by rp.name \
order by total desc")
todos = self.env.cr.fetchall()
df = pd.DataFrame(todos, columns=['total', 'name'])
df["total"] = df["total"].astype(float)
primeiros = df.iloc[:12]
outros = df.iloc[12:]
total = outros['total'].sum()
df2 = pd.DataFrame([[total, 'Outros']], columns=['total', 'name'])
primeiros = primeiros.append(df2)
pie = go.Pie(labels=primeiros['name'], values=primeiros['total'])
prod_df = df
import numpy as np
total = prod_df['total'].sum()
a = prod_df['total'] / total
prod_df['percentual'] = a
prod_df['cum_percentual'] = prod_df['percentual'].cumsum(0)
prod_df['ScoreA'] = prod_df['cum_percentual'] < 0.8
prod_df['A'] = np.where(prod_df['ScoreA'], 'A', None)
prod_df['ScoreB'] = prod_df['cum_percentual'].between(0.8, 0.95)
prod_df['B'] = np.where(prod_df['ScoreB'], 'A', None)
prod_df['ScoreC'] = prod_df['cum_percentual'] > 0.95
prod_df['C'] = np.where(prod_df['ScoreC'], 'C', None)
total = float(prod_df['name'].count())
prod_df['percent'] = (prod_df.index.values / total)
prod_df['label_x'] = prod_df[['name', 'total', 'percentual']].apply(
lambda x: u'{} - R$ {} - {:,.2%}'.format(x[0], x[1], x[2]), axis=1)
# Curva ABC de clientes
a_df = prod_df[prod_df['ScoreA'] == True]
trace1 = go.Scatter(
x=a_df['percent'],
y=a_df['cum_percentual'],
text=a_df['label_x'],
mode='lines+markers',
name='Clientes A',
)
b_df = prod_df[prod_df['ScoreB']]
trace2 = go.Scatter(
x=b_df['percent'],
y=b_df['cum_percentual'],
text=b_df['label_x'],
mode='lines+markers',
name='Clientes B',
)
c_df = prod_df[prod_df['ScoreC']]
trace3 = go.Scatter(
x=c_df['percent'],
y=c_df['cum_percentual'],
text=c_df['label_x'],
mode='lines+markers',
name='Clientes C',
)
layout = go.Layout(showlegend=True,
title="Curva ABC de Clientes",
yaxis={
'tickformat': '%',
'ticksuffix': ' de vendas',
'showticksuffix': 'first'
},
xaxis={'tickformat': '%'})
fig = go.Figure(data=[trace1, trace2, trace3], layout=layout)
plot_html, plotdivid, width, height = _plot_html(
fig, False, "", True, '100%', 525, False)
self.dashboard_graph = plot_html
def plot_ex(figure_or_data,
show_link=True,
link_text='Export to plot.ly',
validate=True,
resizable=False,
lock_aspect_ratio=False,
master=True,
click_to_display=False,
link_to=None,
link_to_id=False,
rel_figure_dir="figures"):
"""
Create a pyGSTi plotly graph locally, returning HTML & JS separately.
Parameters
----------
figure_or_data : plotly.graph_objs.Figure or .Data or dict or list
object that describes a Plotly graph. See https://plot.ly/python/
for examples of graph descriptions.
show_link : bool, optional
display a link in the bottom-right corner of
link_text : str, optional
the text of export link
validate : bool, optional
validate that all of the keys in the figure are valid? omit if you
need to include extra, unnecessary keys in your figure.
resizable : bool, optional
Make the plot resizable by including a "resize" event handler and
any additional initialization.
lock_aspect_ratio : bool, optional
Whether the aspect ratio of the plot should be allowed to change
when it is sized based on it's container.
master : bool, optional
Whether this plot represents the "master" of a group of plots,
all of the others which are "slaves". The relative sizing of the
master of a group will determine the relative sizing of the slaves,
rather than the slave's containing element. Useful for preserving the
size of the features in a group of plots that may be different overall
sizes.
click_to_display : bool, optional
Whether the plot should be rendered immediately or whether a "click"
icon should be shown instead, which must be clicked on to render the
plot.
link_to : None or tuple of {"pdf", "pkl"}
If not-None, the types of pre-rendered/computed versions of this plot
that can be assumed to be present, and therefore linked to by additional
items in the hover-over menu of the plotly plot.
link_to_id : str, optional
The base name (without extension) of the ".pdf" or ".pkl" files that are
to be linked to by menu items. For example, if `link_to` equals
`("pdf",)` and `link_to_id` equals "plot1234", then a menu item linking
to the file "plot1234.pdf" will be added to the renderd plot.
Returns
-------
dict
With 'html' and 'js' keys separately specifying the HTML and javascript
needed to embed the plot.
"""
from plotly import __version__ as _plotly_version
from plotly import tools as _plotlytools
#Processing to enable automatic-resizing & aspect ratio locking
fig = _plotlytools.return_figure_from_figure_or_data(figure_or_data, False)
orig_width = fig.get('layout', {}).get('width', None)
orig_height = fig.get('layout', {}).get('height', None)
if lock_aspect_ratio and orig_width and orig_height:
aspect_ratio = orig_width / orig_height
else:
aspect_ratio = None
#Remove original dimensions of plot so default of 100% is used below
# (and triggers resize-script creation)
if orig_width: del fig['layout']['width']
if orig_height: del fig['layout']['height']
#Special polar plot case - see below - add dummy width & height so
# we can find/replace them with variables in generated javascript.
if 'angularaxis' in fig['layout']:
fig['layout']['width'] = 123
fig['layout']['height'] = 123
config = {}
config['showLink'] = show_link
config['linkText'] = link_text
#Add version-dependent kwargs to _plot_html call below
plotly_version = tuple(map(int, _plotly_version.split('.')))
if plotly_version < (3, 8, 0): # "old" plotly with _plot_html function
from plotly.offline.offline import _plot_html
kwargs = {}
if plotly_version >= (3, 7, 0): # then auto_play arg exists
kwargs['auto_play'] = False
#Note: removing width and height from layout above causes default values to
# be used (the '100%'s hardcoded below) which subsequently trigger adding a resize script.
plot_html, plotdivid, _, _ = _plot_html(
fig,
config,
validate,
'100%',
'100%',
global_requirejs=False, # no need for global_requirejs here
**kwargs) # since we now extract js and remake full script.
else:
from plotly.io import to_html as _to_html
import uuid as _uuid
plot_html = _to_html(fig,
config,
auto_play=False,
include_plotlyjs=False,
include_mathjax=False,
post_script=None,
full_html=False,
animation_opts=None,
validate=validate)
assert (plot_html.startswith("<div>") and plot_html.endswith("</div>"))
plot_html = plot_html[len("<div>"):-len("</div>")].strip()
assert (plot_html.endswith("</script>"))
id_index = plot_html.find('id="')
id_index_end = plot_html.find('"', id_index + len('id="'))
plotdivid = _uuid.UUID(plot_html[id_index + len('id="'):id_index_end])
if orig_width: fig['layout']['width'] = orig_width
if orig_height: fig['layout']['height'] = orig_height
# Separate the HTML and JS in plot_html so we can insert
# initial-sizing JS between them. NOTE: this is FRAGILE and depends
# on Plotly output (_plot_html) being HTML followed by JS
tag = '<script type="text/javascript">'
end_tag = '</script>'
iTag = plot_html.index(tag)
plot_js = plot_html[iTag + len(tag):-len(end_tag)].strip()
plot_html = plot_html[0:iTag].strip()
full_script = ''
#Note: in this case, upper logic (usually in an on-ready hander of the table/plot
# group creation) is responsible for triggering a "create" event on the plot div
# when appropriate (see workspace.py).
#Get javascript for create and (possibly) resize handlers
plotly_create_js = plot_js # the ususal plotly creation javascript
plotly_resize_js = None
if resizable:
#the ususal plotly resize javascript
plotly_resize_js = ' Plotly.Plots.resize(document.getElementById("{id}"));'.format(
id=plotdivid)
if 'angularaxis' in fig['layout']:
#Special case of polar plots: Plotly does *not* allow resizing of polar plots.
# (I don't know why, and it's not documented, but in plotly.js there are explict conditions
# in Plotly.relayout that short-circuit when "gd.frameworks.isPolar" is true). So,
# we just re-create the plot with a different size to mimic resizing.
plot_js = plot_js.replace('"width": 123', '"width": pw').replace(
'"height": 123', '"height": ph')
plotly_resize_js = ('var plotlydiv = $("#{id}");\n'
'plotlydiv.children(".plotly").remove();\n'
'var pw = plotlydiv.width();\n'
'var ph = plotlydiv.height();\n'
'{resized}\n').format(id=plotdivid,
resized=plot_js)
plotly_create_js = plotly_resize_js
aspect_val = aspect_ratio if aspect_ratio else "null"
groupclass = "pygsti-plotgroup-master" \
if master else "pygsti-plotgroup-slave"
if link_to and ('pdf' in link_to) and link_to_id:
link_to_pdf_js = (
"\n"
" btn = $('#{id}').find('.modebar-btn[data-title=\"Show closest data on hover\"]');\n"
" btn = cloneAndReplace( btn ); //Strips all event handlers\n"
" btn.attr('data-title','Download PDF');\n"
" btn.click( function() {{\n"
" window.open('{relfigdir}/{pdfid}.pdf');\n"
" }});\n").format(id=plotdivid,
pdfid=link_to_id,
relfigdir=rel_figure_dir)
plotly_create_js += link_to_pdf_js
if link_to and ('pkl' in link_to) and link_to_id:
link_to_pkl_js = (
"\n"
" btn = $('#{id}').find('.modebar-btn[data-title=\"Zoom\"]');\n"
" btn = cloneAndReplace( btn ); //Strips all event handlers\n"
" btn.attr('data-title','Download python pickle');\n"
" btn.click( function() {{\n"
" window.open('{relfigdir}/{pklid}.pkl');\n"
" }});\n").format(id=plotdivid,
pklid=link_to_id,
relfigdir=rel_figure_dir)
plotly_create_js += link_to_pkl_js
plotly_click_js = ""
if click_to_display and master:
# move plotly plot creation from "create" to "click" handler
plotly_click_js = plotly_create_js
plotly_create_js = ""
full_script = ( # (assume this will all be run within an on-ready handler)
' $("#{id}").addClass("{groupclass}");\n' # perform this right away
' $("#{id}").on("init", function(event) {{\n' # always add init-size handler
' pex_init_plotdiv($("#{id}"), {ow}, {oh});\n'
' pex_init_slaves($("#{id}"));\n'
' console.log("Initialized {id}");\n'
' }});\n'
' $("#{id}").on("click.pygsti", function(event) {{\n'
' plotman.enqueue(function() {{ \n'
' {plotlyClickJS} \n'
' }}, "Click-creating Plot {id}" );\n'
' $("#{id}").off("click.pygsti");\n' # remove this event handler
' console.log("Click-Created {id}");\n'
' }});\n'
' $("#{id}").on("create", function(event, fracw, frach) {{\n' # always add create handler
' pex_update_plotdiv_size($("#{id}"), {ratio}, fracw, frach, {ow}, {oh});\n'
' plotman.enqueue(function() {{ \n'
' $("#{id}").addClass("pygBackground");\n'
' {plotlyCreateJS} \n'
' pex_create_slaves($("#{id}"), {ow}, {oh});\n'
' }}, "Creating Plot {id}" );\n'
' console.log("Created {id}");\n'
' }});\n').format(id=plotdivid,
ratio=aspect_val,
groupclass=groupclass,
ow=orig_width if orig_width else "null",
oh=orig_height if orig_height else "null",
plotlyClickJS=plotly_click_js,
plotlyCreateJS=plotly_create_js)
#Add resize handler if needed
if resizable:
full_script += (
' $("#{id}").on("resize", function(event,fracw,frach) {{\n'
' pex_update_plotdiv_size($("#{id}"), {ratio}, fracw, frach, {ow}, {oh});\n'
' plotman.enqueue(function() {{ \n'
' {plotlyResizeJS} \n'
' pex_resize_slaves($("#{id}"), {ow}, {oh});\n'
' }}, "Resizing Plot {id}" );\n'
' //console.log("Resized {id}");\n'
' }});\n').format(id=plotdivid,
ratio=aspect_val,
ow=orig_width if orig_width else "null",
oh=orig_height if orig_height else "null",
plotlyResizeJS=plotly_resize_js)
return {'html': plot_html, 'js': full_script}
def get(self, request):
params = dict()
template = 'op_all.html'
if request.method == 'GET':
if 'cid' in request.GET:
cid = int(request.GET['cid'])
champ = OPChamp.objects.get(id=cid)
events = OPEvent.objects.select_related(
'p1', 'p2').filter(champ__id=cid)
params['champ'] = champ
params['events'] = events
template = 'op_champ.html'
if 'evid' in request.GET:
import plotly
from plotly.offline.offline import _plot_html
from plotly.graph_objs import Scatter
evid = int(request.GET['evid'])
event = OPEvent.objects.select_related('p1', 'p2',
'champ').get(id=evid)
params['event'] = event
oddschanges = OPOdds.objects.filter(ev=event).order_by('dtc')
params['oddschanges'] = oddschanges
plotly.tools.set_credentials_file(
username='******', api_key='mYs3EJORl98E0vxkWvGr')
xr = [i.dtc for i in oddschanges]
y1r = [i.w1 for i in oddschanges]
y2r = [i.w2 for i in oddschanges]
y1mr = [i.w1max for i in oddschanges]
y2mr = [i.w2max for i in oddschanges]
figure_or_data = [
Scatter(x=xr,
y=y1r,
mode='lines',
name='Avg ' + event.p1.name),
Scatter(x=xr,
y=y2r,
mode='lines',
name='Avg ' + event.p2.name),
Scatter(x=xr,
y=y1mr,
mode='lines',
name='Max ' + event.p1.name),
Scatter(x=xr,
y=y2mr,
mode='lines',
name='Max ' + event.p2.name),
]
config = {}
config['showLink'] = False
plot_html = plot_html, plotdivid, width, height = _plot_html(
figure_or_data, config, True, '100%', 500, False)
resize_script = ''
if width == '100%' or height == '100%':
resize_script = (
''
'<script type="text/javascript">'
'window.removeEventListener("resize");'
'window.addEventListener("resize", function(){{'
'Plotly.Plots.resize(document.getElementById("{id}"));}});'
'</script>').format(id=plotdivid)
params['graph'] = ''.join([plot_html, resize_script])
template = 'op_event.html'
else:
champs = OPChamp.objects.filter(
opevent__isnull=False).distinct()
params['champs'] = champs
params['currentpage'] = 'OP'
return render(request, template, params)
def ethio_ts(df=pd.DataFrame, scale_id=int, year=int):
cols = [
'Testes positivos',
'Influenza A',
'Influenza B',
'VSR',
'Adenovirus',
'Parainfluenza 1',
'Parainfluenza 2',
'Parainfluenza 3',
]
trace = []
if scale_id == 2:
ytitle = 'Casos'
else:
ytitle = 'Incidência'
trace.append(
go.Scatter(
x=df['epiweek'],
y=df[cols[0]],
name=cols[0],
mode='lines',
)
)
for i, col in enumerate(cols[1:]):
trace.append(
go.Scatter(
x=df['epiweek'],
y=df[col],
name=ytitle,
mode='lines',
showlegend=False
)
)
nrows = len(cols)
fig = tools.make_subplots(
rows=nrows,
cols=1,
print_grid=False,
subplot_titles=('Situação dos exames',
'Influenza A',
'Influenza B',
'VSR',
'Adenovirus',
'Parainfluenza 1',
'Parainfluenza 2',
'Parainfluenza 3'))
for i in range(1, (nrows + 1)):
fig.append_trace(trace[i - 1], i, 1)
# Add extra lines to first subplot:
extra_cols = ['Testes negativos',
'Casos sem teste laboratorial',
'Casos aguardando resultado',
'Casos sem informação laboratorial']
for i, col in enumerate(extra_cols):
fig.append_trace(
go.Scatter(
x=df['epiweek'],
y=df[col],
name=col,
mode='lines',
line=dict(color=cl.scales['12']['qual']['Paired'][-i])
), 1, 1)
# X-axis title and range:
lastepiweek = int(episem(lastepiday(year), out='W'))
extra_cols.extend(['Testes positivos'])
if scale_id == 2:
ymax = [
max(5*np.ceil(df[extra_cols].max().max()/5), 5),
max(5*np.ceil(df[cols[1:]].max().max()/5), 5)
]
else:
ymax = [
df[extra_cols].max().max(),
df[cols[1:]].max().max()
]
for i in range(1, (nrows + 1)):
xaxis = 'xaxis%s' % i
yaxis = 'yaxis%s' % i
fig['layout'][xaxis].update(range=[1, lastepiweek])
fig['layout'][yaxis].update(range=[0, ymax[min(1, i-1)]], rangemode='nonnegative')
fig['layout']['xaxis%s' % nrows].update(title='Semana epidemiológica',
zeroline=True, showline=True)
i = int(nrows/2)
fig['layout']['yaxis%s' % i].update(title=ytitle)
fig['layout'].update(margin={'t': 50, 'l': 50})
return _plot_html(
figure_or_data=fig, config={}, validate=True,
default_width='100%', default_height=1600, global_requirejs=''
)[0]
def get(self, request):
params = dict()
template = 'bf_all.html'
if request.method == 'GET':
if 'cid' in request.GET:
cid = int(request.GET['cid'])
events = BFEvent.objects.select_related(
'pid1', 'pid2').filter(cid__id=cid)
params['champ'] = events[0].cid.name if len(events) > 0 else ''
params['events'] = events
template = 'bf_champ.html'
if 'evid' in request.GET and 'isip' in request.GET:
import plotly
from plotly.offline.offline import _plot_html
from plotly.graph_objs import Scatter
evid = int(request.GET['evid'])
isip = request.GET['isip'] == '1'
event = BFEvent.objects.get(id=evid)
params['event'] = event
oddschanges = BFOdds.objects.filter(eid=event,
ip=isip).order_by('dtc')
params['isip0'] = '' if isip else ' disabled'
params['isip1'] = ' disabled' if isip else ''
try:
goip = BFOdds.objects.filter(eid=event,
ip=False).latest('dtc').dtc
except:
goip = ''
params['goip'] = goip
params['oddschanges'] = oddschanges
plotly.tools.set_credentials_file(
username='******', api_key='mYs3EJORl98E0vxkWvGr')
xr = [i.dtc for i in oddschanges]
y1r = [i.b1odds for i in oddschanges]
y2r = [i.b2odds for i in oddschanges]
figure_or_data = [
Scatter(x=xr, y=y1r, mode='lines', name=event.pid1.name),
Scatter(x=xr, y=y2r, mode='lines', name=event.pid2.name)
]
config = {}
config['showLink'] = False
plot_html = plot_html, plotdivid, width, height = _plot_html(
figure_or_data, config, True, '100%', 500, False)
resize_script = ''
if width == '100%' or height == '100%':
resize_script = (
''
'<script type="text/javascript">'
'window.removeEventListener("resize");'
'window.addEventListener("resize", function(){{'
'Plotly.Plots.resize(document.getElementById("{id}"));}});'
'</script>').format(id=plotdivid)
params['graph'] = ''.join([plot_html, resize_script])
template = 'bf_event.html'
else:
params['cid'] = ''
champs = BFChamp.objects.filter(
bfevent__isnull=False).distinct()
params['champs'] = champs
params['currentpage'] = 'BF'
return render(request, template, params)
def plotly_hack(fig):
from plotly.offline.offline import _plot_html
plot_html, plotdivid, width, height = _plot_html(fig, False, "", True,
'100%', 525, False)
return plot_html
def process_data():
# Database details:
connection = MongoClient(MONGODB_HOST, MONGODB_PORT)
collection = connection[DBS_NAME][COLLECTION_NAME]
projects = collection.find(projection=fields)
#projects = collection.find()
# Get data from db and save results to a list
#############################
gets = [] # Hold the get requests
posts = [] # Hold the posts
gets_and_posts = []
gets_full_entry = []
gets_with_a_request = []
posts_full_entry = []
total_entries = []
gets_and_posts_no_head = []
alltimes = []
for data in projects:
time = data['time']
useragent = data['useragent']
request_method = data['request_method']
request = data['request']
src_ip = data['src_ip']
dest_ip = data['dest_ip']
dest_host = data['dest_host']
if request_method == 'GET':
if len(request) > 2: # Dirty way to negate the requests with no data
line = "{0},{1},{2},{3},{4},{5},{6}".format(time,src_ip,useragent,dest_ip,dest_host,request_method,request)
gets_with_a_request.append(line)
if len(request) > 2: # Dirty way to negate the requests with no data
line = "{0},{1},{2},{3},{4},{5},{6}".format(time,src_ip,useragent,dest_ip,dest_host,request_method,request)
else:
line = "{0},{1},{2},{3},{4},{5},No request".format(time,src_ip,useragent,dest_ip,dest_host,request_method)
alltimes.append(time)
total_entries.append(line)
# Used for counting the GETs and POSTs, for making the pie chart
if request_method == 'GET':
gets_and_posts.append('GET Requests')
gets_and_posts_no_head.append(line)
if request_method == 'POST':
gets_and_posts.append('POSTs')
gets_and_posts_no_head.append(line)
###################################
# Get all the GETs (full line) and save to gets_full_entry
if request_method == 'GET':
gets.append(request_method)
gets_full_entry.append(line)
# Get all the POSTs (full line) and save to posts_full_entry
if request_method == 'POST':
posts.append(request_method)
posts_full_entry.append(line)
html_start = """<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<link href="https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.css" rel="stylesheet" />
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.js"></script>
<script src="../../static/plotly-latest.min.js"></script>
</head>
<body>"""
html_end = """
</body>
</html>"""
###################################
# For a Pie Chart of GETS and POSTS:
count_request_method = count_stuff(gets_and_posts)
count_of_request_methods = []
temp = []
for key, value in count_request_method.iteritems():
temp = [key,value]
count_of_request_methods.append(temp)
xcategorylist = []
ycategorylist = []
for item in count_of_request_methods:
xcategorylist.append(item[0])
ycategorylist.append(item[1])
type = 'pieChart'
categorypiechart = pieChart(name=type, color_category='category20c', height=400, width=400)
extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}}
categorypiechart.add_serie(y=ycategorylist, x=xcategorylist, extra=extra_serie)
categorypiechart.buildcontent()
writefile = open('../flask/templates/pcap/pcap_category_piechart.html','w')
writefile.write(html_start + categorypiechart.htmlcontent)
writefile = open('../flask/templates/pcap/pcap_category_piechart.html','a') # Re-open for appending
header = "<b>Time, Source IP, Destination Host, Destination IP, Request Method, Request</b><br>"
writefile.write(header)
#for line in gets_with_a_request:
for line in total_entries:
line = line.split(',')
time, src_ip, dest_host, dest_ip, method, request = line[0],line[1],line[4],line[3],line[5],line[6]
line = ("{0}, {1}, {2}, {3}, {4}<br>".format(time, src_ip, dest_host, dest_ip, method, request))
writefile.write(line)
for line in posts_full_entry:
line = line.split(',')
time, src_ip, dest_host, dest_ip, method, request = line[0],line[1],line[4],line[3],line[5],line[6]
line = ("{0}, {1}, {2}, {3}, {4}, {5}<br>".format(time, src_ip, dest_host, dest_ip, method, request))
writefile.write(line)
writefile.write(html_end)
# Timeseries of GETs vs POSTs with plotly
xtime = []
ytime = []
for item in gets_and_posts_no_head:
item = item.split(',')
xtime.append(item[0])
ytime.append(item[5])
# Time Series
df_timeseries = pd.DataFrame(ytime,xtime)
plot_html, plotdivid, width, height = _plot_html(df_timeseries.iplot(asFigure=True, kind ='bar', subplots=False, shared_xaxes=True, fill=True, title='Time Series of HTTP Requests',dimensions=(800,450)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('../flask/templates/pcap/pcap_timeseries.html', 'w')
f.write(html_bar_chart)
f.close()
# Gets and Posts on a timeseries:
xtime = []
yrequest = []
posts_to_dst_ips = []
gets_to_dst_ips = []
total_dst_ips = []
total_src_ips = []
total_dst_hosts = []
for item in total_entries:
item = item.split(',')
time = item[0]
src_ip = item[2]
dst_ip = item[3]
dst_host = item[4]
request = item[5]
if src_ip not in total_src_ips:
total_src_ips.append(src_ip)
if dst_ip not in total_dst_ips:
total_dst_ips.append(dst_ip)
if dst_host not in total_dst_hosts:
total_dst_hosts.append(dst_host)
# Separate the dst_ip's based on GETS and POSTS (for a map)
if request == 'POST':
posts_to_dst_ips.append(dst_ip)
if request == 'GET':
gets_to_dst_ips.append(dst_ip)
# Get the count of POSTS for a timeseries
if request == 'POST':
xtime.append(time)
yrequest.append(1)
else:
xtime.append(time)
yrequest.append(0)
# Create maps of the POSTS vs GETS:
quickmap.ip_map_world('../../static/miller-2048x1502-color.jpg',posts_to_dst_ips,'../flask/templates/pcap/posts_to_dst_ips.svg')
quickmap.ip_map_world('../../static/miller-2048x1502-color.jpg',gets_to_dst_ips,'../flask/templates/pcap/gets_to_dst_ips.svg')
df_timeseries = pd.DataFrame(yrequest,xtime)
plot_html, plotdivid, width, height = _plot_html(df_timeseries.iplot(asFigure=True, kind ='bar', subplots=False, shared_xaxes=True, fill=True, title='All Requests by time, showing the POSTs',dimensions=(800,450)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('../flask/templates/pcap/getsandposts_timeseries.html', 'w')
f.write(html_bar_chart)
f.close()
# TO DO: Check out domains with investigate
#######################
# STATS:
number_of_gets = str(len(gets)) + " GET requests seen<br>"
number_of_posts = str(len(posts)) + " POSTS seen<br>"
stats_number_of_src_ips = str(len(total_src_ips)) + " total Source IP addresses<br>"
stats_number_of_dst_ips = str(len(total_dst_ips)) + " total Destination IP addresses<br>"
stats_number_of_dst_hosts = str(len(total_dst_hosts)) + " total Destination Hosts<br>"
stats = number_of_gets + number_of_posts + stats_number_of_src_ips + stats_number_of_dst_ips + stats_number_of_dst_hosts
writefile = open('../flask/templates/pcap/stats.html','w')
writefile.write(stats)
def create_tweet_chart(cls, df: pd.DataFrame, year_week,
disease: str) -> 'Plotly_HTML':
"""
:param df:
:param year_week:
:param disease:
:return:
"""
ks_cases = [
'casos notif. {}'.format(disease),
]
df_tweet = df.reset_index()[['SE', 'tweets'] + ks_cases]
df_tweet = df_tweet[df_tweet.SE >= year_week - 200]
df_tweet.rename(columns={'tweets': 'menções'}, inplace=True)
df_grp = df_tweet.groupby(df.index)[['menções'] +
ks_cases].sum().reset_index()
df_grp['SE'] = df_grp.SE.map(lambda v: '%s/%s' %
(str(v)[:4], str(v)[-2:]))
fig_tweet = df_grp.iplot(x=['SE'],
y=['menções'],
asFigure=True,
showlegend=True,
xTitle='Período (Ano/Semana)',
color=['rgb(128,128,128)'])
fig_cases = df_grp.iplot(x=['SE'],
y=ks_cases,
asFigure=True,
secondary_y=ks_cases,
secondary_y_title='Casos',
showlegend=True,
xTitle='Período (Ano/Semana)',
color=['rgb(0,0,255)'])
fig_cases.data.extend(fig_tweet.data)
fig_cases['layout']['xaxis1'].update(tickangle=-60,
nticks=len(df_grp) // 24)
fig_cases['layout']['yaxis1'].update(
title='Tweets', range=[0, df_grp['menções'].max()])
fig_cases['layout']['yaxis2'].update(
range=[0, df_grp[ks_cases].max().max()])
fig_cases['layout'].update(
title='Casos {} / Menções mídia social'.format(disease))
fig_cases['layout']['legend'].update(x=-.1,
y=1.2,
traceorder='normal',
font=dict(family='sans-serif',
size=12,
color='#000'),
bgcolor='#FFFFFF',
bordercolor='#E2E2E2',
borderwidth=1)
return _plot_html(figure_or_data=fig_cases,
config={},
validate=True,
default_width='100%',
default_height=300,
global_requirejs='')[0]
def process_data():
pcapdata = []
for ts, buf in pcap:
eth = dpkt.ethernet.Ethernet(buf)
ip = eth.data
tcp = ip.data
dtstring=datetime.datetime.fromtimestamp(ts).strftime('%d-%m-%Y %H:%M:%S:%f') # Convert time to string
#dt=datetime.datetime.strptime(dtstring,'%d-%m-%Y %H:%M:%S:%f') # Convert string time to datetime
try:
if len(tcp.data) > 0:
http = dpkt.http.Request(tcp.data)
headers = http.headers
dest_ip = socket.inet_ntoa(ip.dst)
src_ip = socket.inet_ntoa(ip.src)
dest_host = headers['host']
uri = http.uri
useragent = headers['user-agent']
request = repr(http['body'])
#data = (dtstring, http.method)
data = ("{0},{1},{2},{3},{4},{5},{6}".format(dtstring, src_ip, useragent, dest_host, dest_ip, http.method, request))
pcapdata.append(data)
except:
continue
# Get data from db and save results to a list
#############################
gets = [] # Hold the get requests
posts = [] # Hold the posts
gets_and_posts = []
gets_full_entry = []
gets_with_a_request = []
posts_full_entry = []
total_entries = []
gets_and_posts_no_head = []
alltimes = []
for data in pcapdata:
data = data.split(',')
time = data[0]
useragent = data[2]
request_method = data[5]
request = data[6]
src_ip = data[1]
dest_ip = data[4]
dest_host = data[3]
if request_method == 'GET':
if len(request) > 2: # Dirty way to negate the requests with no data
line = "{0},{1},{2},{3},{4},{5},{6}".format(time,src_ip,useragent,dest_ip,dest_host,request_method,request)
gets_with_a_request.append(line)
if len(request) > 2: # Dirty way to negate the requests with no data
line = "{0},{1},{2},{3},{4},{5},{6}".format(time,src_ip,useragent,dest_ip,dest_host,request_method,request)
else:
line = "{0},{1},{2},{3},{4},{5},No request".format(time,src_ip,useragent,dest_ip,dest_host,request_method)
alltimes.append(time)
total_entries.append(line)
# Used for counting the GETs and POSTs, for making the pie chart
if request_method == 'GET':
gets_and_posts.append('GET Requests')
gets_and_posts_no_head.append(line)
if request_method == 'POST':
gets_and_posts.append('POSTs')
gets_and_posts_no_head.append(line)
###################################
# Get all the GETs (full line) and save to gets_full_entry
if request_method == 'GET':
gets.append(request_method)
gets_full_entry.append(line)
# Get all the POSTs (full line) and save to posts_full_entry
if request_method == 'POST':
posts.append(request_method)
posts_full_entry.append(line)
html_start = """<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<link href="../../flask/static/bower_components/nvd3/build/nv.d3.min.css" rel="stylesheet" />
<script src="../../flask/static/bower_components/d3/d3.min.js"></script>
<script src="../../flask/static/bower_components/nvd3/build/nv.d3.min.js"></script>
<script src="../../flask/static/plotly-latest.min.js"></script>
</head>
<body>"""
html_end = """
</body>
</html>"""
###################################
# For a Pie Chart of GETS and POSTS:
count_request_method = count_stuff(gets_and_posts)
count_of_request_methods = []
temp = []
for key, value in count_request_method.iteritems():
temp = [key,value]
count_of_request_methods.append(temp)
xcategorylist = []
ycategorylist = []
for item in count_of_request_methods:
xcategorylist.append(item[0])
ycategorylist.append(item[1])
type = 'pieChart'
categorypiechart = pieChart(name=type, color_category='category20c', height=400, width=400)
extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}}
categorypiechart.add_serie(y=ycategorylist, x=xcategorylist, extra=extra_serie)
categorypiechart.buildcontent()
writefile = open('pcapstatic/pcap_category_piechart.html','w')
writefile.write(html_start + categorypiechart.htmlcontent)
writefile = open('pcapstatic/pcap_category_piechart.html','a') # Re-open for appending
header = "<b>Time, Source IP, Destination Host, Destination IP, Request Method, Request</b><br>"
writefile.write(header)
#for line in gets_with_a_request:
for line in total_entries:
line = line.split(',')
time, src_ip, dest_host, dest_ip, method, request = line[0],line[1],line[4],line[3],line[5],line[6]
line = ("{0}, {1}, {2}, {3}, {4}<br>".format(time, src_ip, dest_host, dest_ip, method, request))
writefile.write(line)
for line in posts_full_entry:
line = line.split(',')
time, src_ip, dest_host, dest_ip, method, request = line[0],line[1],line[4],line[3],line[5],line[6]
line = ("{0}, {1}, {2}, {3}, {4}, {5}<br>".format(time, src_ip, dest_host, dest_ip, method, request))
writefile.write(line)
writefile.write(html_end)
# Timeseries of GETs vs POSTs with plotly
xtime = []
ytime = []
for item in gets_and_posts_no_head:
item = item.split(',')
xtime.append(item[0])
ytime.append(item[5])
# Time Series
df_timeseries = pd.DataFrame(ytime,xtime)
plot_html, plotdivid, width, height = _plot_html(df_timeseries.iplot(asFigure=True, kind ='bar', subplots=False, shared_xaxes=True, fill=True, title='Time Series of HTTP Requests',dimensions=(800,450)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('pcapstatic/pcap_timeseries.html', 'w')
f.write(html_bar_chart)
f.close()
# Gets and Posts on a timeseries:
xtime = []
yrequest = []
posts_to_dst_ips = []
gets_to_dst_ips = []
total_dst_ips = []
total_src_ips = []
total_dst_hosts = []
for item in total_entries:
item = item.split(',')
time = item[0]
src_ip = item[2]
dst_ip = item[3]
dst_host = item[4]
request = item[5]
if src_ip not in total_src_ips:
total_src_ips.append(src_ip)
if dst_ip not in total_dst_ips:
total_dst_ips.append(dst_ip)
if dst_host not in total_dst_hosts:
total_dst_hosts.append(dst_host)
# Separate the dst_ip's based on GETS and POSTS (for a map)
if request == 'POST':
posts_to_dst_ips.append(dst_ip)
if request == 'GET':
gets_to_dst_ips.append(dst_ip)
# Get the count of POSTS for a timeseries
if request == 'POST':
xtime.append(time)
yrequest.append(1)
else:
xtime.append(time)
yrequest.append(0)
# Create maps of the POSTS vs GETS:
quickmap.ip_map_world('miller-2048x1502-color.jpg',posts_to_dst_ips,'pcapstatic/posts_to_dst_ips.svg')
quickmap.ip_map_world('miller-2048x1502-color.jpg',gets_to_dst_ips,'pcapstatic/gets_to_dst_ips.svg')
df_timeseries = pd.DataFrame(yrequest,xtime)
plot_html, plotdivid, width, height = _plot_html(df_timeseries.iplot(asFigure=True, kind ='bar', subplots=False, shared_xaxes=True, fill=True, title='All Requests by time, showing the POSTs',dimensions=(800,450)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('pcapstatic/getsandposts_timeseries.html', 'w')
f.write(html_bar_chart)
f.close()
# TO DO: Check out domains with investigate
#######################
# STATS:
number_of_gets = str(len(gets)) + " GET requests seen<br>"
number_of_posts = str(len(posts)) + " POSTS seen<br>"
stats_number_of_src_ips = str(len(total_src_ips)) + " total Source IP addresses<br>"
stats_number_of_dst_ips = str(len(total_dst_ips)) + " total Destination IP addresses<br>"
stats_number_of_dst_hosts = str(len(total_dst_hosts)) + " total Destination Hosts<br>"
stats = number_of_gets + number_of_posts + stats_number_of_src_ips + stats_number_of_dst_ips + stats_number_of_dst_hosts
writefile = open('pcapstatic/stats.html','w')
writefile.write(stats)
def process_data():
# Count the number of times a something is seen:
def count_stuff(listofitems):
listofitems_counted = Counter()
for d in listofitems:
t = d.split(',')[0]
info = d.split(',')[1]
listofitems_counted[t] += 1
return(listofitems_counted)
#####################################
# Process input file:
event_ids = []
task_categories = []
all_items = []
successful_logons = []
successful_logoffs = []
logon_attempts = []
failed_logon_attempts = []
# Open the input file, convert datetime to datetime object and add separate items:
with open(input_file) as f:
for item in f:
rawline = item.strip()
s = rawline.split(',')
audit_result = s[0]
dateandtime = s[1]
dt = datetime.datetime.strptime(dateandtime, "%m/%d/%Y %I:%M:%S %p")
source = s[2]
event_id = s[3]
task_category = s[4]
info = s[5]
task_categories.append(task_category)
event_ids.append(event_id)
line = "{0},{1}".format(dt,info)
all_items.append(line)
if event_id == '4624': # Successful Logons
line = "{0},{1}".format(dt,info)
successful_logons.append(line)
if event_id == '4634': # Successful Logoffs
line = "{0},{1}".format(dt,info)
successful_logoffs.append(line)
if event_id == '4648': # Logon attempt
line = "{0},{1}".format(dt,info)
logon_attempts.append(line)
if event_id == '4625': # Failed Logon attempts
line = "{0},{1}".format(dt,info)
failed_logon_attempts.append(line)
#for item in successful_logons:
# print item
def make_a_list_of_dates_and_counts(listofitems):
# Turn the count_of_successful_logons into a dictionary
counted_items = []
temp = []
for key, value in listofitems.iteritems():
temp = [key,value]
counted_items.append(temp)
return counted_items
_all_items = count_stuff(all_items)
_count_of_successful_logons = count_stuff(successful_logons)
_count_of_successful_logoffs = count_stuff(successful_logoffs)
_count_of_logon_attempts = count_stuff(logon_attempts)
_count_of_failed_logon_attempts = count_stuff(failed_logon_attempts)
all_events = make_a_list_of_dates_and_counts(_all_items)
successful_logons = make_a_list_of_dates_and_counts(_count_of_successful_logons)
successful_logoffs = make_a_list_of_dates_and_counts(_count_of_successful_logoffs)
logon_attempts = make_a_list_of_dates_and_counts(_count_of_logon_attempts)
failed_logon_attempts = make_a_list_of_dates_and_counts(_count_of_failed_logon_attempts)
#############################
# HTML for plotly plots:
#############################
html_start = """<html>
<head>
<script src="../flask/static/plotly-latest.min.js"></script>
</head>
<body>"""
html_end = """
</body>
</html>"""
#print security_categories_list
#####################################
# CHART GENERATION
#####################################
#########################
# Successful Logons
x = []
y = []
for item in successful_logons:
x.append(item[0])
y.append(item[1])
df_successful_logons = pd.DataFrame(y, x)
plot_html, plotdivid, width, height = _plot_html(df_successful_logons.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Successful Logons',dimensions=(600,600)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('successful_logons.html', 'w')
f.write(html_bar_chart)
f.close()
##########################
# successful logoffs
# x = []
# y = []
# for item in successful_logoffs:
# x.append(item[0])
# y.append(item[1])
# df_successful_logoffs = pd.DataFrame(y, x)
# plot_html, plotdivid, width, height = _plot_html(df_successful_logoffs.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Successful Logoffs',dimensions=(600,600)), False, "", True, '100%', 525, False)
# html_bar_chart = html_start + plot_html + html_end
# f = open('successful_logoffs.html', 'w')
# f.write(html_bar_chart)
# f.close()
##########################
# Failed logon attempts
x = []
y = []
for item in failed_logon_attempts:
x.append(item[0])
y.append(item[1])
if len(x) >= 1:
df_failed_logon_attempts = pd.DataFrame(y, x)
plot_html, plotdivid, width, height = _plot_html(df_failed_logon_attempts.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Failed Login Attempts',dimensions=(600,600)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('failed_logon_attempts.html', 'w')
f.write(html_bar_chart)
f.close()
else:
print('No failed login attempts')
##########################
# logon attempts
x = []
y = []
for item in logon_attempts:
x.append(item[0])
y.append(item[1])
df_logon_attempts = pd.DataFrame(y, x)
plot_html, plotdivid, width, height = _plot_html(df_logon_attempts.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Login Attempts',dimensions=(600,600)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('logon_attempts.html', 'w')
f.write(html_bar_chart)
f.close()
##########################
# All Events
x = []
y = []
for item in all_events:
x.append(item[0])
y.append(item[1])
df_all_events = pd.DataFrame(y, x)
plot_html, plotdivid, width, height = _plot_html(df_all_events.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='All Events',dimensions=(600,600)), False, "", True, '100%', 525, False)
html_bar_chart = html_start + plot_html + html_end
f = open('all_events.html', 'w')
f.write(html_bar_chart)
f.close()
def get_context_data(self, *args, **kwargs):
#for now, displays random data, hook this up to experiment data later
context = super(LineChartAnimatedDataView,
self).get_context_data(**kwargs)
xMax = 50
N = 100
yMax = 5
margin = 1.5
x = np.linspace(0, xMax, N) #should be a context argument
y = np.random.rand(N) * yMax #should be a context argument
#display code
data = [
dict(x=x,
y=y,
mode='lines',
line=dict(width=2, color=CHART_LINE_COLORS[0])),
]
frames = [
dict(data=[
dict(x=x[0:k + 1],
y=y[0:k + 1],
mode='lines',
line=dict(color=CHART_LINE_COLORS[0], width=2))
]) for k in range(N)
]
layout = dict(
plot_bgcolor='transparent',
paper_bgcolor='transparent',
showlegend=False,
font=dict(family='"Lato","Open Sans",Helvetica,sans-serif',
color='#ffffff'),
xaxis=dict(range=[np.min(x) - margin,
np.max(x) + margin],
autorange=False,
color='#ddd'),
yaxis=dict(range=[np.min(y) - margin,
np.max(y) + margin],
autorange=False,
color='#ddd'),
title='Random Data Test 1', #should be a context argument
titlefont=dict(family='"Lato","Open Sans",Helvetica,sans-serif',
color='#ffffff'),
hovermode='closest',
updatemenus=[{
'buttons': [{
'args': [[frames[-1]], {
'frame': {
'duration': 0,
'redraw': False
},
'mode': 'immediate',
'transition': {
'duration': 0
}
}],
'label':
'Skip to End',
'method':
'animate'
}],
'font':
dict(family='"Lato","Open Sans",Helvetica,sans-serif',
color='#ffffff'),
'bordercolor':
'#ffffff',
'direction':
'left',
'pad': {
'l': 10
},
'showactive':
False,
'type':
'buttons',
'x':
0.025,
'xanchor':
'left',
'y':
1,
'yanchor':
'top'
}])
figure = dict(data=data, layout=layout, frames=frames)
plot_html = plot_html, plot_div_id, width, height = _plot_html(
figure, {'show_link': False}, True, '100%', '100%', False)
resize_script = ''
if width == '100%' or height == '100%':
resize_script = (
''
'<script type="text/javascript">'
'window.removeEventListener("resize" );'
'window.addEventListener("resize", function(){{'
'Plotly.Plots.resize(document.getElementById("{id}"));}});'
'</script>').format(id=plot_div_id)
context['html_str'] = ''.join([plot_html, resize_script])
return context
def create_incidence_chart(
cls, df: pd.DataFrame, year_week: int, threshold_pre_epidemic: float,
threshold_pos_epidemic: float, threshold_epidemic: float
) -> 'Plotly_HTML':
"""
@see: https://stackoverflow.com/questions/45526734/
hide-legend-entries-in-a-plotly-figure
:param df:
:param year_week:
:param threshold_pre_epidemic: float,
:param threshold_pos_epidemic: float
:param threshold_epidemic: float
:return:
"""
df = df.reset_index()[[
'SE', 'incidência', 'casos notif.', 'level_code'
]]
# 200 = 2 years
df = df[
df.SE >= year_week - 200
]
df['SE'] = df.SE.map(lambda v: '%s/%s' % (str(v)[:4], str(v)[-2:]))
k = 'incidência'
df['alerta verde'] = df[df.level_code == 1][k]
df['alerta amarelo'] = df[df.level_code == 2][k]
df['alerta laranja'] = df[df.level_code == 3][k]
df['alerta vermelho'] = df[df.level_code == 4][k]
df['limiar epidêmico'] = threshold_epidemic
df['limiar pós epidêmico'] = threshold_pos_epidemic
df['limiar pré epidêmico'] = threshold_pre_epidemic
figure_bar = df.iplot(
asFigure=True, kind='bar', x=['SE'], y=[
'alerta verde',
'alerta amarelo',
'alerta laranja',
'alerta vermelho'
],
legend=True,
showlegend=True,
yTitle='Incidência', xTitle='Período (Ano/Semana)',
color=[
'rgb(0,255,0)', 'rgb(255,255,0)',
'rgb(255,150,0)', 'rgb(255,0,0)'
], hoverinfo='x+y+name'
)
figure_threshold = df.iplot(
asFigure=True, x=['SE'], y=[
'limiar pré epidêmico',
'limiar pós epidêmico',
'limiar epidêmico'
],
legend=False,
showlegend=False,
color=[
'rgb(0,255,0)', 'rgb(255,150,0)', 'rgb(255,0,0)'
], hoverinfo='none'
)
figure_line = df.iplot(
asFigure=True, x=['SE'], y=['casos notif.'],
legend=False,
showlegend=False,
secondary_y=['casos notif.'], secondary_y_title='Casos',
hoverinfo='x+y+name', color=['rgb(33,33,33)']
)
figure_line['layout']['legend'].update(
x=-.27, y=0.5,
traceorder='normal',
font=dict(
family='sans-serif',
size=12,
color='#000'
),
bgcolor='#FFFFFF',
bordercolor='#E2E2E2',
borderwidth=1,
orientation='l',
)
figure_line['layout']['yaxis1'].update(
title='Incidência'
)
figure_line['layout'].update(
title=(
'Limiares de incidência:: ' +
'pré epidêmico=%s; ' +
'pós epidêmico=%s; ' +
'epidêmico=%s;'
) % (
'{:.1f}'.format(threshold_pre_epidemic),
'{:.1f}'.format(threshold_pos_epidemic),
'{:.1f}'.format(threshold_epidemic)
), showlegend=True
)
figure_threshold.data.extend(figure_bar.data)
figure_line.data.extend(figure_threshold.data)
figure_line['layout']['yaxis2'].update(
showgrid=False,
range=[0, df['casos notif.'].max()]
)
figure_line['layout']['xaxis1'].update(
tickangle=-60, nticks=len(df) // 4,
title='Período (Ano/Semana)'
)
for trace in figure_line['data']:
if trace['name'] == 'casos notif.':
trace['visible'] = 'legendonly'
return _plot_html(
figure_or_data=figure_line, config={}, validate=True,
default_width='100%', default_height=500, global_requirejs=''
)[0]
