def generate_graphs(): global consumption_list global total_power_consumption global cummulative_consumption_list time_list = [] name_list = [] power_list = [] html = "" count = 0 for item in consumption_list: if count == 0: time_list.append(time(0, 0, 0)) cummulative_consumption_list[count] = item else: cummulative_consumption_list[ count] = item + cummulative_consumption_list[count - 1] time_list.append( (datetime.combine(date(1, 1, 1), time_list[count - 1]) + timedelta(minutes=1)).time()) #print time_list[count] count += 1 plot_data = [go.Scatter(x=time_list, y=consumption_list, name='Watts')] layout = go.Layout(title=("Simulated Total Power Consumption"), xaxis=dict(title="Time"), yaxis=dict(title="Power consumption in Watts")) py_fig = dict(data=plot_data, layout=layout) plot_html, plotdivid, width, height = _plot_html(py_fig, False, "", True, '100%', 525, False) #html = "<div class=\"row\"><h3>Overall Power Consumption</h3>" html += "<div class=\"row\">%s</div>" % plot_html for appliance in appliance_list: name_list.append(appliance.get_name()) power_list.append(appliance.get_total_watts()) plot_data = [go.Bar(x=name_list, y=power_list, name='Power Consumption')] layout = go.Layout(title=("Simulated Appliance Power Consumption"), xaxis=dict(title="Appliance"), yaxis=dict(title="Power consumption in Watts")) py_fig = dict(data=plot_data, layout=layout) plot_html, plotdivid, width, height = _plot_html(py_fig, False, "", True, '100%', 525, False) #html += "<div class=\"row\"><h3>Appliance Consumption</h3>" html += "\n<div class=\"row\">%s</div>" % plot_html plot_data = [ go.Scatter(x=time_list, y=cummulative_consumption_list, name='Watts') ] layout = go.Layout(title=("Cummulative Total Power Consumption"), xaxis=dict(title="Time"), yaxis=dict(title="Power consumption in Watts")) py_fig = dict(data=plot_data, layout=layout) plot_html, plotdivid, width, height = _plot_html(py_fig, False, "", True, '100%', 525, False) html += "\n<div class=\"row\">%s</div>" % plot_html return (html, total_power_consumption, consumption_list)
def report_category(number): category = num2category[number]['name'] with pd.HDFStore(args.report, 'r') as store: plotly_data = [] correct, wrong, count, accuracy, top_level_accuracy = category_stats(number) plotly_data.append(go.Scatter( x=np.linspace(0,100, num=len(correct)), y=correct.score, mode='lines', name='Correct', hoverinfo='name+y', text=[ json.dumps({ 'path': path, 'prediction': category }) for path in correct.index ])) wrong_categories = [ num2category[c]['name'] for c in wrong.category ] plotly_data.append(go.Scatter( x=np.linspace(0,100, num=len(wrong)), y=wrong.score, mode='lines', name='Wrong', hoverinfo='name+y', text=[ json.dumps({ 'path': path, 'prediction': prediction }) for path, prediction in zip(wrong.index, wrong_categories)])) layout = go.Layout(hovermode='closest', title='Performance of correct vs wrong classified pictures', xaxis={'title': '%'}, yaxis={'title': 'score'}) figure = go.Figure(data=plotly_data, layout=layout) performance_plot, performance_id, _,_ = _plot_html(figure, False, '', True, '50%', '100%', False) categories_counter = Counter(wrong_categories) labels, values = zip(*categories_counter.items()) pie = go.Pie(labels=labels, values=values, showlegend=False, textinfo='text', text=[None]*len(values)) layout= go.Layout(hovermode='closest', title='Wrongly classified pictures ({}%) labelled {}'.format(np.round(100-accuracy, 1), category)) figure = go.Figure(data=[pie], layout=layout) pie, _, _, _ = _plot_html(figure, False, '', True, '100%', '100%', False) wrong_as_this = store['{}/wrong/in'.format(number)] wrong_out = sorted(zip(wrong.index, [ num2category[c]['name'] for c in wrong.category], wrong.score), key=lambda x: x[2], reverse=True) wrong_in = sorted(zip(wrong_as_this.index, [ num2category[c]['name'] for c in wrong_as_this.category], wrong_as_this.score), key=lambda x: x[2], reverse=True) return flask.render_template('report.html', accuracy=accuracy, category=category, performance_plot=performance_plot, performance_id=performance_id, pie=pie, wrong_out=wrong_out, wrong_in = wrong_in, count=count, top_level_accuracy=top_level_accuracy)
def _ipython_display_(self): disable_ipython_scrollbar() xaxis, yaxis = self.config['x'], self.config.get('y', None) yaxes = [yaxis] if yaxis is not None else \ [col for col in self.table.columns if col != xaxis] xvals = self.table[xaxis].tolist() traces = [ dict(x=xvals, y=self.table[axis].tolist(), name=axis) for axis in yaxes ] layout = dict( xaxis=dict(title=xaxis), yaxis=dict(title=self.config['table']), legend=dict(x=0.7, y=1), margin=dict(r=0, t=40), ) fig = dict(data=traces, layout=layout) display( HTML( _plot_html(fig, False, '', True, '100%', 525, global_requirejs=True)[0]))
def plot_to_html(self, fig): plotly_html_div, plotdivid, width, height = _plot_html( figure_or_data=fig, config="", validate=True, default_width='90%', default_height="100%", global_requirejs=False) return plotly_html_div
def plot2(mini, maxi, len): # plotly try: data = [{ 'x': list(range(len)), 'y': np.random.randint(mini, maxi, len) }] plot_html, plotdivid, width, height = _plot_html( data, '', False, '80%', 525, False, False) return plot_html except: return '<h1><font color="red">plot failed!</font></h1>'
def create_climate_chart( cls, df: pd.DataFrame, var_climate, year_week, climate_crit, climate_title ) -> 'Plotly_HTML': """ :param df: :param var_climate: :param year_week: :param climate_crit: :param climate_title: :return: """ k = var_climate.replace('_', '.') df_climate = df.reset_index()[['SE', k]] df_climate = df_climate[ df_climate.SE >= year_week - 200 ] df_climate['SE'] = df_climate.SE.map( lambda v: '%s/%s' % (str(v)[:4], str(v)[-2:]) ) df_climate['Limiar favorável transmissão'] = climate_crit figure = df_climate.iplot( asFigure=True, x=['SE'], y=[k, 'Limiar favorável transmissão'], showlegend=True, yTitle=climate_title, xTitle='Período (Ano/Semana)' ) figure['layout']['xaxis1'].update( tickangle=-60, nticks=len(df_climate)//4 ) figure['layout']['legend'].update( x=-.1, y=1.2, traceorder='normal', font=dict( family='sans-serif', size=12, color='#000' ), bgcolor='#FFFFFF', bordercolor='#E2E2E2', borderwidth=1 ) return _plot_html( figure_or_data=figure, config={}, validate=True, default_width='100%', default_height=500, global_requirejs='' )[0]
def create_tweet_chart( cls, df: pd.DataFrame, year_week ) -> 'Plotly_HTML': """ :param df: :param var_climate: :param year_week: :param climate_crit: :param climate_title: :return: """ df_tweet = df.reset_index()[['SE', 'tweets']] df_tweet = df_tweet[ df_tweet.SE >= year_week - 200 ] df_tweet['SE'] = df_tweet.SE.map( lambda v: '%s/%s' % (str(v)[:4], str(v)[-2:]) ) df_tweet.rename(columns={'tweets': 'menções'}, inplace=True) figure = df_tweet.iplot( x=['SE'], y=['menções'], asFigure=True, showlegend=True, xTitle='Período (Ano/Semana)' ) figure['layout']['xaxis1'].update( tickangle=-60, nticks=len(df_tweet)//4 ) figure['layout']['yaxis1'].update(title='Tweets') figure['layout']['legend'].update( x=-.1, y=1.2, traceorder='normal', font=dict( family='sans-serif', size=12, color='#000' ), bgcolor='#FFFFFF', bordercolor='#E2E2E2', borderwidth=1 ) return _plot_html( figure_or_data=figure, config={}, validate=True, default_width='100%', default_height=500, global_requirejs='' )[0]
def create_climate_chart(cls, df: pd.DataFrame, var_climate, year_week, climate_crit, climate_title) -> 'Plotly_HTML': """ :param df: :param var_climate: :param year_week: :param climate_crit: :param climate_title: :return: """ k = var_climate.replace('_', '.') df_climate = df.reset_index()[['SE', k]] df_climate = df_climate[df_climate.SE >= year_week - 200] df_climate['SE'] = df_climate.SE.map(lambda v: '%s/%s' % (str(v)[:4], str(v)[-2:])) df_climate['Limiar favorável transmissão'] = climate_crit figure = df_climate.iplot(asFigure=True, x=['SE'], y=[k, 'Limiar favorável transmissão'], showlegend=True, yTitle=climate_title, xTitle='Período (Ano/Semana)') figure['layout']['xaxis1'].update(tickangle=-60, nticks=len(df_climate) // 4) figure['layout']['legend'].update(x=-.1, y=1.2, traceorder='normal', font=dict(family='sans-serif', size=12, color='#000'), bgcolor='#FFFFFF', bordercolor='#E2E2E2', borderwidth=1) return _plot_html(figure_or_data=figure, config={}, validate=True, default_width='100%', default_height=500, global_requirejs='')[0]
def html(fig): config = dict(showLink=False, displaylogo=False) plot_html, plotdivid, w, h = pl_offline._plot_html( fig, config, validate=True, default_width='100%', default_height='100%', global_requirejs=False) script_split = plot_html.find('<script ') plot_content = { 'div': plot_html[:script_split], 'script': plot_html[script_split:], 'id': plotdivid } return plot_content
def __init__(self, figure, click_callback=None, relayout_callback=None, **kwargs): super().__init__(**kwargs) html, uid, _, _ = _plot_html(figure, False, "", True, None, None, True) #{1} self.uid = str(uid) jscode = JsTemplate.render(uid=self.uid, onclick=click_callback is not None, onrelayout=relayout_callback is not None) html += jscode #{2} self.value = html #{3} self.click_callback = click_callback self.relayout_callback = relayout_callback self.comm_manager.register_target(self.uid, self._on_open) #{4}
def plotly_hack(fig): r"""Work around to get plotly on html. Parameters ---------- fig : plotly figure Returns ------- html : string the html string """ from plotly.offline.offline import _plot_html plot_html, plotdivid, width, height = _plot_html( fig, False, "", True, '100%', 525, False) return plot_html
def getplotly(dataseries, layout): config = {} config['showLink'] = False figure_or_data = {"data": dataseries, "layout": layout} plot_html = plot_html, plotdivid, width, height = _plot_html( figure_or_data, config, False, '100%', 600, False) resize_script = '' if width == '100%' or height == '100%': resize_script = ( '' '<script type="text/javascript">' 'window.removeEventListener("resize");' 'window.addEventListener("resize", function(){{' 'Plotly.Plots.resize(document.getElementById("{id}"));}});' '</script>').format(id=plotdivid) graph = ''.join([plot_html, resize_script]) return graph
def opportunities_boxplot(df: pd.DataFrame, week: int=None): """ :param df: :param week: :return: """ title_param = { 'name': df.territory_name.unique()[0], 'week': '' } if week not in [0, None]: title_param['week'] = 'até a semana epidemiológica %s ' % week df_plot = df.iloc[:, :-1].dropna(how='all', axis=1) # Set ymax to higher upper fence: q1 = df_plot.quantile(.25, axis=0) q3 = df_plot.quantile(.75, axis=0) ymax = (q3 + 1 * (q3 - q1)).max() # Set bottom and right margin to avoid xaxis labels beeing cut off figure = df_plot.iplot( kind='box', boxpoints=False, margin={'b': 130, 't': 50, 'r': 120, 'l': 50 }, showlegend=False, yTitle='Dias', asFigure=True ) figure['layout']['yaxis1'].update(range=[0, ymax]) figure['layout']['xaxis1'].update(tickangle=30) return _plot_html( figure_or_data=figure, config={}, validate=True, default_width='100%', default_height=450, global_requirejs='' )[0]
def create_tweet_chart(cls, df: pd.DataFrame, year_week) -> 'Plotly_HTML': """ :param df: :param var_climate: :param year_week: :param climate_crit: :param climate_title: :return: """ df_tweet = df.reset_index()[['SE', 'tweets']] df_tweet = df_tweet[df_tweet.SE >= year_week - 200] df_tweet['SE'] = df_tweet.SE.map(lambda v: '%s/%s' % (str(v)[:4], str(v)[-2:])) df_tweet.rename(columns={'tweets': 'menções'}, inplace=True) figure = df_tweet.iplot(x=['SE'], y=['menções'], asFigure=True, showlegend=True, xTitle='Período (Ano/Semana)') figure['layout']['xaxis1'].update(tickangle=-60, nticks=len(df_tweet) // 4) figure['layout']['yaxis1'].update(title='Tweets') figure['layout']['legend'].update(x=-.1, y=1.2, traceorder='normal', font=dict(family='sans-serif', size=12, color='#000'), bgcolor='#FFFFFF', bordercolor='#E2E2E2', borderwidth=1) return _plot_html(figure_or_data=figure, config={}, validate=True, default_width='100%', default_height=500, global_requirejs='')[0]
def plot_plotly(chart, width='100%', height=525): # produce the html in Ipython compatible format plot_html, plotdivid, width, height = _plot_html(chart, {'showLink': False}, True, width, height, True) # define the plotly js library source url head = '<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>' # extract the div element from the ipython html div = plot_html[0:plot_html.index('<script')] # extract the script element from the ipython html script = plot_html[plot_html.index('Plotly.newPlot'):plot_html.index('});</script>')] + ';' # combine div and script to build the body contents body = '<body>{div}<script>{script}</script></body>'.format(div=div, script=script) # instantiate a figure object figure = Figure() # add the head figure.header.add_child(Element(head)) # add the body figure.html.add_child(Element(body)) return figure
def offline_plotly_data(data, filename='plotly.html', config=None, validate=True, default_width='100%', default_height=525, global_requirejs=False): """ Write a plotly scatter plot to HTML file that doesn't require server >>> import pandas as pd >>> from plotly.offline.offline import _plot_html >>> from plotly.graph_objs import Scatter, Marker, Layout, YAxis, XAxis >>> df = pd.read_csv('https://plot.ly/~etpinard/191.csv') >>> df.columns = [eval(c) if c[0] in '"\'' else str(c) for c in df.columns] >>> data = {'data': [ >>> Scatter(x=df[continent+', x'], >>> y=df[continent+', y'], >>> text=df[continent+', text'], >>> marker=Marker(size=df[continent+', size'], sizemode='area', sizeref=131868,), >>> mode='markers', >>> name=continent) for continent in ['Africa', 'Americas', 'Asia', 'Europe', 'Oceania'] >>> ], >>> 'layout': Layout(xaxis=XAxis(title='Life Expectancy'), yaxis=YAxis(title='GDP per Capita', type='log')) >>> } >>> html = offline_plotly_data(data) """ config_default = dict(DEFAULT_PLOTLY_CONFIG) if config is not None: config_default.update(config) with open(os.path.join(DATA_PATH, 'plotly.js.min'), 'rt') as f: js = f.read() html, divid, width, height = _plot_html(data, config=config_default, validate=validate, default_width=default_width, default_height=default_height, global_requirejs=global_requirejs) html = PLOTLY_HTML.format(plotlyjs=js, plotlyhtml=html) if filename and isinstance(filename, str): with open(filename, 'wt') as f: f.write(html) return html
def opportunities_boxplot(df: pd.DataFrame, week: int = None): """ :param df: :param week: :return: """ title_param = {'name': df.territory_name.unique()[0], 'week': ''} if week not in [0, None]: title_param['week'] = 'até a semana epidemiológica %s ' % week df_plot = df.iloc[:, :-1].dropna(how='all', axis=1) # Set ymax to higher upper fence: q1 = df_plot.quantile(.25, axis=0) q3 = df_plot.quantile(.75, axis=0) ymax = (q3 + 1 * (q3 - q1)).max() # Set bottom and right margin to avoid xaxis labels beeing cut off figure = df_plot.iplot(kind='box', boxpoints=False, margin={ 'b': 130, 't': 50, 'r': 120, 'l': 50 }, showlegend=False, yTitle='Dias', asFigure=True) figure['layout']['yaxis1'].update(range=[0, ymax]) figure['layout']['xaxis1'].update(tickangle=30) return _plot_html(figure_or_data=figure, config={}, validate=True, default_width='100%', default_height=450, global_requirejs='')[0]
def ethio_ts(df=pd.DataFrame, scale_id=int, year=int): cols = [ 'Testes positivos', 'Influenza A', 'SARS-CoV-2', 'Influenza B', 'VSR', 'Adenovirus', 'Parainfluenza 1', 'Parainfluenza 2', 'Parainfluenza 3', ] trace = [] if scale_id == 2: ytitle = 'Casos' else: ytitle = 'Incidência' trace.append( go.Scatter( x=df['epiweek'], y=df[cols[0]], name=cols[0], mode='lines', )) for i, col in enumerate(cols[1:]): trace.append( go.Scatter(x=df['epiweek'], y=df[col], name=ytitle, mode='lines', showlegend=False)) nrows = len(cols) fig = tools.make_subplots( rows=nrows, cols=1, print_grid=False, subplot_titles=('Situação dos exames', 'Influenza A', 'SARS-CoV-2', 'Influenza B', 'VSR', 'Adenovirus', 'Parainfluenza 1', 'Parainfluenza 2', 'Parainfluenza 3')) for i in range(1, (nrows + 1)): fig.append_trace(trace[i - 1], i, 1) # Add extra lines to first subplot: extra_cols = [ 'Testes negativos', 'Casos sem teste laboratorial', 'Casos aguardando resultado', 'Casos sem informação laboratorial' ] for i, col in enumerate(extra_cols): fig.append_trace( go.Scatter(x=df['epiweek'], y=df[col], name=col, mode='lines', line=dict(color=cl.scales['12']['qual']['Paired'][-i])), 1, 1) # X-axis title and range: lastepiweek = int(episem(lastepiday(year), out='W')) extra_cols.extend(['Testes positivos']) if scale_id == 2: ymax = [ max(5 * np.ceil(df[extra_cols].max().max() / 5), 5), max(5 * np.ceil(df[cols[1:]].max().max() / 5), 5) ] else: ymax = [df[extra_cols].max().max(), df[cols[1:]].max().max()] for i in range(1, (nrows + 1)): xaxis = 'xaxis%s' % i yaxis = 'yaxis%s' % i fig['layout'][xaxis].update(range=[1, lastepiweek]) fig['layout'][yaxis].update(range=[0, ymax[min(1, i - 1)]], rangemode='nonnegative') fig['layout']['xaxis%s' % nrows].update(title='Semana epidemiológica', zeroline=True, showline=True) i = int(nrows / 2) fig['layout']['yaxis%s' % i].update(title=ytitle) fig['layout'].update(margin={'t': 50, 'l': 50}) return _plot_html(figure_or_data=fig, config={}, validate=True, default_width='100%', default_height=1600, global_requirejs='')[0]
</head> <body> <table> ''' for model in args.models: plotly_data, accuracy, top_k_accuracy, stats = evaluate(model, h5_files, args.top_k, categories) layout= go.Layout( showlegend=False, hovermode='closest') figure = go.Figure(data=plotly_data, layout=layout) plot_html, plotdivid, width, height = _plot_html(figure, False, '', True, '100%', '100%', False) html += '<tr><td colspan=2><h3>{}</h3>Overall accuracy: {} Overall top-{} accuracy: {}</td></tr><tr><td>{}</td>'.format(os.path.basename(model), accuracy, args.top_k, top_k_accuracy, plot_html) html += ''' <td> <img id="hover-image-'''+str(plotdivid)+'''" src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7"> <div id="hoverinfo-'''+str(plotdivid)+'''"></div> </td> </tr> <script> var myPlot = document.getElementById("''' + str(plotdivid) + '''"), hoverInfo = document.getElementById("hoverinfo-'''+str(plotdivid)+'''"),
xaxis=dict( title="False positive (hh100%={}, yt100%={})".format( 2649, 4254), range=[0, 2000], ), yaxis=dict( title="True positive rate", range=[0, 1], )) # Plot and embed in ipython notebook! fig = dict(data=data, layout=layout) #df=iplot(fig) # plot_html, plotdivid, width, height = _plot_html(fig, False, "", True, '100%', 525,'') # _plot_html(figure_or_data, config, validate, default_width, default_height, global_requirejs) plot_html, plotdivid, width, height = _plot_html(fig, "", True, '100%', 525, '') html_start = """ <html> <head> <script src="https://cdn.plot.ly/plotly-latest.min.js"></script> </head> <body>""" html_end = """ </body> </html>""" html_final = html_start + plot_html + html_end f = open("compare.html", 'w') f.write(html_final)
def create_incidence_chart(cls, df: pd.DataFrame, year_week: int, threshold_pre_epidemic: float, threshold_pos_epidemic: float, threshold_epidemic: float) -> 'Plotly_HTML': """ @see: https://stackoverflow.com/questions/45526734/ hide-legend-entries-in-a-plotly-figure :param df: :param year_week: :param threshold_pre_epidemic: float, :param threshold_pos_epidemic: float :param threshold_epidemic: float :return: """ df = df.reset_index()[[ 'SE', 'incidência', 'casos notif.', 'level_code' ]] # 200 = 2 years df = df[df.SE >= year_week - 200] df['SE'] = df.SE.map(lambda v: '%s/%s' % (str(v)[:4], str(v)[-2:])) k = 'incidência' df['alerta verde'] = df[df.level_code == 1][k] df['alerta amarelo'] = df[df.level_code == 2][k] df['alerta laranja'] = df[df.level_code == 3][k] df['alerta vermelho'] = df[df.level_code == 4][k] df['limiar epidêmico'] = threshold_epidemic df['limiar pós epidêmico'] = threshold_pos_epidemic df['limiar pré epidêmico'] = threshold_pre_epidemic figure_bar = df.iplot(asFigure=True, kind='bar', x=['SE'], y=[ 'alerta verde', 'alerta amarelo', 'alerta laranja', 'alerta vermelho' ], legend=True, showlegend=True, yTitle='Incidência', xTitle='Período (Ano/Semana)', color=[ 'rgb(0,255,0)', 'rgb(255,255,0)', 'rgb(255,150,0)', 'rgb(255,0,0)' ], hoverinfo='x+y+name') figure_threshold = df.iplot( asFigure=True, x=['SE'], y=[ 'limiar pré epidêmico', 'limiar pós epidêmico', 'limiar epidêmico' ], legend=False, showlegend=False, color=['rgb(0,255,0)', 'rgb(255,150,0)', 'rgb(255,0,0)'], hoverinfo='none') figure_line = df.iplot(asFigure=True, x=['SE'], y=['casos notif.'], legend=False, showlegend=False, secondary_y=['casos notif.'], secondary_y_title='Casos', hoverinfo='x+y+name', color=['rgb(33,33,33)']) figure_line['layout']['legend'].update( x=-.27, y=0.5, traceorder='normal', font=dict(family='sans-serif', size=12, color='#000'), bgcolor='#FFFFFF', bordercolor='#E2E2E2', borderwidth=1, orientation='l', ) figure_line['layout']['yaxis1'].update(title='Incidência') figure_line['layout'].update( title=('Limiares de incidência:: ' + 'pré epidêmico=%s; ' + 'pós epidêmico=%s; ' + 'epidêmico=%s;') % ('{:.1f}'.format(threshold_pre_epidemic), '{:.1f}'.format(threshold_pos_epidemic), '{:.1f}'.format(threshold_epidemic)), showlegend=True) figure_threshold.data.extend(figure_bar.data) figure_line.data.extend(figure_threshold.data) figure_line['layout']['yaxis2'].update( showgrid=False, range=[0, df['casos notif.'].max()]) figure_line['layout']['xaxis1'].update(tickangle=-60, nticks=len(df) // 4, title='Período (Ano/Semana)') for trace in figure_line['data']: if trace['name'] == 'casos notif.': trace['visible'] = 'legendonly' return _plot_html(figure_or_data=figure_line, config={}, validate=True, default_width='100%', default_height=500, global_requirejs='')[0]
def draw_chart(self): import plotly.graph_objs as go from plotly.offline.offline import _plot_html import pandas as pd diarios = [] bancos = self.line_ids.filtered(lambda x: x.liquidity) for item in bancos: diarios.append((item.amount, item.name)) movimentacoes = [] for item in self.line_ids.filtered(lambda x: not x.liquidity): movimentacoes.append((item.amount, item.date, item.line_type)) diarios = pd.DataFrame(diarios, columns=['total', 'name']) moves = pd.DataFrame( movimentacoes, columns=['total', 'date_maturity', 'type']) moves['total'] = moves["total"].astype(float) moves['date_maturity'] = pd.to_datetime(moves["date_maturity"]) moves['receitas'] = moves["total"] moves['despesas'] = moves["total"] moves.ix[moves.type == 'payable', 'receitas'] = 0.0 moves.ix[moves.type == 'receivable', 'despesas'] = 0.0 moves = moves.sort_values(by="date_maturity") moves["acumulado"] = moves["total"].cumsum() moves["acumulado"] += diarios["total"].sum() saldo = [] saldo_inicial = 0.0 for index, row in diarios.iterrows(): saldo.append(go.Bar( x=["Saldo"], y=[row["total"]], name=row["name"] )) saldo_inicial += row["total"] acumulado_x = pd.Series(["Saldo"]) acumulado_y = pd.Series([saldo_inicial]) trace3 = go.Bar( x=moves['date_maturity'], y=moves['receitas'], name='Receitas', ) trace4 = go.Bar( x=moves['date_maturity'], y=moves['despesas'], name='Despesas', ) moves.drop_duplicates( subset='date_maturity', keep='last', inplace=True) x = acumulado_x.append(moves["date_maturity"]) y = acumulado_y.append(moves["acumulado"]) trace5 = go.Scatter( x=x, y=y, mode='lines+markers', name="Saldo", line=dict( shape='spline' ), ) data = [trace3, trace4, trace5] layout = go.Layout( barmode='stack', xaxis=dict( tickformat="%d-%m-%Y" ), ) fig = go.Figure(data=data, layout=layout) plot_html, plotdivid, width, height = _plot_html( fig, {}, True, '100%', 525, False) return plot_html
def create_tweet_chart( cls, df: pd.DataFrame, year_week, disease: str ) -> 'Plotly_HTML': """ :param df: :param year_week: :param disease: :return: """ ks_cases = [ 'casos notif. {}'.format(disease), ] df_tweet = df.reset_index()[['SE', 'tweets'] + ks_cases] df_tweet = df_tweet[ df_tweet.SE >= year_week - 200 ] df_tweet.rename(columns={'tweets': 'menções'}, inplace=True) df_grp = df_tweet.groupby(df.index)[ ['menções'] + ks_cases ].sum().reset_index() df_grp['SE'] = df_grp.SE.map( lambda v: '%s/%s' % (str(v)[:4], str(v)[-2:]) ) fig_tweet = df_grp.iplot( x=['SE'], y=['menções'], asFigure=True, showlegend=True, xTitle='Período (Ano/Semana)', color=['rgb(128,128,128)'] ) fig_cases = df_grp.iplot( x=['SE'], y=ks_cases, asFigure=True, secondary_y=ks_cases, secondary_y_title='Casos', showlegend=True, xTitle='Período (Ano/Semana)', color=['rgb(0,0,255)'] ) fig_cases.data.extend(fig_tweet.data) fig_cases['layout']['xaxis1'].update( tickangle=-60, nticks=len(df_grp) // 24 ) fig_cases['layout']['yaxis1'].update( title='Tweets', range=[0, df_grp['menções'].max()] ) fig_cases['layout']['yaxis2'].update( range=[0, df_grp[ks_cases].max().max()] ) fig_cases['layout'].update( title='Casos {} / Menções mídia social'.format(disease) ) fig_cases['layout']['legend'].update( x=-.1, y=1.2, traceorder='normal', font=dict( family='sans-serif', size=12, color='#000' ), bgcolor='#FFFFFF', bordercolor='#E2E2E2', borderwidth=1 ) return _plot_html( figure_or_data=fig_cases, config={}, validate=True, default_width='100%', default_height=300, global_requirejs='' )[0]
#fig = tls.make_subplots(rows=2, cols=1) errorfig_vcc= go.Figure(data=error_vcc, layout = layout1) errorfig_vdbl= go.Figure(data=error_vdbl, layout = layout1) errorfig_vcc['layout'].update(title=device+' ICC Ratio_PMV')#height=600, width=600, errorfig_vdbl['layout'].update(title=device+' ICC Ratio_VDBL') #fig = go.Figure(data=databoth, layout=layout1) #fig.append_trace(error_vcc, 1, 1) #fig.append_trace(error_vdbl, 2, 1) #plot(errorfig) from plotly.offline.offline import _plot_html plot_html, plotdivid, width, height = _plot_html(errorfig_vcc, False, "", True, '100%', 525,global_requirejs=False) plot_html_1, plotdivid, width, height = _plot_html(errorfig_vdbl, False, "", True, '100%', 525,global_requirejs=False) file="<html><head><script src='https://cdn.plot.ly/plotly-latest.min.js'></script></head><body>" + plot_html+plot_html_1+ "</body></html>" Html_file= open(outfolder1+device+'_Die_Intercept.html',"w") Html_file.write(file) Html_file.close() print(device + " Die ICC Done")
combtest100 = scatter100 + bar100 figtest0 = go.Figure(data=combtest0, layout=layout) figtest0['layout'].update(barmode='stack') figtest0['layout'].update(yaxis=dict(title='Count')) figtest0['layout'].update(title=device + ' Lot Level Test Program Plot: 0C') figtest100 = go.Figure(data=combtest100, layout=layout) figtest100['layout'].update(barmode='stack') figtest100['layout'].update(yaxis=dict(title='Count')) figtest100['layout'].update(title=device + ' Lot Level Test Program Plot: 100C') from plotly.offline.offline import _plot_html plot_test_0, plotdivid, width, height = _plot_html( figtest0, False, "", True, '100%', 525, global_requirejs=False) #,global_requirejs=False plot_test_100, plotdivid, width, height = _plot_html( figtest100, False, "", True, '100%', 525, global_requirejs=False) #,global_requirejs=False file_test0 = "<html><head><script src='https://cdn.plot.ly/plotly-latest.min.js'></script></head><body>" + plot_test_0 + "</body></html>" file_test100 = "<html><head><script src='https://cdn.plot.ly/plotly-latest.min.js'></script></head><body>" + plot_test_100 + "</body></html>" html_test0 = open(outfolder2 + "/" + "Lot_WS_Rev_0.html", "w") html_test0.write(file_test0) html_test0.close() html_test100 = open(outfolder2 + "/" + "Lot_WS_Rev_100.html", "w") html_test100.write(file_test100)
def query(compares, targets, between, by, chartMode, barMode, pieMode, sortBy, jsonfile): # compares = ['assigned', 'not_assigned', 'guest'] # target = {"m":"","h":"","s": "","p":""} # between = 'masseur'|'helper'|'shop'|'date' conn = pymysql.connect(host='localhost', port=3306, user='******', passwd='d4sg', db='d4sg_vim',charset='utf8') cur = conn.cursor() cur.execute("SELECT wid,mid,hid,sid,assigned,not_assigned,guest_num,log_date FROM worklog") worklogs = list(cur.fetchall()) cur.execute("SELECT * FROM masseur") masseurs = list(cur.fetchall()) cur.execute("SELECT * FROM helper") helpers = list(cur.fetchall()) cur.execute("SELECT * FROM shop") shops = list(cur.fetchall()) # find target target_masseurs = masseurs target_helpers = helpers target_shops = shops timeSeriesPlot = False fromDate = None toDate = None if targets["m"] != "": target_masseurs = filter(lambda x: x[1] == targets["m"].decode("utf-8"), target_masseurs) for i in target_masseurs: worklogs = filter(lambda x: x[1] == i[0], worklogs) if targets["h"] != "": target_helpers = filter(lambda x: x[1] == targets["h"].decode("utf-8"), target_helpers) for i in target_helpers: worklogs = filter(lambda x: x[2] == i[0], worklogs) if targets["s"] != "": target_shops = filter(lambda x: x[1] == targets["s"].decode("utf-8"), target_shops) for i in target_shops: worklogs = filter(lambda x: x[3] == i[0], worklogs) if targets["p"] != None: if chartMode == "line": timeSeriesPlot = True fromDate = targets["p"]["from"].date() toDate = targets["p"]["to"].date() worklogs = filter(lambda x: x[7] >= fromDate and x[7] <= toDate, worklogs) # between condition # dataList = [ # (TARGET_NAME,[ WORKLOGS ]) # ] dataList = [] if between == "masseur": for masseur in target_masseurs: mid = masseur[0] logs = filter(lambda x: x[1] == mid ,worklogs) if len(logs) == 0: continue; else: dataList.append((masseur[1],logs)) elif between == "helper": for helper in target_helpers: hid = helper[0] logs = filter(lambda x: x[2] == hid ,worklogs) if len(logs) == 0: continue; else: dataList.append((helper[1],logs)) elif between == "shop": for shop in target_shops: sid = shop[0] logs = filter(lambda x: x[3] == sid ,worklogs) if len(logs) == 0: continue; else: dataList.append((shop[1],logs)) else: # for time period # dataList = [ # (TIME_PERIOD,[ WORKLOGS ]) # ] fromDate = between["from"].date() toDate = between["to"].date() if between["step"] == "week": step = timedelta(days=7) to = fromDate + step while to <= toDate: dateRange = fromDate.strftime("%Y-%m-%d")+'至'+(to-timedelta(days=1)).strftime("%Y-%m-%d") logs = filter(lambda x: x[7] >= fromDate and x[7] < to, worklogs) if len(logs) == 0: fromDate = to to = to + step continue; else: dataList.append((dateRange,logs)) fromDate = to to = to + step elif between["step"] == "month": while fromDate.month <= toDate.month: month = fromDate.strftime("%Y年%m月") logs = filter(lambda x: x[7].month == fromDate.month,worklogs) fromDate = add_months(fromDate,1) if len(logs) == 0: continue; else: dataList.append((month,logs)) elif between["step"] == "season": while fromDate.month <= toDate.month: to = add_months(fromDate,2) monthRange = fromDate.strftime("%Y年%m月") +'至'+to.strftime("%Y年%m月") logs = filter(lambda x: x[7].month >= fromDate.month and x[7].month < to.month, worklogs) fromDate = add_months(fromDate,3) if len(logs) == 0: continue; else: dataList.append((monthRange,logs)) # compare result and create plot data = [] layout = None if timeSeriesPlot: # line chart daynum = fromDate - toDate x = [toDate + timedelta(days=num) for num in range(daynum.days, 1)] for name, logs in dataList: y = None if by == "sum": for compare in compares: y = [(date,reduce(apply_lambda(compares), grp, 0)) for date, grp in itertools.groupby(logs, key=lambda x: x[7])] elif by == "count": y = [(date,len(list(grp))) for date, grp in itertools.groupby(logs, key=lambda x: x[7])] elif by == "average": y = [(date, list(grp))for date, grp in itertools.groupby(logs, key=lambda x: x[7])] y = [(date,reduce(apply_lambda(compares), grp, 0)/float(len(grp))) if len(grp) != 0 else (date, 0) for date, grp in y] for d in x: if not d in [date for date,value in y]: y.append((d, 0)) y.sort(key=lambda a: a[0], reverse=False) trace = Scatter( x = x, y = [v for d,v in y], name = name ) data.append(trace) layout = dict( xaxis = dict( tickformat = "%Y年%m月%d日" ), yaxis=dict( title='+'.join(map(lambda x: EnglishToChinese[x], compares)), titlefont=dict( family='Arial, sans-serif', size=14, color='black' ), ) ) fig = Figure(data=data, layout=layout) plot_html, plotdivid, width, height = _plot_html(fig, False, "", True, '100%', 525, False) print plot_html else: # tmp = [(TARGET_NAME,{COMPARE:[VALUES]})] tmp = [] for name, logs in dataList: values = {} for compare in compares: if by == "sum": values[compare] = reduce(apply_lambda([compare]),logs , 0) elif by == "count": values[compare] = len(logs) elif by == "average": if len(logs) != 0: values[compare] = reduce(apply_lambda([compare]),logs , 0)/float(len(logs)) tmp.append((name, values)) # Sort List by SORTBY if sortBy != None: if sortBy in compares: tmp = sorted(tmp, key=lambda a: a[1][sortBy], reverse=True) elif sortBy == "assigned_not_assigned": tmp = sorted(tmp, key= lambda a: a[1]["assigned"]+a[1]["not_assigned"], reverse=True) if jsonfile: jsonArray = [] fieldname = "" if between == "masseur" or between == "helper": fieldname = "人名" elif between == "店名": fieldname = "店名" else: fieldname = "時間" for item in tmp: obj = {} obj[fieldname] = item[0] for compare in compares: obj[compare] = item[1][compare] jsonArray.append(obj) print json.dumps(jsonArray,sort_keys=True, indent=4, separators=(',', ': ')) else: x = map(lambda x: x[0], tmp) y = {} for compare in compares: y[compare] = map(lambda a: a[1][compare], tmp) if chartMode == "pie": if pieMode == "sum" and"assigned" in compares and "not_assigned" in compares: length = len(y["assigned"]) values = [] for i in range(0, length): values.append(y["assigned"][i]+y["not_assigned"][i]) data.append({ "labels":x, "values":values, "type":"pie", "name":EnglishToChinese["assigned_not_assigned"], "hoverinfo":"label+percent+name" }) layout = {} elif pieMode == "split": iterator = 0 for compare in compares: iterator += 1 xLeft = float(1)/len(compares) * (iterator-1) xRight = float(1)/len(compares) * iterator data.append({ "labels":x, "values":y[compare], "type":"pie", "name":EnglishToChinese[compare], "domain": {'x': [xLeft, xRight], 'y': [0,0.8]}, "hoverinfo":"label+percent+name" }) layout = {} elif chartMode == "bar": for compare in compares: trace = Bar( x = x, y = y[compare], name = EnglishToChinese[compare] ) data.append(trace) layout = Layout( barmode= barMode ) fig = Figure(data=data, layout=layout) plot_html, plotdivid, width, height = _plot_html(fig, False, "", True, '100%', 525, False) # plotly.offline.plot(fig) print plot_html cur.close() conn.close()
def get(self, request): params=dict() template='champs.html' if request.method == 'GET': if 'cid' in request.GET: cid=int(request.GET['cid']) events=MEvent.objects.select_related('p1','p2').filter(champ__id=cid).order_by('-dt') params['champ']=MChamp.objects.get(id=cid) params['events']=events template='events.html' if 'evid' in request.GET and 'sn' in request.GET: import plotly from plotly.offline.offline import _plot_html from plotly.graph_objs import Scatter, Layout setsCount=1 meid=int(request.GET['evid']) sn=int(request.GET['sn']) gid=0 if 'gid' in request.GET: gid=int(request.GET['gid']) currentgame=LSGame.objects.prefetch_related('lspoint_set').get(id=gid) params['gid']=gid mevent=MEvent.objects.select_related('p1','p2','champ').get(id=meid) params['mevent']=mevent setsCount=len(mevent.res.split(' ')) params['champ']=mevent.champ params['player1']=mevent.p1 params['player2']=mevent.p2 # # PlayerSetStats section # pss1=PlayerSetStats.objects.filter(meid=mevent.meid,pid=mevent.p1.mpid).order_by('sc') pss2=PlayerSetStats.objects.filter(meid=mevent.meid,pid=mevent.p2.mpid).order_by('sc') if (len(pss1)>0 and len(pss1)>0): pssstr1='[%s]' % ','.join([str(x) for x in pss1]) pssstr2='[%s]' % ','.join([str(x) for x in pss2]) params['pss']='<script type="text/javascript">var pss = [%s,%s];StatInit();</script>' % (pssstr1,pssstr2) else: params['pss']='' lsevent=LSEvent.objects.prefetch_related('lsgame_set').filter(meid=mevent.meid) isls=len(lsevent)>0 bfevent=BFEvent.objects.prefetch_related('bfodds_set').filter(meid=mevent.meid) isbf=len(bfevent)>0 if sn==0: # OP data for Not In-Play only opevent=OPEvent.objects.prefetch_related('opodds_set').filter(meid=mevent.meid) isop=len(opevent)>0 if isop: opev=opevent[0] if isbf: try: lastbf=BFOdds.objects.filter(eid=bfevent[0],ip=0).latest('dtc').dtc opchanges=OPOdds.objects.filter(ev=opev, dtc__lte=lastbf).order_by('dtc') except: opchanges=OPOdds.objects.filter(ev=opev).order_by('dtc') xrop=[i.dtc_tz for i in opchanges] y1rop=[i.w1 for i in opchanges] if opev.reversed==0 else [i.w2 for i in opchanges] y2rop=[i.w2 for i in opchanges] if opev.reversed==0 else [i.w1 for i in opchanges] y1ropm=[i.w1max for i in opchanges] if opev.reversed==0 else [i.w2max for i in opchanges] y2ropm=[i.w2max for i in opchanges] if opev.reversed==0 else [i.w1max for i in opchanges] ts=None te=None gameshapes=[] if isls: linestylegames=dict(color='rgb(128, 0, 128)',width=3,dash='dot') linestylepoints=dict(color='rgb(128, 128, 128)',width=0.75) lsev=lsevent[0] params['lsevent']=lsev shapespoint=[] annos=[] annospoint=[] games=[] if gid>0: isfirstgame=currentgame.sc1==0 and currentgame.sc2==0 and currentgame.setn==1 islastgame=LSGame.objects.filter(eid=lsev, dtc__gt=currentgame.dtc).count()==1 currentpoints=LSPoint.objects.filter(gid=currentgame).order_by('dtc') if not isfirstgame: prevgame=LSGame.objects.filter(eid=lsev, dtc__lt=currentgame.dtc).latest('dtc') currentgamestart=currentgame.dtc nextgame=LSGame.objects.filter(eid=lsev, dtc__gt=currentgame.dtc).order_by('dtc')[0] currentgameend=LSPoint.objects.filter(gid=nextgame).order_by('dtc')[0].dtc_tz pts=LSPoint.objects.filter(gid=prevgame).order_by('-dtc')[1].dtc if not isfirstgame else currentgamestart-timedelta(seconds=60) pte=LSPoint.objects.filter(gid=nextgame).order_by('dtc')[0].dtc if islastgame else LSPoint.objects.filter(gid=nextgame).order_by('dtc')[1].dtc shapespoint.append(dict(type='rect',yref='paper',x0=timezone.make_naive(pts),y0=0,x1=currentgame.dtc_tz,y1=1,fillcolor='#ffccbc', opacity=0.3,line={'width': 0})) shapespoint.append(dict(type='rect',yref='paper',x0=currentgameend,y0=0,x1=timezone.make_naive(pte),y1=1,fillcolor='#ffccbc', opacity=0.3,line={'width': 0})) if lsev.reversed==0: prevgametxt='Not IP' if isfirstgame else '%s*-%s'%(prevgame.sc1,prevgame.sc2) if prevgame.serve==1 else '%s-%s*'%(prevgame.sc1,prevgame.sc2) currentgametxt='%s*-%s'%(currentgame.sc1,currentgame.sc2) if currentgame.serve==1 else '%s-%s*'%(currentgame.sc1,currentgame.sc2) nextgametxt='' if islastgame else '%s*-%s'%(nextgame.sc1,nextgame.sc2) if nextgame.serve==1 else '%s-%s*'%(nextgame.sc1,nextgame.sc2) else: prevgametxt='Not IP' if isfirstgame else '%s-%s*'%(prevgame.sc2,prevgame.sc1) if prevgame.serve==1 else '%s*-%s'%(prevgame.sc2,prevgame.sc1) currentgametxt='%s-%s*'%(currentgame.sc2,currentgame.sc1) if currentgame.serve==1 else '%s*-%s'%(currentgame.sc2,currentgame.sc1) nextgametxt='' if islastgame else '%s-%s*'%(nextgame.sc2,nextgame.sc1) if nextgame.serve==1 else '%s*-%s'%(nextgame.sc2,nextgame.sc1) annospoint.append(dict(x=currentgame.dtc_tz-timedelta(seconds=10),y=1,xref='x',yref='paper', showarrow=False,text=prevgametxt, textangle=-90, font=dict(family='verdana', size=14, color='#1f77b4'),opacity=0.7 )) annospoint.append(dict(x=currentgameend+timedelta(seconds=10),y=1,xref='x',yref='paper', showarrow=False,text=nextgametxt, textangle=-90, font=dict(family='verdana', size=14, color='#1f77b4'),opacity=0.7 )) for point in currentpoints: shapespoint.append(dict(type='line',yref='paper',x0=point.dtc_tz,y0=0,x1=point.dtc_tz,y1=1,line=linestylepoints)) annospoint.append(dict(x=point.dtc_tz+timedelta(seconds=3),y=1,xref='x',yref='paper', showarrow=False,text=self.getPointText(point.sc1, point.sc2,lsev.reversed), textangle=-90, font=dict(family='verdana', size=11, color='#1f77b4'),opacity=0.8 )) shapespoint.append(dict(type='line',yref='paper',x0=currentgame.dtc_tz,y0=0,x1=currentgame.dtc_tz,y1=1,line=linestylegames)) shapespoint.append(dict(type='line',yref='paper',x0=currentgameend,y0=0,x1=currentgameend,y1=1,line=linestylegames)) gamelay=Layout(title=currentgametxt,shapes = shapespoint, annotations=annospoint,yaxis=dict(type='log', autorange=True),) if(sn==0): # not IP tmp=LSGame.objects.filter(eid=lsev,setn=1).order_by('dtc') if len(tmp)==0: ts=lsev.dtc if not isbf else BFOdds.objects.filter(eid=bfevent[0],ip=0).latest('dtc').dtc te=ts else: ts=tmp[0].dtc te=tmp[1].dtc gameshapes.append(dict(type='line',yref='paper',x0=timezone.make_naive(ts),y0=0,x1=timezone.make_naive(ts),y1=1,line=linestylegames)) gameshapes.append(dict(type='rect',yref='paper',x0=timezone.make_naive(ts),y0=0,x1=timezone.make_naive(te),y1=1,fillcolor='#ffccbc', opacity=0.3,line={'width': 0})) if isbf: bfchanges=BFOdds.objects.filter(eid=bfevent[0], dtc__lte=te,b1odds__gt=0,b2odds__gt=0,l1odds__gt=0,l2odds__gt=0).order_by('dtc') elif(sn<setsCount): # First or Middle set #gs=LSGame.objects.filter(eid=lsev,setn=sn-1).order_by('-dtc')[0].dtc curgame=LSGame.objects.filter(eid=lsev,setn=sn).order_by('dtc')[0] curgames=LSGame.objects.filter(eid=lsev,setn=sn).order_by('dtc')[0].dtc gameshapes.append(dict(type='line',yref='paper',x0=timezone.make_naive(curgames),y0=0,x1=timezone.make_naive(curgames),y1=1,line=linestylegames)) ts=curgames-timedelta(seconds=900) if sn==1 else curgames #if sn==1: gameshapes.append(dict(type='rect',yref='paper',x0=timezone.make_naive(ts),y0=0,x1=timezone.make_naive(curgames),y1=1,fillcolor='#ffccbc', opacity=0.3,line={'width': 0})) #else: # gameshapes.append(dict(type='rect',yref='paper',x0=ts,y0=0,x1=curgames,y1=1,fillcolor='#ffccbc', opacity=0.3,line={'width': 0})) te=LSGame.objects.filter(eid=lsev,setn=sn+1).order_by('dtc')[1].dtc ge=LSGame.objects.filter(eid=lsev,setn=sn+1).order_by('dtc')[0].dtc games=LSGame.objects.prefetch_related('lspoint_set').filter(eid=lsev, setn__gt=0, dtc__gte=ts, dtc__lte=te).order_by('dtc') if gid>0: ts=pts te=pte if isbf: bfchanges=BFOdds.objects.filter(eid=bfevent[0], dtc__gte=ts, dtc__lte=te,b1odds__gt=0,b2odds__gt=0,l1odds__gt=0,l2odds__gt=0).order_by('dtc') gameshapes.append(dict(type='line',yref='paper',x0=timezone.make_naive(ge),y0=0,x1=timezone.make_naive(ge),y1=1,line=linestylegames)) gameshapes.append(dict(type='rect',yref='paper',x0=timezone.make_naive(ge),y0=0,x1=timezone.make_naive(te),y1=1,fillcolor='#ffccbc', opacity=0.3,line={'width': 0})) else: # Last set ts=LSGame.objects.filter(eid=lsev,setn=sn-1).order_by('-dtc')[0].dtc endset=LSGame.objects.filter(eid=lsev,setn=6) games=LSGame.objects.prefetch_related('lspoint_set').filter(eid=lsev, dtc__gte=ts, setn__lt=6).order_by('dtc') te=games[1].dtc if gid>0: ts=pts te=endset[0].dtc if islastgame else pte else: gameshapes.append(dict(type='line',yref='paper',x0=timezone.make_naive(te),y0=0,x1=timezone.make_naive(te),y1=1,line=linestylegames)) gameshapes.append(dict(type='rect',yref='paper',x0=timezone.make_naive(ts),y0=0,x1=timezone.make_naive(te),y1=1,fillcolor='#ffccbc', opacity=0.3,line={'width': 0})) try: te=endset[0].dtc except: te=BFOdds.objects.filter(eid=bfevent[0]).latest('dtc').dtc if isbf: bfchanges=BFOdds.objects.filter(eid=bfevent[0], dtc__gte=ts, dtc__lte=te,b1odds__gt=0,b2odds__gt=0,l1odds__gt=0,l2odds__gt=0).order_by('dtc') if len(endset)>0 or gid>0 else BFOdds.objects.filter(eid=bfevent[0], dtc__gte=ts,b1odds__gt=0,b2odds__gt=0,l1odds__gt=0,l2odds__gt=0).order_by('dtc') params['games']=[] if len(games)>0: c=0 for g in games: #points=LSPoint.objects.filter(gid=g).order_by('dtc') params['games'].append(g) if gid==0: if lsev.reversed==0: gametxt='%s*-%s'%(g.sc1,g.sc2) if g.serve==1 else '%s-%s*'%(g.sc1,g.sc2) else: gametxt='%s-%s*'%(g.sc2,g.sc1) if g.serve==1 else '%s*-%s'%(g.sc2,g.sc1) if not (sn<setsCount and c==len(games)-1): annos.append(dict(x=games[c].dtc_tz+timedelta(seconds=30),y=1,xref='x',yref='paper', showarrow=False,text=gametxt, textangle=-90, font=dict(family='verdana', size=12, color='#1f77b4'),opacity=0.8 )) if g.serve==1 and c<len(games)-2 and c>0: gameshapes.append(dict(type='rect',yref='paper',x0=g.dtc_tz,y0=0,x1=games[c+1].dtc_tz,y1=1,fillcolor='#d3d3d3', opacity=0.3,line={'width': 0})) c+=1 if (sn==0): title='Not In-Play' else: title='Set #%s' % sn lay=Layout(title=title,shapes = gameshapes, annotations=annos,yaxis=dict(type='log', autorange=True),) if isbf: bfev=bfevent[0] params['tscheduled']=bfev.dt if mevent.res!='': params['tcomplete']=bfev.dtc if (ts==None and te==None): bfchanges=BFOdds.objects.filter(eid=bfev,ip=0 if sn==0 else 1 ).order_by('dtc') params=self.getSetButtons(params,sn,setsCount,isls) try: goip=BFOdds.objects.filter(eid=bfev,ip=False).latest('dtc').dtc_tz except: goip='' params['goip']=goip params['bfchanges']=bfchanges plotly.tools.set_credentials_file(username='******', api_key='mYs3EJORl98E0vxkWvGr') xr=[i.dtc_tz for i in bfchanges] if bfev.reversed==0: y1r=[i.b1odds for i in bfchanges] y1rl=[i.l1odds for i in bfchanges] y2r=[i.b2odds for i in bfchanges] y2rl=[i.l2odds for i in bfchanges] p1=bfev.pid1.name p2=bfev.pid2.name else: y2r=[i.b1odds for i in bfchanges] y2rl=[i.l1odds for i in bfchanges] y1r=[i.b2odds for i in bfchanges] y1rl=[i.l2odds for i in bfchanges] p2=bfev.pid1.name p1=bfev.pid2.name dataseries=[] dataseries.append(Scatter(x=xr, y=y1r,mode = 'lines',name = p1+' (Back)', line=dict( color='rgb(40, 53, 147)', ))) dataseries.append(Scatter(x=xr, y=y1rl,mode = 'lines',name = p1+' (Lay)', line=dict( color='rgb(121, 134, 203)', ),visible="legendonly")) if sn==0 and isop: dataseries.append(Scatter(x=xrop, y=y1rop,mode = 'lines',name = p1+' (Avg)', line=dict( color='rgb(46, 125, 50)', ))) dataseries.append(Scatter(x=xrop, y=y1ropm,mode = 'lines',name = p1+' (Max)', line=dict( color='rgb(142, 36, 170)', ),visible="legendonly")) dataseries.append(Scatter(x=xr, y=y2r,mode = 'lines',name = p2+' (Back)', line=dict( color='rgb(216, 67, 21)', ))) dataseries.append(Scatter(x=xr, y=y2rl,mode = 'lines',name = p2+' (Lay)', line=dict( color='rgb(255, 138, 101)', ),visible="legendonly")) if sn==0 and isop: dataseries.append(Scatter(x=xrop, y=y2rop,mode = 'lines',name = p2+' (Avg)', line=dict( color='rgb(249, 168, 37)', ))) dataseries.append(Scatter(x=xrop, y=y2ropm,mode = 'lines',name = p2+' (Max)', line=dict( color='rgb(233, 30, 99)', ),visible="legendonly")) if gid>0: figure_or_data = { "data": dataseries, "layout":gamelay } elif isls and sn>0: figure_or_data = { "layout": lay, "data": dataseries } else: figure_or_data = { "data": dataseries, "layout":Layout(title='Non-IP',shapes = gameshapes,yaxis=dict(type='log', autorange=True),) } config = {} config['showLink'] = False #config['layout'] = layout plot_html = plot_html, plotdivid, width, height = _plot_html( figure_or_data,config, False, '100%', 600, False) resize_script = '' if width == '100%' or height == '100%': resize_script = ( '' '<script type="text/javascript">' 'window.removeEventListener("resize");' 'window.addEventListener("resize", function(){{' 'Plotly.Plots.resize(document.getElementById("{id}"));}});' '</script>' ).format(id=plotdivid) params['graph'] = ''.join([ plot_html, resize_script]) template='selected.html' else: params['cid']='' atp=MChamp.objects.filter(gender=1).order_by('name') wta=MChamp.objects.filter(gender=0).order_by('name') params['atp']=atp params['wta']=wta params['currentpage']='Tennis' return render(request,template,params)
def process_data(): # Database details: connection = MongoClient(MONGODB_HOST, MONGODB_PORT) collection = connection[DBS_NAME][COLLECTION_NAME] projects = collection.find(projection=fields) # Count the number of times a something is seen: def count_stuff(listofitems): listofitems_counted = Counter() for d in listofitems: t = d.split(',')[0] listofitems_counted[d] += 1 return(listofitems_counted) # Get data from db and save results to a list ############################# timeanddomain = [] # Hold the list of domains for counting for project in projects: dateandtime = project['time'] domain = project['domain'] line = ("{0},{1}").format(dateandtime, domain) timeanddomain.append(line) # Save the time of domain hits all_times = [] # Hold the times for item in timeanddomain: #print item time = item.split(',')[0] all_times.append(time) # Separate IP addresses from Domains ############################# temp1 = [] # Hold the domains that are not IPV4 unique_domains = [] # Hold the domains that are not IPV4 and IPV6. domain_count(domain) will use this to count the domains ip = [] # Hold the IP addresses (IPV4 and IPV6) empty_items = 0 for item in timeanddomain: domain = item.split(',')[1] if valid_ipv4(domain) == True: ip.append(domain) if valid_ipv6(domain) == True: ip.append(domain) # This next one takes all non-ipv4 items (domains) and adds them to the temp1 list: if valid_ipv4(domain) == False: temp1.append(domain) # This one reads the temp1 list that contains both domains and ipv6 addresses and takes only the non-ipv6 entries and adds them to the unique_domains list: for domain in temp1: if valid_ipv6(domain) == False: unique_domains.append(domain) count_of_domains = count_stuff(unique_domains) # Count the domains and save as count_of_domains count_of_times = count_stuff(all_times) # Count the domains and save as count_of_domains # Turn the count_of_domains into a dictionary # Used to set a threshold and view domains contacted over or under a certain number domainslist = [] temp = [] for key, value in count_of_domains.iteritems(): temp = [key,value] domainslist.append(temp) timeslist = [] temp = [] for key, value in count_of_times.iteritems(): temp = [key,value] timeslist.append(temp) ############################# # Determine normal in a loose manner: ############################# normal_traffic = [] suspicious_traffic = [] # Print domains with a certain number of visits for item in domainslist: domain = item[0] count = item[1] # if count is greater than or equal to a number: if count >= 2: #print("{0}, {1}".format(domain,count)) normal_traffic.append(item) # if count is equal to a number: if count < 5: #print("{0}, {1}".format(domain,count)) suspicious_traffic.append(item) # if count is less than or equal to a number: # if count <= 10: # print("{0}, {1}".format(domain,count)) ###################################################### # Take the suspicious domains and unique them, getting them ready to look at using third party tools: domain_counts = [] # firstlevel.tld, count domain_fullrequest_counts = [] # actual request, firstlevel.tld, count temp = [] # Temporary holder to unique the firstlevel.tld, as used below for line in suspicious_traffic: fulldomain = line[0] dom = tldextract.extract(fulldomain) if dom.suffix != '': domain = dom.domain + '.' + dom.suffix # Just the first level domain # Create a newline, which has the firstlevel domain, the full domain with subdomains and the count domain_fullrequest_count = "{0},{1},{2}".format(domain,fulldomain,line[1]) if domain in temp: continue else: domain_count = "{0},{1}".format(domain,line[1]) domain_counts.append(domain_count) domain_fullrequest_counts.append(domain_fullrequest_count) temp.append(domain) # Unique the domains ################################# # Check the suspicious traffic in # OpenDNS Investigate: token = () with open('investigate_token.txt') as API_KEY: token = API_KEY.read() token = token.rstrip() inv = investigate.Investigate(token) categories_list = [] security_categories_list = [] wl_domains = [] bl_domains = [] not_determined_domains = [] for line in domain_fullrequest_counts: line = line.split(',') domain = line[0] domain_fullrequest_count = "{0},{1},{2}".format(line[0],line[1],line[2]) res = inv.categorization(domain, labels=True) status = res[domain]['status'] if status == 0: # Get domain categorization and add it to categories_list content_category = res[domain]['content_categories'] if content_category == []: continue else: for value in content_category: categories_list.append(value) ############## not_determined_domains.append(domain_fullrequest_count) if status == 1: # Get domain categorization and add it to categories_list content_category = res[domain]['content_categories'] if content_category == []: continue else: for value in content_category: categories_list.append(value) ############## wl_domains.append(domain_fullrequest_count) #print domain_fullrequest_count if status == -1: bl_domains.append(domain_fullrequest_count) security_category = res[domain]['security_categories'] if security_category == []: continue else: for value in security_category: security_categories_list.append(value) categories_list.append(value) # Get domain categorization and add it to categories_list content_category = res[domain]['content_categories'] if content_category == []: continue else: for value in content_category: categories_list.append(value) ############## #domain_and_cat = str(domain) + ":" + str(security_category) #print (domain, security_category) count_of_categories = count_stuff(categories_list) # Count the categories and save as count_of_categories # Turn the count_of_categories into a dictionary category_list = [] temp = [] for key, value in count_of_categories.iteritems(): temp = [key,value] category_list.append(temp) count_of_security_categories = count_stuff(security_categories_list) # Count the categories and save as count_of_security_categories # Turn the count_of_security_categories into a dictionary security_category_list = [] temp = [] for key, value in count_of_security_categories.iteritems(): temp = [key,value] security_category_list.append(temp) #print security_categories_list ##################################### # CHART GENERATION ##################################### # Time visit Barchart: xtimedata = [] ytimedata = [] for item in timeanddomain: item = item.split(',') xtimedata.append(item[0]) ytimedata.append(item[1]) ''' timebarchart = discreteBarChart(name='discreteBarChart', height=600, width=1000) timebarchart.add_serie(y=ytimedata, x=xtimedata) timebarchart.buildhtml() timebarchart = discreteBarChart(name='discreteBarChart', height=600, width=1000) timebarchart.add_serie(y=df['date'], x=df['counts']) #timebarchart.add_serie(y=df.index.values, x=df['domain']) timebarchart.buildhtml() writefile = open('../flask/templates/timebar.html','w') writefile.write(timebarchart.htmlcontent)''' ##################################### #alldomains Barchart: xdomaindata = [] ydomaindata = [] for item in domainslist: xdomaindata.append(item[0]) ydomaindata.append(item[1]) ### Plotly does this better. All domains visit Barchart: '''alldomains = discreteBarChart(name='discreteBarChart', height=300, width=1000) alldomains.add_serie(y=ydomaindata, x=xdomaindata) alldomains.buildhtml() writefile = open('../flask/templates/alldomains.html','w') writefile.write(alldomains.htmlcontent)''' ##################################### # For a chart of suspicious domains: xsuspicious = [] ysuspicious = [] for item in domain_counts: item = item.split(',') xsuspicious.append(item[0]) # Domains ysuspicious.append(item[1]) # Count ### Plotly does this better. suspicious visit Barchart: '''suspiciousbarchart = discreteBarChart(name='discreteBarChart', height=600, width=1000) suspiciousbarchart.add_serie(y=ysuspicious, x=xsuspicious) suspiciousbarchart.buildhtml() writefile = open('../flask/templates/suspicious_domains.html','w') writefile.write(suspiciousbarchart.htmlcontent)''' ##################################### # For a chart of blacklisted domains: xblacklisted = [] yblacklisted = [] for item in bl_domains: item = item.split(',') xblacklisted.append(item[0]) # Domains yblacklisted.append(item[2]) # Count ### Plotly does this better. Blacklisted Barchart: '''blacklistedbarchart = discreteBarChart(name='discreteBarChart', height=300, width=600) blacklistedbarchart.add_serie(y=yblacklisted, x=xblacklisted) blacklistedbarchart.buildhtml() writefile = open('../flask/templates/blacklisted_domains.html','w') writefile.write(blacklistedbarchart.htmlcontent)''' ##################################### # For a chart of whitelisted domains: xwhitelisted = [] ywhitelisted = [] for item in wl_domains: item = item.split(',') xwhitelisted.append(item[0]) # Domains ywhitelisted.append(item[2]) # Count ### Plotly does this better. whitelisted Barchart: '''whitelistedbarchart = discreteBarChart(name='discreteBarChart', height=300, width=600) whitelistedbarchart.add_serie(y=ywhitelisted, x=xwhitelisted) whitelistedbarchart.buildhtml() writefile = open('../flask/templates/whitelisted_domains.html','w') writefile.write(whitelistedbarchart.htmlcontent)''' ##################################### # For a Pie Chart of categories: xcategorylist = [] ycategorylist = [] for item in category_list: xcategorylist.append(item[0]) # Categories ycategorylist.append(item[1]) # Count ### Blacklisted Barchart: type = 'pieChart' categorypiechart = pieChart(name=type, color_category='category20c', height=1000, width=1000) xdata = xcategorylist ydata = ycategorylist extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}} categorypiechart.add_serie(y=ydata, x=xdata, extra=extra_serie) categorypiechart.buildhtml() writefile = open('../flask/templates/category_piechart.html','w') writefile.write(categorypiechart.htmlcontent) df_categories = pd.DataFrame(data=ycategorylist,index=xcategorylist) ##################################### # For a Pie Chart of security categories: xseccategorylist = [] yseccategorylist = [] for item in security_category_list: xseccategorylist.append(item[0]) # Categories yseccategorylist.append(item[1]) # Count ### security category piehart: type = 'pieChart' security_categorypiechart = pieChart(name=type, color_category='category20c', height=450, width=450) xdata = xseccategorylist ydata = yseccategorylist extra_serie = {"tooltip": {"y_start": "", "y_end": ""}} security_categorypiechart.add_serie(y=ydata, x=xdata, extra=extra_serie) security_categorypiechart.buildhtml() writefile = open('../flask/templates/security_category_piechart.html','w') writefile.write(security_categorypiechart.htmlcontent) ##################################### # For a chart of neutral listed domains: xneutrallisted = [] yneutrallisted = [] for item in not_determined_domains: item = item.split(',') xneutrallisted.append(item[0]) # Domains yneutrallisted.append(item[2]) # Count ### Plotly does this better. neutral listed Barchart: '''neutrallistedbarchart = discreteBarChart(name='discreteBarChart', height=300, width=600) neutrallistedbarchart.add_serie(y=yneutrallisted, x=xneutrallisted) neutrallistedbarchart.buildhtml() writefile = open('../flask/templates/neutral_domains.html','w') writefile.write(neutrallistedbarchart.htmlcontent)''' ################################# stats_number_of_u_domains = str(len(unique_domains)) + " unique domains seen<br>" stats_number_of_t_domains = str(len(timeanddomain)) + " total domains<br>" stats_number_of_ips = str(len(ip)) + " total IP addresses<br>" stats_normaltraffic = ('Normal traffic (domains visited over 3 times): {0}<br>').format(len(normal_traffic)) stats_suspicioustraffic = ('Amount of suspicious traffic domains visited under 2 times): {0}<br>').format(len(suspicious_traffic)) stats_wl = ('<br>Of the suspicious domains (uniqued):<br>Whitelisted domains (OpenDNS): {0}<br>'.format(len(wl_domains))) stats_bl = ('Blacklisted domains (OpenDNS): {0}<br>'.format(len(bl_domains))) stats_neutral = ('Neutral domains (OpenDNS): {0}'.format(len(not_determined_domains))) stats_categories_list = ('Number of categories seen: {0}'.format(len(categories_list))) stats = stats_number_of_u_domains + stats_number_of_t_domains + stats_number_of_ips + stats_normaltraffic + stats_suspicioustraffic + stats_wl + stats_bl + stats_neutral writefile = open('../flask/templates/stats.html','w') writefile.write(stats) # This one's for plotly: xtime = [] ytime = [] for item in timeanddomain: item = item.split(',') xtime.append(item[0]) ytime.append(item[1]) d = {'date':xtime,'domain':ytime} df = pd.DataFrame(data=d) df['counts'] = df.groupby('domain').transform('count') # For plotly: df2 = pd.DataFrame(data=ytime,index=xtime) df2.index = pd.to_datetime(df2.index) ############################# # HTML for plotly plots: ############################# html_start = """<html> <head> <script src="../static/plotly-latest.min.js"></script> </head> <body>""" html_end = """ </body> </html>""" # Line plot: '''plot_html, plotdivid, width, height = _plot_html(df2.iplot(asFigure=True, kind ='scatter', subplots=True, shared_xaxes=True, fill=True, title='Count by day',dimensions=(800,800)), False, "", True, '100%', 525, False) html_bar_chart = html_start + plot_html + html_end f = open('../flask/templates/plottest_scatter.html', 'w') f.write(html_bar_chart) f.close()''' # Suspicious domains df_suspicious = pd.DataFrame(ysuspicious, xsuspicious) plot_html, plotdivid, width, height = _plot_html(df_suspicious.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Suspicious Traffic',dimensions=(600,600)), False, "", True, '100%', 525, False) html_bar_chart = html_start + plot_html + html_end f = open('../flask/templates/suspicious_traffic.html', 'w') f.write(html_bar_chart) f.close() # All domains visited df_alldomains = pd.DataFrame(ydomaindata, xdomaindata) plot_html, plotdivid, width, height = _plot_html(df_alldomains.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='All Traffic',dimensions=(600,300)), False, "", True, '100%', 525, False) html_bar_chart = html_start + plot_html + html_end f = open('../flask/templates/all_traffic.html', 'w') f.write(html_bar_chart) f.close() # Whitelisted df_whitelisted = pd.DataFrame(ywhitelisted, xwhitelisted) plot_html, plotdivid, width, height = _plot_html(df_whitelisted.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Whitelisted Traffic',dimensions=(600,300)), False, "", True, '100%', 525, False) html_bar_chart = html_start + plot_html + html_end f = open('../flask/templates/whitelisted_traffic.html', 'w') f.write(html_bar_chart) f.close() # Not categorized df_not_categorized = pd.DataFrame(yneutrallisted, xneutrallisted) plot_html, plotdivid, width, height = _plot_html(df_not_categorized.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Non-categorized Traffic',dimensions=(600,300)), False, "", True, '100%', 525, False) html_bar_chart = html_start + plot_html + html_end f = open('../flask/templates/not_categorized_traffic.html', 'w') f.write(html_bar_chart) f.close() # Blacklisted df_blacklisted = pd.DataFrame(yblacklisted, xblacklisted) plot_html, plotdivid, width, height = _plot_html(df_blacklisted.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Blacklisted Traffic',dimensions=(600,300)), False, "", True, '100%', 525, False) html_bar_chart = html_start + plot_html + html_end f = open('../flask/templates/blacklisted_traffic.html', 'w') f.write(html_bar_chart) f.close() # Time Series df_timeseries = pd.DataFrame(ytimedata,xtimedata) plot_html, plotdivid, width, height = _plot_html(df_timeseries.iplot(asFigure=True, kind ='bar', subplots=False, shared_xaxes=True, fill=True, title='Time Series',dimensions=(800,450)), False, "", True, '100%', 525, False) html_bar_chart = html_start + plot_html + html_end f = open('../flask/templates/timeseries.html', 'w') f.write(html_bar_chart) f.close()
def _kanban_dashboard_graph(self): import plotly.graph_objs as go from plotly import tools from plotly.graph_objs import Scatter from plotly.offline.offline import _plot_html import pandas as pd import pandas.io.sql as psql self.env.cr.execute("select sum(amount_total) as total, rp.name from \ account_invoice ac inner join res_partner rp on \ ac.partner_id = rp.id group by rp.name \ order by total desc") todos = self.env.cr.fetchall() df = pd.DataFrame(todos, columns=['total', 'name']) df["total"] = df["total"].astype(float) primeiros = df.iloc[:12] outros = df.iloc[12:] total = outros['total'].sum() df2 = pd.DataFrame([[total, 'Outros']], columns=['total', 'name']) primeiros = primeiros.append(df2) pie = go.Pie(labels=primeiros['name'], values=primeiros['total']) prod_df = df import numpy as np total = prod_df['total'].sum() a = prod_df['total'] / total prod_df['percentual'] = a prod_df['cum_percentual'] = prod_df['percentual'].cumsum(0) prod_df['ScoreA'] = prod_df['cum_percentual'] < 0.8 prod_df['A'] = np.where(prod_df['ScoreA'], 'A', None) prod_df['ScoreB'] = prod_df['cum_percentual'].between(0.8, 0.95) prod_df['B'] = np.where(prod_df['ScoreB'], 'A', None) prod_df['ScoreC'] = prod_df['cum_percentual'] > 0.95 prod_df['C'] = np.where(prod_df['ScoreC'], 'C', None) total = float(prod_df['name'].count()) prod_df['percent'] = (prod_df.index.values / total) prod_df['label_x'] = prod_df[['name', 'total', 'percentual']].apply( lambda x: u'{} - R$ {} - {:,.2%}'.format(x[0], x[1], x[2]), axis=1) # Curva ABC de clientes a_df = prod_df[prod_df['ScoreA'] == True] trace1 = go.Scatter( x=a_df['percent'], y=a_df['cum_percentual'], text=a_df['label_x'], mode='lines+markers', name='Clientes A', ) b_df = prod_df[prod_df['ScoreB']] trace2 = go.Scatter( x=b_df['percent'], y=b_df['cum_percentual'], text=b_df['label_x'], mode='lines+markers', name='Clientes B', ) c_df = prod_df[prod_df['ScoreC']] trace3 = go.Scatter( x=c_df['percent'], y=c_df['cum_percentual'], text=c_df['label_x'], mode='lines+markers', name='Clientes C', ) layout = go.Layout(showlegend=True, title="Curva ABC de Clientes", yaxis={ 'tickformat': '%', 'ticksuffix': ' de vendas', 'showticksuffix': 'first' }, xaxis={'tickformat': '%'}) fig = go.Figure(data=[trace1, trace2, trace3], layout=layout) plot_html, plotdivid, width, height = _plot_html( fig, False, "", True, '100%', 525, False) self.dashboard_graph = plot_html
def plot_ex(figure_or_data, show_link=True, link_text='Export to plot.ly', validate=True, resizable=False, lock_aspect_ratio=False, master=True, click_to_display=False, link_to=None, link_to_id=False, rel_figure_dir="figures"): """ Create a pyGSTi plotly graph locally, returning HTML & JS separately. Parameters ---------- figure_or_data : plotly.graph_objs.Figure or .Data or dict or list object that describes a Plotly graph. See https://plot.ly/python/ for examples of graph descriptions. show_link : bool, optional display a link in the bottom-right corner of link_text : str, optional the text of export link validate : bool, optional validate that all of the keys in the figure are valid? omit if you need to include extra, unnecessary keys in your figure. resizable : bool, optional Make the plot resizable by including a "resize" event handler and any additional initialization. lock_aspect_ratio : bool, optional Whether the aspect ratio of the plot should be allowed to change when it is sized based on it's container. master : bool, optional Whether this plot represents the "master" of a group of plots, all of the others which are "slaves". The relative sizing of the master of a group will determine the relative sizing of the slaves, rather than the slave's containing element. Useful for preserving the size of the features in a group of plots that may be different overall sizes. click_to_display : bool, optional Whether the plot should be rendered immediately or whether a "click" icon should be shown instead, which must be clicked on to render the plot. link_to : None or tuple of {"pdf", "pkl"} If not-None, the types of pre-rendered/computed versions of this plot that can be assumed to be present, and therefore linked to by additional items in the hover-over menu of the plotly plot. link_to_id : str, optional The base name (without extension) of the ".pdf" or ".pkl" files that are to be linked to by menu items. For example, if `link_to` equals `("pdf",)` and `link_to_id` equals "plot1234", then a menu item linking to the file "plot1234.pdf" will be added to the renderd plot. Returns ------- dict With 'html' and 'js' keys separately specifying the HTML and javascript needed to embed the plot. """ from plotly import __version__ as _plotly_version from plotly import tools as _plotlytools #Processing to enable automatic-resizing & aspect ratio locking fig = _plotlytools.return_figure_from_figure_or_data(figure_or_data, False) orig_width = fig.get('layout', {}).get('width', None) orig_height = fig.get('layout', {}).get('height', None) if lock_aspect_ratio and orig_width and orig_height: aspect_ratio = orig_width / orig_height else: aspect_ratio = None #Remove original dimensions of plot so default of 100% is used below # (and triggers resize-script creation) if orig_width: del fig['layout']['width'] if orig_height: del fig['layout']['height'] #Special polar plot case - see below - add dummy width & height so # we can find/replace them with variables in generated javascript. if 'angularaxis' in fig['layout']: fig['layout']['width'] = 123 fig['layout']['height'] = 123 config = {} config['showLink'] = show_link config['linkText'] = link_text #Add version-dependent kwargs to _plot_html call below plotly_version = tuple(map(int, _plotly_version.split('.'))) if plotly_version < (3, 8, 0): # "old" plotly with _plot_html function from plotly.offline.offline import _plot_html kwargs = {} if plotly_version >= (3, 7, 0): # then auto_play arg exists kwargs['auto_play'] = False #Note: removing width and height from layout above causes default values to # be used (the '100%'s hardcoded below) which subsequently trigger adding a resize script. plot_html, plotdivid, _, _ = _plot_html( fig, config, validate, '100%', '100%', global_requirejs=False, # no need for global_requirejs here **kwargs) # since we now extract js and remake full script. else: from plotly.io import to_html as _to_html import uuid as _uuid plot_html = _to_html(fig, config, auto_play=False, include_plotlyjs=False, include_mathjax=False, post_script=None, full_html=False, animation_opts=None, validate=validate) assert (plot_html.startswith("<div>") and plot_html.endswith("</div>")) plot_html = plot_html[len("<div>"):-len("</div>")].strip() assert (plot_html.endswith("</script>")) id_index = plot_html.find('id="') id_index_end = plot_html.find('"', id_index + len('id="')) plotdivid = _uuid.UUID(plot_html[id_index + len('id="'):id_index_end]) if orig_width: fig['layout']['width'] = orig_width if orig_height: fig['layout']['height'] = orig_height # Separate the HTML and JS in plot_html so we can insert # initial-sizing JS between them. NOTE: this is FRAGILE and depends # on Plotly output (_plot_html) being HTML followed by JS tag = '<script type="text/javascript">' end_tag = '</script>' iTag = plot_html.index(tag) plot_js = plot_html[iTag + len(tag):-len(end_tag)].strip() plot_html = plot_html[0:iTag].strip() full_script = '' #Note: in this case, upper logic (usually in an on-ready hander of the table/plot # group creation) is responsible for triggering a "create" event on the plot div # when appropriate (see workspace.py). #Get javascript for create and (possibly) resize handlers plotly_create_js = plot_js # the ususal plotly creation javascript plotly_resize_js = None if resizable: #the ususal plotly resize javascript plotly_resize_js = ' Plotly.Plots.resize(document.getElementById("{id}"));'.format( id=plotdivid) if 'angularaxis' in fig['layout']: #Special case of polar plots: Plotly does *not* allow resizing of polar plots. # (I don't know why, and it's not documented, but in plotly.js there are explict conditions # in Plotly.relayout that short-circuit when "gd.frameworks.isPolar" is true). So, # we just re-create the plot with a different size to mimic resizing. plot_js = plot_js.replace('"width": 123', '"width": pw').replace( '"height": 123', '"height": ph') plotly_resize_js = ('var plotlydiv = $("#{id}");\n' 'plotlydiv.children(".plotly").remove();\n' 'var pw = plotlydiv.width();\n' 'var ph = plotlydiv.height();\n' '{resized}\n').format(id=plotdivid, resized=plot_js) plotly_create_js = plotly_resize_js aspect_val = aspect_ratio if aspect_ratio else "null" groupclass = "pygsti-plotgroup-master" \ if master else "pygsti-plotgroup-slave" if link_to and ('pdf' in link_to) and link_to_id: link_to_pdf_js = ( "\n" " btn = $('#{id}').find('.modebar-btn[data-title=\"Show closest data on hover\"]');\n" " btn = cloneAndReplace( btn ); //Strips all event handlers\n" " btn.attr('data-title','Download PDF');\n" " btn.click( function() {{\n" " window.open('{relfigdir}/{pdfid}.pdf');\n" " }});\n").format(id=plotdivid, pdfid=link_to_id, relfigdir=rel_figure_dir) plotly_create_js += link_to_pdf_js if link_to and ('pkl' in link_to) and link_to_id: link_to_pkl_js = ( "\n" " btn = $('#{id}').find('.modebar-btn[data-title=\"Zoom\"]');\n" " btn = cloneAndReplace( btn ); //Strips all event handlers\n" " btn.attr('data-title','Download python pickle');\n" " btn.click( function() {{\n" " window.open('{relfigdir}/{pklid}.pkl');\n" " }});\n").format(id=plotdivid, pklid=link_to_id, relfigdir=rel_figure_dir) plotly_create_js += link_to_pkl_js plotly_click_js = "" if click_to_display and master: # move plotly plot creation from "create" to "click" handler plotly_click_js = plotly_create_js plotly_create_js = "" full_script = ( # (assume this will all be run within an on-ready handler) ' $("#{id}").addClass("{groupclass}");\n' # perform this right away ' $("#{id}").on("init", function(event) {{\n' # always add init-size handler ' pex_init_plotdiv($("#{id}"), {ow}, {oh});\n' ' pex_init_slaves($("#{id}"));\n' ' console.log("Initialized {id}");\n' ' }});\n' ' $("#{id}").on("click.pygsti", function(event) {{\n' ' plotman.enqueue(function() {{ \n' ' {plotlyClickJS} \n' ' }}, "Click-creating Plot {id}" );\n' ' $("#{id}").off("click.pygsti");\n' # remove this event handler ' console.log("Click-Created {id}");\n' ' }});\n' ' $("#{id}").on("create", function(event, fracw, frach) {{\n' # always add create handler ' pex_update_plotdiv_size($("#{id}"), {ratio}, fracw, frach, {ow}, {oh});\n' ' plotman.enqueue(function() {{ \n' ' $("#{id}").addClass("pygBackground");\n' ' {plotlyCreateJS} \n' ' pex_create_slaves($("#{id}"), {ow}, {oh});\n' ' }}, "Creating Plot {id}" );\n' ' console.log("Created {id}");\n' ' }});\n').format(id=plotdivid, ratio=aspect_val, groupclass=groupclass, ow=orig_width if orig_width else "null", oh=orig_height if orig_height else "null", plotlyClickJS=plotly_click_js, plotlyCreateJS=plotly_create_js) #Add resize handler if needed if resizable: full_script += ( ' $("#{id}").on("resize", function(event,fracw,frach) {{\n' ' pex_update_plotdiv_size($("#{id}"), {ratio}, fracw, frach, {ow}, {oh});\n' ' plotman.enqueue(function() {{ \n' ' {plotlyResizeJS} \n' ' pex_resize_slaves($("#{id}"), {ow}, {oh});\n' ' }}, "Resizing Plot {id}" );\n' ' //console.log("Resized {id}");\n' ' }});\n').format(id=plotdivid, ratio=aspect_val, ow=orig_width if orig_width else "null", oh=orig_height if orig_height else "null", plotlyResizeJS=plotly_resize_js) return {'html': plot_html, 'js': full_script}
def get(self, request): params = dict() template = 'op_all.html' if request.method == 'GET': if 'cid' in request.GET: cid = int(request.GET['cid']) champ = OPChamp.objects.get(id=cid) events = OPEvent.objects.select_related( 'p1', 'p2').filter(champ__id=cid) params['champ'] = champ params['events'] = events template = 'op_champ.html' if 'evid' in request.GET: import plotly from plotly.offline.offline import _plot_html from plotly.graph_objs import Scatter evid = int(request.GET['evid']) event = OPEvent.objects.select_related('p1', 'p2', 'champ').get(id=evid) params['event'] = event oddschanges = OPOdds.objects.filter(ev=event).order_by('dtc') params['oddschanges'] = oddschanges plotly.tools.set_credentials_file( username='******', api_key='mYs3EJORl98E0vxkWvGr') xr = [i.dtc for i in oddschanges] y1r = [i.w1 for i in oddschanges] y2r = [i.w2 for i in oddschanges] y1mr = [i.w1max for i in oddschanges] y2mr = [i.w2max for i in oddschanges] figure_or_data = [ Scatter(x=xr, y=y1r, mode='lines', name='Avg ' + event.p1.name), Scatter(x=xr, y=y2r, mode='lines', name='Avg ' + event.p2.name), Scatter(x=xr, y=y1mr, mode='lines', name='Max ' + event.p1.name), Scatter(x=xr, y=y2mr, mode='lines', name='Max ' + event.p2.name), ] config = {} config['showLink'] = False plot_html = plot_html, plotdivid, width, height = _plot_html( figure_or_data, config, True, '100%', 500, False) resize_script = '' if width == '100%' or height == '100%': resize_script = ( '' '<script type="text/javascript">' 'window.removeEventListener("resize");' 'window.addEventListener("resize", function(){{' 'Plotly.Plots.resize(document.getElementById("{id}"));}});' '</script>').format(id=plotdivid) params['graph'] = ''.join([plot_html, resize_script]) template = 'op_event.html' else: champs = OPChamp.objects.filter( opevent__isnull=False).distinct() params['champs'] = champs params['currentpage'] = 'OP' return render(request, template, params)
def ethio_ts(df=pd.DataFrame, scale_id=int, year=int): cols = [ 'Testes positivos', 'Influenza A', 'Influenza B', 'VSR', 'Adenovirus', 'Parainfluenza 1', 'Parainfluenza 2', 'Parainfluenza 3', ] trace = [] if scale_id == 2: ytitle = 'Casos' else: ytitle = 'Incidência' trace.append( go.Scatter( x=df['epiweek'], y=df[cols[0]], name=cols[0], mode='lines', ) ) for i, col in enumerate(cols[1:]): trace.append( go.Scatter( x=df['epiweek'], y=df[col], name=ytitle, mode='lines', showlegend=False ) ) nrows = len(cols) fig = tools.make_subplots( rows=nrows, cols=1, print_grid=False, subplot_titles=('Situação dos exames', 'Influenza A', 'Influenza B', 'VSR', 'Adenovirus', 'Parainfluenza 1', 'Parainfluenza 2', 'Parainfluenza 3')) for i in range(1, (nrows + 1)): fig.append_trace(trace[i - 1], i, 1) # Add extra lines to first subplot: extra_cols = ['Testes negativos', 'Casos sem teste laboratorial', 'Casos aguardando resultado', 'Casos sem informação laboratorial'] for i, col in enumerate(extra_cols): fig.append_trace( go.Scatter( x=df['epiweek'], y=df[col], name=col, mode='lines', line=dict(color=cl.scales['12']['qual']['Paired'][-i]) ), 1, 1) # X-axis title and range: lastepiweek = int(episem(lastepiday(year), out='W')) extra_cols.extend(['Testes positivos']) if scale_id == 2: ymax = [ max(5*np.ceil(df[extra_cols].max().max()/5), 5), max(5*np.ceil(df[cols[1:]].max().max()/5), 5) ] else: ymax = [ df[extra_cols].max().max(), df[cols[1:]].max().max() ] for i in range(1, (nrows + 1)): xaxis = 'xaxis%s' % i yaxis = 'yaxis%s' % i fig['layout'][xaxis].update(range=[1, lastepiweek]) fig['layout'][yaxis].update(range=[0, ymax[min(1, i-1)]], rangemode='nonnegative') fig['layout']['xaxis%s' % nrows].update(title='Semana epidemiológica', zeroline=True, showline=True) i = int(nrows/2) fig['layout']['yaxis%s' % i].update(title=ytitle) fig['layout'].update(margin={'t': 50, 'l': 50}) return _plot_html( figure_or_data=fig, config={}, validate=True, default_width='100%', default_height=1600, global_requirejs='' )[0]
def get(self, request): params = dict() template = 'bf_all.html' if request.method == 'GET': if 'cid' in request.GET: cid = int(request.GET['cid']) events = BFEvent.objects.select_related( 'pid1', 'pid2').filter(cid__id=cid) params['champ'] = events[0].cid.name if len(events) > 0 else '' params['events'] = events template = 'bf_champ.html' if 'evid' in request.GET and 'isip' in request.GET: import plotly from plotly.offline.offline import _plot_html from plotly.graph_objs import Scatter evid = int(request.GET['evid']) isip = request.GET['isip'] == '1' event = BFEvent.objects.get(id=evid) params['event'] = event oddschanges = BFOdds.objects.filter(eid=event, ip=isip).order_by('dtc') params['isip0'] = '' if isip else ' disabled' params['isip1'] = ' disabled' if isip else '' try: goip = BFOdds.objects.filter(eid=event, ip=False).latest('dtc').dtc except: goip = '' params['goip'] = goip params['oddschanges'] = oddschanges plotly.tools.set_credentials_file( username='******', api_key='mYs3EJORl98E0vxkWvGr') xr = [i.dtc for i in oddschanges] y1r = [i.b1odds for i in oddschanges] y2r = [i.b2odds for i in oddschanges] figure_or_data = [ Scatter(x=xr, y=y1r, mode='lines', name=event.pid1.name), Scatter(x=xr, y=y2r, mode='lines', name=event.pid2.name) ] config = {} config['showLink'] = False plot_html = plot_html, plotdivid, width, height = _plot_html( figure_or_data, config, True, '100%', 500, False) resize_script = '' if width == '100%' or height == '100%': resize_script = ( '' '<script type="text/javascript">' 'window.removeEventListener("resize");' 'window.addEventListener("resize", function(){{' 'Plotly.Plots.resize(document.getElementById("{id}"));}});' '</script>').format(id=plotdivid) params['graph'] = ''.join([plot_html, resize_script]) template = 'bf_event.html' else: params['cid'] = '' champs = BFChamp.objects.filter( bfevent__isnull=False).distinct() params['champs'] = champs params['currentpage'] = 'BF' return render(request, template, params)
def plotly_hack(fig): from plotly.offline.offline import _plot_html plot_html, plotdivid, width, height = _plot_html(fig, False, "", True, '100%', 525, False) return plot_html
def process_data(): # Database details: connection = MongoClient(MONGODB_HOST, MONGODB_PORT) collection = connection[DBS_NAME][COLLECTION_NAME] projects = collection.find(projection=fields) #projects = collection.find() # Get data from db and save results to a list ############################# gets = [] # Hold the get requests posts = [] # Hold the posts gets_and_posts = [] gets_full_entry = [] gets_with_a_request = [] posts_full_entry = [] total_entries = [] gets_and_posts_no_head = [] alltimes = [] for data in projects: time = data['time'] useragent = data['useragent'] request_method = data['request_method'] request = data['request'] src_ip = data['src_ip'] dest_ip = data['dest_ip'] dest_host = data['dest_host'] if request_method == 'GET': if len(request) > 2: # Dirty way to negate the requests with no data line = "{0},{1},{2},{3},{4},{5},{6}".format(time,src_ip,useragent,dest_ip,dest_host,request_method,request) gets_with_a_request.append(line) if len(request) > 2: # Dirty way to negate the requests with no data line = "{0},{1},{2},{3},{4},{5},{6}".format(time,src_ip,useragent,dest_ip,dest_host,request_method,request) else: line = "{0},{1},{2},{3},{4},{5},No request".format(time,src_ip,useragent,dest_ip,dest_host,request_method) alltimes.append(time) total_entries.append(line) # Used for counting the GETs and POSTs, for making the pie chart if request_method == 'GET': gets_and_posts.append('GET Requests') gets_and_posts_no_head.append(line) if request_method == 'POST': gets_and_posts.append('POSTs') gets_and_posts_no_head.append(line) ################################### # Get all the GETs (full line) and save to gets_full_entry if request_method == 'GET': gets.append(request_method) gets_full_entry.append(line) # Get all the POSTs (full line) and save to posts_full_entry if request_method == 'POST': posts.append(request_method) posts_full_entry.append(line) html_start = """<!DOCTYPE html> <html lang="en"> <head> <meta charset="utf-8" /> <link href="https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.css" rel="stylesheet" /> <script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js"></script> <script src="https://cdnjs.cloudflare.com/ajax/libs/nvd3/1.7.0/nv.d3.min.js"></script> <script src="../../static/plotly-latest.min.js"></script> </head> <body>""" html_end = """ </body> </html>""" ################################### # For a Pie Chart of GETS and POSTS: count_request_method = count_stuff(gets_and_posts) count_of_request_methods = [] temp = [] for key, value in count_request_method.iteritems(): temp = [key,value] count_of_request_methods.append(temp) xcategorylist = [] ycategorylist = [] for item in count_of_request_methods: xcategorylist.append(item[0]) ycategorylist.append(item[1]) type = 'pieChart' categorypiechart = pieChart(name=type, color_category='category20c', height=400, width=400) extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}} categorypiechart.add_serie(y=ycategorylist, x=xcategorylist, extra=extra_serie) categorypiechart.buildcontent() writefile = open('../flask/templates/pcap/pcap_category_piechart.html','w') writefile.write(html_start + categorypiechart.htmlcontent) writefile = open('../flask/templates/pcap/pcap_category_piechart.html','a') # Re-open for appending header = "<b>Time, Source IP, Destination Host, Destination IP, Request Method, Request</b><br>" writefile.write(header) #for line in gets_with_a_request: for line in total_entries: line = line.split(',') time, src_ip, dest_host, dest_ip, method, request = line[0],line[1],line[4],line[3],line[5],line[6] line = ("{0}, {1}, {2}, {3}, {4}<br>".format(time, src_ip, dest_host, dest_ip, method, request)) writefile.write(line) for line in posts_full_entry: line = line.split(',') time, src_ip, dest_host, dest_ip, method, request = line[0],line[1],line[4],line[3],line[5],line[6] line = ("{0}, {1}, {2}, {3}, {4}, {5}<br>".format(time, src_ip, dest_host, dest_ip, method, request)) writefile.write(line) writefile.write(html_end) # Timeseries of GETs vs POSTs with plotly xtime = [] ytime = [] for item in gets_and_posts_no_head: item = item.split(',') xtime.append(item[0]) ytime.append(item[5]) # Time Series df_timeseries = pd.DataFrame(ytime,xtime) plot_html, plotdivid, width, height = _plot_html(df_timeseries.iplot(asFigure=True, kind ='bar', subplots=False, shared_xaxes=True, fill=True, title='Time Series of HTTP Requests',dimensions=(800,450)), False, "", True, '100%', 525, False) html_bar_chart = html_start + plot_html + html_end f = open('../flask/templates/pcap/pcap_timeseries.html', 'w') f.write(html_bar_chart) f.close() # Gets and Posts on a timeseries: xtime = [] yrequest = [] posts_to_dst_ips = [] gets_to_dst_ips = [] total_dst_ips = [] total_src_ips = [] total_dst_hosts = [] for item in total_entries: item = item.split(',') time = item[0] src_ip = item[2] dst_ip = item[3] dst_host = item[4] request = item[5] if src_ip not in total_src_ips: total_src_ips.append(src_ip) if dst_ip not in total_dst_ips: total_dst_ips.append(dst_ip) if dst_host not in total_dst_hosts: total_dst_hosts.append(dst_host) # Separate the dst_ip's based on GETS and POSTS (for a map) if request == 'POST': posts_to_dst_ips.append(dst_ip) if request == 'GET': gets_to_dst_ips.append(dst_ip) # Get the count of POSTS for a timeseries if request == 'POST': xtime.append(time) yrequest.append(1) else: xtime.append(time) yrequest.append(0) # Create maps of the POSTS vs GETS: quickmap.ip_map_world('../../static/miller-2048x1502-color.jpg',posts_to_dst_ips,'../flask/templates/pcap/posts_to_dst_ips.svg') quickmap.ip_map_world('../../static/miller-2048x1502-color.jpg',gets_to_dst_ips,'../flask/templates/pcap/gets_to_dst_ips.svg') df_timeseries = pd.DataFrame(yrequest,xtime) plot_html, plotdivid, width, height = _plot_html(df_timeseries.iplot(asFigure=True, kind ='bar', subplots=False, shared_xaxes=True, fill=True, title='All Requests by time, showing the POSTs',dimensions=(800,450)), False, "", True, '100%', 525, False) html_bar_chart = html_start + plot_html + html_end f = open('../flask/templates/pcap/getsandposts_timeseries.html', 'w') f.write(html_bar_chart) f.close() # TO DO: Check out domains with investigate ####################### # STATS: number_of_gets = str(len(gets)) + " GET requests seen<br>" number_of_posts = str(len(posts)) + " POSTS seen<br>" stats_number_of_src_ips = str(len(total_src_ips)) + " total Source IP addresses<br>" stats_number_of_dst_ips = str(len(total_dst_ips)) + " total Destination IP addresses<br>" stats_number_of_dst_hosts = str(len(total_dst_hosts)) + " total Destination Hosts<br>" stats = number_of_gets + number_of_posts + stats_number_of_src_ips + stats_number_of_dst_ips + stats_number_of_dst_hosts writefile = open('../flask/templates/pcap/stats.html','w') writefile.write(stats)
def create_tweet_chart(cls, df: pd.DataFrame, year_week, disease: str) -> 'Plotly_HTML': """ :param df: :param year_week: :param disease: :return: """ ks_cases = [ 'casos notif. {}'.format(disease), ] df_tweet = df.reset_index()[['SE', 'tweets'] + ks_cases] df_tweet = df_tweet[df_tweet.SE >= year_week - 200] df_tweet.rename(columns={'tweets': 'menções'}, inplace=True) df_grp = df_tweet.groupby(df.index)[['menções'] + ks_cases].sum().reset_index() df_grp['SE'] = df_grp.SE.map(lambda v: '%s/%s' % (str(v)[:4], str(v)[-2:])) fig_tweet = df_grp.iplot(x=['SE'], y=['menções'], asFigure=True, showlegend=True, xTitle='Período (Ano/Semana)', color=['rgb(128,128,128)']) fig_cases = df_grp.iplot(x=['SE'], y=ks_cases, asFigure=True, secondary_y=ks_cases, secondary_y_title='Casos', showlegend=True, xTitle='Período (Ano/Semana)', color=['rgb(0,0,255)']) fig_cases.data.extend(fig_tweet.data) fig_cases['layout']['xaxis1'].update(tickangle=-60, nticks=len(df_grp) // 24) fig_cases['layout']['yaxis1'].update( title='Tweets', range=[0, df_grp['menções'].max()]) fig_cases['layout']['yaxis2'].update( range=[0, df_grp[ks_cases].max().max()]) fig_cases['layout'].update( title='Casos {} / Menções mídia social'.format(disease)) fig_cases['layout']['legend'].update(x=-.1, y=1.2, traceorder='normal', font=dict(family='sans-serif', size=12, color='#000'), bgcolor='#FFFFFF', bordercolor='#E2E2E2', borderwidth=1) return _plot_html(figure_or_data=fig_cases, config={}, validate=True, default_width='100%', default_height=300, global_requirejs='')[0]
def process_data(): pcapdata = [] for ts, buf in pcap: eth = dpkt.ethernet.Ethernet(buf) ip = eth.data tcp = ip.data dtstring=datetime.datetime.fromtimestamp(ts).strftime('%d-%m-%Y %H:%M:%S:%f') # Convert time to string #dt=datetime.datetime.strptime(dtstring,'%d-%m-%Y %H:%M:%S:%f') # Convert string time to datetime try: if len(tcp.data) > 0: http = dpkt.http.Request(tcp.data) headers = http.headers dest_ip = socket.inet_ntoa(ip.dst) src_ip = socket.inet_ntoa(ip.src) dest_host = headers['host'] uri = http.uri useragent = headers['user-agent'] request = repr(http['body']) #data = (dtstring, http.method) data = ("{0},{1},{2},{3},{4},{5},{6}".format(dtstring, src_ip, useragent, dest_host, dest_ip, http.method, request)) pcapdata.append(data) except: continue # Get data from db and save results to a list ############################# gets = [] # Hold the get requests posts = [] # Hold the posts gets_and_posts = [] gets_full_entry = [] gets_with_a_request = [] posts_full_entry = [] total_entries = [] gets_and_posts_no_head = [] alltimes = [] for data in pcapdata: data = data.split(',') time = data[0] useragent = data[2] request_method = data[5] request = data[6] src_ip = data[1] dest_ip = data[4] dest_host = data[3] if request_method == 'GET': if len(request) > 2: # Dirty way to negate the requests with no data line = "{0},{1},{2},{3},{4},{5},{6}".format(time,src_ip,useragent,dest_ip,dest_host,request_method,request) gets_with_a_request.append(line) if len(request) > 2: # Dirty way to negate the requests with no data line = "{0},{1},{2},{3},{4},{5},{6}".format(time,src_ip,useragent,dest_ip,dest_host,request_method,request) else: line = "{0},{1},{2},{3},{4},{5},No request".format(time,src_ip,useragent,dest_ip,dest_host,request_method) alltimes.append(time) total_entries.append(line) # Used for counting the GETs and POSTs, for making the pie chart if request_method == 'GET': gets_and_posts.append('GET Requests') gets_and_posts_no_head.append(line) if request_method == 'POST': gets_and_posts.append('POSTs') gets_and_posts_no_head.append(line) ################################### # Get all the GETs (full line) and save to gets_full_entry if request_method == 'GET': gets.append(request_method) gets_full_entry.append(line) # Get all the POSTs (full line) and save to posts_full_entry if request_method == 'POST': posts.append(request_method) posts_full_entry.append(line) html_start = """<!DOCTYPE html> <html lang="en"> <head> <meta charset="utf-8" /> <link href="../../flask/static/bower_components/nvd3/build/nv.d3.min.css" rel="stylesheet" /> <script src="../../flask/static/bower_components/d3/d3.min.js"></script> <script src="../../flask/static/bower_components/nvd3/build/nv.d3.min.js"></script> <script src="../../flask/static/plotly-latest.min.js"></script> </head> <body>""" html_end = """ </body> </html>""" ################################### # For a Pie Chart of GETS and POSTS: count_request_method = count_stuff(gets_and_posts) count_of_request_methods = [] temp = [] for key, value in count_request_method.iteritems(): temp = [key,value] count_of_request_methods.append(temp) xcategorylist = [] ycategorylist = [] for item in count_of_request_methods: xcategorylist.append(item[0]) ycategorylist.append(item[1]) type = 'pieChart' categorypiechart = pieChart(name=type, color_category='category20c', height=400, width=400) extra_serie = {"tooltip": {"y_start": "", "y_end": " cal"}} categorypiechart.add_serie(y=ycategorylist, x=xcategorylist, extra=extra_serie) categorypiechart.buildcontent() writefile = open('pcapstatic/pcap_category_piechart.html','w') writefile.write(html_start + categorypiechart.htmlcontent) writefile = open('pcapstatic/pcap_category_piechart.html','a') # Re-open for appending header = "<b>Time, Source IP, Destination Host, Destination IP, Request Method, Request</b><br>" writefile.write(header) #for line in gets_with_a_request: for line in total_entries: line = line.split(',') time, src_ip, dest_host, dest_ip, method, request = line[0],line[1],line[4],line[3],line[5],line[6] line = ("{0}, {1}, {2}, {3}, {4}<br>".format(time, src_ip, dest_host, dest_ip, method, request)) writefile.write(line) for line in posts_full_entry: line = line.split(',') time, src_ip, dest_host, dest_ip, method, request = line[0],line[1],line[4],line[3],line[5],line[6] line = ("{0}, {1}, {2}, {3}, {4}, {5}<br>".format(time, src_ip, dest_host, dest_ip, method, request)) writefile.write(line) writefile.write(html_end) # Timeseries of GETs vs POSTs with plotly xtime = [] ytime = [] for item in gets_and_posts_no_head: item = item.split(',') xtime.append(item[0]) ytime.append(item[5]) # Time Series df_timeseries = pd.DataFrame(ytime,xtime) plot_html, plotdivid, width, height = _plot_html(df_timeseries.iplot(asFigure=True, kind ='bar', subplots=False, shared_xaxes=True, fill=True, title='Time Series of HTTP Requests',dimensions=(800,450)), False, "", True, '100%', 525, False) html_bar_chart = html_start + plot_html + html_end f = open('pcapstatic/pcap_timeseries.html', 'w') f.write(html_bar_chart) f.close() # Gets and Posts on a timeseries: xtime = [] yrequest = [] posts_to_dst_ips = [] gets_to_dst_ips = [] total_dst_ips = [] total_src_ips = [] total_dst_hosts = [] for item in total_entries: item = item.split(',') time = item[0] src_ip = item[2] dst_ip = item[3] dst_host = item[4] request = item[5] if src_ip not in total_src_ips: total_src_ips.append(src_ip) if dst_ip not in total_dst_ips: total_dst_ips.append(dst_ip) if dst_host not in total_dst_hosts: total_dst_hosts.append(dst_host) # Separate the dst_ip's based on GETS and POSTS (for a map) if request == 'POST': posts_to_dst_ips.append(dst_ip) if request == 'GET': gets_to_dst_ips.append(dst_ip) # Get the count of POSTS for a timeseries if request == 'POST': xtime.append(time) yrequest.append(1) else: xtime.append(time) yrequest.append(0) # Create maps of the POSTS vs GETS: quickmap.ip_map_world('miller-2048x1502-color.jpg',posts_to_dst_ips,'pcapstatic/posts_to_dst_ips.svg') quickmap.ip_map_world('miller-2048x1502-color.jpg',gets_to_dst_ips,'pcapstatic/gets_to_dst_ips.svg') df_timeseries = pd.DataFrame(yrequest,xtime) plot_html, plotdivid, width, height = _plot_html(df_timeseries.iplot(asFigure=True, kind ='bar', subplots=False, shared_xaxes=True, fill=True, title='All Requests by time, showing the POSTs',dimensions=(800,450)), False, "", True, '100%', 525, False) html_bar_chart = html_start + plot_html + html_end f = open('pcapstatic/getsandposts_timeseries.html', 'w') f.write(html_bar_chart) f.close() # TO DO: Check out domains with investigate ####################### # STATS: number_of_gets = str(len(gets)) + " GET requests seen<br>" number_of_posts = str(len(posts)) + " POSTS seen<br>" stats_number_of_src_ips = str(len(total_src_ips)) + " total Source IP addresses<br>" stats_number_of_dst_ips = str(len(total_dst_ips)) + " total Destination IP addresses<br>" stats_number_of_dst_hosts = str(len(total_dst_hosts)) + " total Destination Hosts<br>" stats = number_of_gets + number_of_posts + stats_number_of_src_ips + stats_number_of_dst_ips + stats_number_of_dst_hosts writefile = open('pcapstatic/stats.html','w') writefile.write(stats)
def process_data(): # Count the number of times a something is seen: def count_stuff(listofitems): listofitems_counted = Counter() for d in listofitems: t = d.split(',')[0] info = d.split(',')[1] listofitems_counted[t] += 1 return(listofitems_counted) ##################################### # Process input file: event_ids = [] task_categories = [] all_items = [] successful_logons = [] successful_logoffs = [] logon_attempts = [] failed_logon_attempts = [] # Open the input file, convert datetime to datetime object and add separate items: with open(input_file) as f: for item in f: rawline = item.strip() s = rawline.split(',') audit_result = s[0] dateandtime = s[1] dt = datetime.datetime.strptime(dateandtime, "%m/%d/%Y %I:%M:%S %p") source = s[2] event_id = s[3] task_category = s[4] info = s[5] task_categories.append(task_category) event_ids.append(event_id) line = "{0},{1}".format(dt,info) all_items.append(line) if event_id == '4624': # Successful Logons line = "{0},{1}".format(dt,info) successful_logons.append(line) if event_id == '4634': # Successful Logoffs line = "{0},{1}".format(dt,info) successful_logoffs.append(line) if event_id == '4648': # Logon attempt line = "{0},{1}".format(dt,info) logon_attempts.append(line) if event_id == '4625': # Failed Logon attempts line = "{0},{1}".format(dt,info) failed_logon_attempts.append(line) #for item in successful_logons: # print item def make_a_list_of_dates_and_counts(listofitems): # Turn the count_of_successful_logons into a dictionary counted_items = [] temp = [] for key, value in listofitems.iteritems(): temp = [key,value] counted_items.append(temp) return counted_items _all_items = count_stuff(all_items) _count_of_successful_logons = count_stuff(successful_logons) _count_of_successful_logoffs = count_stuff(successful_logoffs) _count_of_logon_attempts = count_stuff(logon_attempts) _count_of_failed_logon_attempts = count_stuff(failed_logon_attempts) all_events = make_a_list_of_dates_and_counts(_all_items) successful_logons = make_a_list_of_dates_and_counts(_count_of_successful_logons) successful_logoffs = make_a_list_of_dates_and_counts(_count_of_successful_logoffs) logon_attempts = make_a_list_of_dates_and_counts(_count_of_logon_attempts) failed_logon_attempts = make_a_list_of_dates_and_counts(_count_of_failed_logon_attempts) ############################# # HTML for plotly plots: ############################# html_start = """<html> <head> <script src="../flask/static/plotly-latest.min.js"></script> </head> <body>""" html_end = """ </body> </html>""" #print security_categories_list ##################################### # CHART GENERATION ##################################### ######################### # Successful Logons x = [] y = [] for item in successful_logons: x.append(item[0]) y.append(item[1]) df_successful_logons = pd.DataFrame(y, x) plot_html, plotdivid, width, height = _plot_html(df_successful_logons.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Successful Logons',dimensions=(600,600)), False, "", True, '100%', 525, False) html_bar_chart = html_start + plot_html + html_end f = open('successful_logons.html', 'w') f.write(html_bar_chart) f.close() ########################## # successful logoffs # x = [] # y = [] # for item in successful_logoffs: # x.append(item[0]) # y.append(item[1]) # df_successful_logoffs = pd.DataFrame(y, x) # plot_html, plotdivid, width, height = _plot_html(df_successful_logoffs.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Successful Logoffs',dimensions=(600,600)), False, "", True, '100%', 525, False) # html_bar_chart = html_start + plot_html + html_end # f = open('successful_logoffs.html', 'w') # f.write(html_bar_chart) # f.close() ########################## # Failed logon attempts x = [] y = [] for item in failed_logon_attempts: x.append(item[0]) y.append(item[1]) if len(x) >= 1: df_failed_logon_attempts = pd.DataFrame(y, x) plot_html, plotdivid, width, height = _plot_html(df_failed_logon_attempts.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Failed Login Attempts',dimensions=(600,600)), False, "", True, '100%', 525, False) html_bar_chart = html_start + plot_html + html_end f = open('failed_logon_attempts.html', 'w') f.write(html_bar_chart) f.close() else: print('No failed login attempts') ########################## # logon attempts x = [] y = [] for item in logon_attempts: x.append(item[0]) y.append(item[1]) df_logon_attempts = pd.DataFrame(y, x) plot_html, plotdivid, width, height = _plot_html(df_logon_attempts.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='Login Attempts',dimensions=(600,600)), False, "", True, '100%', 525, False) html_bar_chart = html_start + plot_html + html_end f = open('logon_attempts.html', 'w') f.write(html_bar_chart) f.close() ########################## # All Events x = [] y = [] for item in all_events: x.append(item[0]) y.append(item[1]) df_all_events = pd.DataFrame(y, x) plot_html, plotdivid, width, height = _plot_html(df_all_events.iplot(asFigure=True, kind ='bar', subplots=True, shared_xaxes=True, fill=False, title='All Events',dimensions=(600,600)), False, "", True, '100%', 525, False) html_bar_chart = html_start + plot_html + html_end f = open('all_events.html', 'w') f.write(html_bar_chart) f.close()
def get_context_data(self, *args, **kwargs): #for now, displays random data, hook this up to experiment data later context = super(LineChartAnimatedDataView, self).get_context_data(**kwargs) xMax = 50 N = 100 yMax = 5 margin = 1.5 x = np.linspace(0, xMax, N) #should be a context argument y = np.random.rand(N) * yMax #should be a context argument #display code data = [ dict(x=x, y=y, mode='lines', line=dict(width=2, color=CHART_LINE_COLORS[0])), ] frames = [ dict(data=[ dict(x=x[0:k + 1], y=y[0:k + 1], mode='lines', line=dict(color=CHART_LINE_COLORS[0], width=2)) ]) for k in range(N) ] layout = dict( plot_bgcolor='transparent', paper_bgcolor='transparent', showlegend=False, font=dict(family='"Lato","Open Sans",Helvetica,sans-serif', color='#ffffff'), xaxis=dict(range=[np.min(x) - margin, np.max(x) + margin], autorange=False, color='#ddd'), yaxis=dict(range=[np.min(y) - margin, np.max(y) + margin], autorange=False, color='#ddd'), title='Random Data Test 1', #should be a context argument titlefont=dict(family='"Lato","Open Sans",Helvetica,sans-serif', color='#ffffff'), hovermode='closest', updatemenus=[{ 'buttons': [{ 'args': [[frames[-1]], { 'frame': { 'duration': 0, 'redraw': False }, 'mode': 'immediate', 'transition': { 'duration': 0 } }], 'label': 'Skip to End', 'method': 'animate' }], 'font': dict(family='"Lato","Open Sans",Helvetica,sans-serif', color='#ffffff'), 'bordercolor': '#ffffff', 'direction': 'left', 'pad': { 'l': 10 }, 'showactive': False, 'type': 'buttons', 'x': 0.025, 'xanchor': 'left', 'y': 1, 'yanchor': 'top' }]) figure = dict(data=data, layout=layout, frames=frames) plot_html = plot_html, plot_div_id, width, height = _plot_html( figure, {'show_link': False}, True, '100%', '100%', False) resize_script = '' if width == '100%' or height == '100%': resize_script = ( '' '<script type="text/javascript">' 'window.removeEventListener("resize" );' 'window.addEventListener("resize", function(){{' 'Plotly.Plots.resize(document.getElementById("{id}"));}});' '</script>').format(id=plot_div_id) context['html_str'] = ''.join([plot_html, resize_script]) return context
def create_incidence_chart( cls, df: pd.DataFrame, year_week: int, threshold_pre_epidemic: float, threshold_pos_epidemic: float, threshold_epidemic: float ) -> 'Plotly_HTML': """ @see: https://stackoverflow.com/questions/45526734/ hide-legend-entries-in-a-plotly-figure :param df: :param year_week: :param threshold_pre_epidemic: float, :param threshold_pos_epidemic: float :param threshold_epidemic: float :return: """ df = df.reset_index()[[ 'SE', 'incidência', 'casos notif.', 'level_code' ]] # 200 = 2 years df = df[ df.SE >= year_week - 200 ] df['SE'] = df.SE.map(lambda v: '%s/%s' % (str(v)[:4], str(v)[-2:])) k = 'incidência' df['alerta verde'] = df[df.level_code == 1][k] df['alerta amarelo'] = df[df.level_code == 2][k] df['alerta laranja'] = df[df.level_code == 3][k] df['alerta vermelho'] = df[df.level_code == 4][k] df['limiar epidêmico'] = threshold_epidemic df['limiar pós epidêmico'] = threshold_pos_epidemic df['limiar pré epidêmico'] = threshold_pre_epidemic figure_bar = df.iplot( asFigure=True, kind='bar', x=['SE'], y=[ 'alerta verde', 'alerta amarelo', 'alerta laranja', 'alerta vermelho' ], legend=True, showlegend=True, yTitle='Incidência', xTitle='Período (Ano/Semana)', color=[ 'rgb(0,255,0)', 'rgb(255,255,0)', 'rgb(255,150,0)', 'rgb(255,0,0)' ], hoverinfo='x+y+name' ) figure_threshold = df.iplot( asFigure=True, x=['SE'], y=[ 'limiar pré epidêmico', 'limiar pós epidêmico', 'limiar epidêmico' ], legend=False, showlegend=False, color=[ 'rgb(0,255,0)', 'rgb(255,150,0)', 'rgb(255,0,0)' ], hoverinfo='none' ) figure_line = df.iplot( asFigure=True, x=['SE'], y=['casos notif.'], legend=False, showlegend=False, secondary_y=['casos notif.'], secondary_y_title='Casos', hoverinfo='x+y+name', color=['rgb(33,33,33)'] ) figure_line['layout']['legend'].update( x=-.27, y=0.5, traceorder='normal', font=dict( family='sans-serif', size=12, color='#000' ), bgcolor='#FFFFFF', bordercolor='#E2E2E2', borderwidth=1, orientation='l', ) figure_line['layout']['yaxis1'].update( title='Incidência' ) figure_line['layout'].update( title=( 'Limiares de incidência:: ' + 'pré epidêmico=%s; ' + 'pós epidêmico=%s; ' + 'epidêmico=%s;' ) % ( '{:.1f}'.format(threshold_pre_epidemic), '{:.1f}'.format(threshold_pos_epidemic), '{:.1f}'.format(threshold_epidemic) ), showlegend=True ) figure_threshold.data.extend(figure_bar.data) figure_line.data.extend(figure_threshold.data) figure_line['layout']['yaxis2'].update( showgrid=False, range=[0, df['casos notif.'].max()] ) figure_line['layout']['xaxis1'].update( tickangle=-60, nticks=len(df) // 4, title='Período (Ano/Semana)' ) for trace in figure_line['data']: if trace['name'] == 'casos notif.': trace['visible'] = 'legendonly' return _plot_html( figure_or_data=figure_line, config={}, validate=True, default_width='100%', default_height=500, global_requirejs='' )[0]