class AnalysisApp: def __init__(self, dfs_path=os.path.join('static', 'dfs'), title='Analysis Dashboard', app_name='app', callback_side='backend'): """ callback_side='client' is not implemented for multiple outputs yet in dash """ if DEBUG: print('AnalysisApp init') self.dfs_path = dfs_path self.title = title self.callback_side = callback_side self.app_name = app_name self.template = self.CHART_FONT = self.opacity = None self.get_style() # Create Dahs App self.app = DjangoDash('analysis_app', add_bootstrap_links=True, suppress_callback_exceptions=True) # self.app.css.append_css({'external_url': 'https://codepen.io/amyoshino/pen/jzXypZ.css'}) # external_js = ["https://code.jquery.com/jquery-3.2.1.min.js", "https://codepen.io/bcd/pen/YaXojL.js"] # for js in external_js: self.app.scripts.append_script({"external_url": js}) # Instantiate components empty self.input_components, self.div_charts, self.div_tables, self.function_outputs = [], [], [], [] self.main_inputs, self.inputs_as_output, self.function_inputs, self.parameter_inputs = [], [], [], [] # Read Data self.read_data() if DEBUG: print(' -> Data Read') # Initialize Components self.read_input_configuration() self.create_input_components() self.create_output_components() self.update_inputs() # self.update_output() # Create App Layout self.app.layout = html.Div([ html.H2(self.title), dcc.Dropdown(id='test', options=[{ 'label': 'chart1', 'value': 'chart1' }, { 'label': 'chart2', 'value': 'chart2' }, { 'label': 'San Francisco', 'value': 'SF' }], placeholder="Select a city", multi=True), self.input_components, self.div_charts, self.div_tables, ], className="principal") # self.app.config.suppress_callback_exceptions = True # Cache # self.cache = Cache(self.app.server, config={ # # try 'filesystem' if you don't want to setup redis # 'CACHE_TYPE': 'filesystem', # 'CACHE_DIR': 'cache-directory' # }) # self.cache.memoize(timeout=5)(self.update_output) # Associate callbacks if self.callback_side == 'backend': self.app.callback( inputs=self.main_inputs, output=self.inputs_as_output, )(self.update_inputs) self.app.callback( inputs=self.function_inputs + [Input('correlation_chart', 'selectedData')], output=self.function_outputs, )(self.update_output) self.app.callback(output=[ Output('chart1', component_property='style'), Output('chart2', component_property='style') ], inputs=[Input('test', 'value')])(self.hide_charts) elif self.callback_side == 'client': ip = ','.join(self.parameter_inputs) self.app.clientside_callback( f""" function({ip}) {{ return update_inputs({ip}); }} """, output=self.inputs_as_output, inputs=self.main_inputs, ) # (self.update_inputs) self.app.clientside_callback( f""" function({ip}) {{ return update_output({ip}); }} """, output=self.function_outputs, inputs=self.function_inputs, ) # (self.update_output) if DEBUG: print(' -> Layout & Callbacks Ready') def hide_charts(self, show_charts): print('@@@@@@@@@@@@@@@@') print(show_charts) retorno = [] for chart_name in ['chart1', 'chart2']: if chart_name in show_charts: retorno.append({'display': 'block'}) else: retorno.append({'display': 'none'}) return retorno def get_style(self): conf_path = os.path.join('static', 'analysis_app', 'conf_files', f'{self.app_name}.txt') with open(conf_path) as json_file: json_conf = json.load(json_file, encoding='cp1252') self.charts = json_conf['charts'] self.tables = json_conf['tables'] self.template = json_conf['style']['template'] self.CHART_FONT = json_conf['style']['chart_font'] self.opacity = json_conf['style']['opacity'] if DEBUG: print(' -> Style Ready', self.template) @try_catch def get_csv(self, df_name='test_df.csv', path='static/dfs/'): """ Returns a pandas dataframe from a csv """ self.df = pd.read_csv(os.path.join(path, df_name)) @try_catch def read_data(self, dataframe_name=None): # Read data self.dataframes = [df_name for df_name in os.listdir(self.dfs_path)] if dataframe_name is None: dataframe_name = self.dataframes[0] self.get_csv(dataframe_name) self.columns_str = self.df.select_dtypes(include='object').columns self.columns_numeric = self.df.select_dtypes( include=['float64', 'int']).columns @try_catch def get_component(self, i, v): """ Generates de dcc component based on the attributes passed """ if v['control_type'] == 'dropdown': return html.P( dcc.Dropdown( options=[{ 'label': x, 'value': x } for x in v['data']], value=v['data'][0], className=v['className'], id=f'{i}', persistence=True, persistence_type='local', # local|memory clearable=True, searchable=True, placeholder=f"Select a {i}", disabled=False, )) elif v['control_type'] == 'slider': return html.P( dcc.RangeSlider( id=f'{i}', min=v['data']['min'], max=v['data']['max'], step=v['data']['step'], value=v['data']['value'], ), ) @try_catch def read_input_configuration(self, input_f_path=''): # TODO: read from file # Create the Inputs from Configuration self.inputs_conf = { 'dataframe': { 'property': 'value', 'control_type': 'dropdown', 'data': self.dataframes, 'className': 'col-6', 'main_control': True }, 'categorical': { 'property': 'value', 'control_type': 'dropdown', 'data': self.columns_str, 'className': 'col-6', 'main_control': False }, 'numerical': { 'property': 'value', 'control_type': 'dropdown', 'data': self.columns_numeric, 'className': 'col-6', 'main_control': False }, } if DEBUG: print(' -> Input Configuration Ready') @try_catch def save_input_config(self, input_f_path=''): # TODO: pass @try_catch def create_input_components(self): self.input_components = html.Div(className='row', id='controls_div', children=[]) self.main_inputs, self.inputs_as_output, self.function_inputs = [], [], [ ] # Restart Components self.parameter_inputs = [] for i, v in self.inputs_conf.items(): self.input_components.children.append( html.Div(className='col', children=[self.get_component(i, v)]), ) if v['main_control']: self.main_inputs.append(Input(i, v['property'])) else: self.parameter_inputs.append(i) self.parameter_inputs.append(f'{i}_selected_data') self.function_inputs.append(Input(i, v['property'])) # self.function_inputs.append(Input(i, 'selectedData')) self.inputs_as_output.append(Output(i, 'options')) self.inputs_as_output.append(Output(i, v['property'])) if DEBUG: print(' -> Input Components Ready') @try_catch def create_output_components(self): self.function_outputs = [] self.div_charts = html.Div(className='row', id='charts_div', children=[]) new_row = "row" for chart_name in self.charts: self.div_charts.children.append( html.Div([ html.Div([ dcc.Graph(id=chart_name, style={}), ], className="col card", id=f'{chart_name}_div'), ], className=f"{new_row}")) self.function_outputs.append(Output(chart_name, 'figure')) self.div_tables = html.Div(className='row', id='tables_div', children=[]) for table_name in self.tables: self.div_tables.children.append( html.Div(id=f'{table_name}', className='col card')) self.function_outputs.append(Output(table_name, 'children')) if DEBUG: print(' -> Output Components Ready') @try_catch def update_inputs(self, dataframe_name='test_df.csv'): # self.parameter_inputs """ Update the values of the Input Controls """ self.read_data(dataframe_name=dataframe_name) return [{'label': col, 'value': col} for col in self.columns_str], self.columns_str[0], \ [{'label': col, 'value': col} for col in self.columns_numeric], self.columns_numeric[0] @try_catch def get_boxplot(self, categorical_column, variable_column, df): try: names = self.df[categorical_column].unique() data = [ go.Box( # marker=dict(color=COLORS[provider], opacity=self.opacity, name=name, x=df[df[categorical_column] == name][categorical_column], y=df[df[categorical_column] == name][variable_column]) for name in names ] layout = { 'legend_orientation': 'h', 'title': go.layout.Title(text=f"Distribution of {variable_column}", ), 'template': self.template } except Exception as e: print(str(e)) data, layout = [], {} return go.Figure(data=data, layout=layout) @try_catch def get_histogram(self, categorical_column, variable_column, df, x_name='sepal_length', y_name='petal_length'): '''try: names = df[categorical_column].unique() data = [ dict( type='scatter', mode='markers', x=name_df[x_name], y=name_df[y_name], name=name, ) for name_df, name in [(df[df[categorical_column] == name], name) for name in names] ] layout = { 'title': go.layout.Title(text=f"{y_name} vs {x_name}", font=CHART_FONT), 'xaxis': go.layout.XAxis( title=go.layout.xaxis.Title(text=x_name, font=CHART_FONT)), 'yaxis': go.layout.YAxis( title=go.layout.yaxis.Title(text=y_name, font=CHART_FONT)), 'template': template, } except: data, layout = [], {} return go.Figure(data=data, layout=layout)''' return px.histogram( df, x=variable_column, y=variable_column, color=categorical_column, marginal="box", # or violin, rug hover_data=self.df.columns, template=self.template) @try_catch def get_null_map(self, df, columns_numeric): layout = { 'legend_orientation': 'h', 'title': go.layout.Title(text=f"Null Distribution", ), 'template': self.template } return go.Figure(data=go.Heatmap( z=self.df[columns_numeric].isnull().astype(int).to_numpy()), layout=layout) @try_catch def generate_correlation_chart(self, categorical_column, **kwargs): index_vals = self.df[categorical_column].astype('category').cat.codes fig = go.Figure(data=go.Splom( dimensions=[ dict(label=c, values=self.df[c]) for c in self.columns_numeric ], text=self.df[categorical_column], marker=dict( color=index_vals, showscale=False, # colors encode categorical variables line_color='white', line_width=0.5))) fig.update_layout(title='Correlations', template=self.template) # ,width=600, height=600, return fig @try_catch def generate_outlayers(self, categorical_column): # def get_outlayers(self, param='Total Unsuccessful', date=None, dataframe=None): D = [] for category in self.df[categorical_column].unique(): specific_df = self.df[self.df[categorical_column] == category] for feature in self.columns_numeric: specific_df['measuring'] = specific_df[feature] qv1 = specific_df[feature].quantile(0.25) qv3 = specific_df[feature].quantile(0.75) qv_limit = 1.5 * (qv3 - qv1) un_outliers_mask = (specific_df[feature] > qv3 + qv_limit) | ( specific_df[feature] < qv1 - qv_limit) un_outliers_data = specific_df[feature][un_outliers_mask] un_outliers_name = specific_df[un_outliers_mask] if un_outliers_data.shape[0] > 0: for i in [{ 'feature': feature, 'category': category, 'value': val } for val in un_outliers_data]: D.append(i) return pd.DataFrame(D) @try_catch # categorical_column, variable_column #### def update_output(self, categorical_column, variable_column, selected_data): if selected_data is None: self.selected_df = self.df else: selected_points = [ p['pointNumber'] for p in selected_data['points'] ] self.selected_df = self.df[self.df.index.isin(selected_points)] print(f'LENGTH OF DATAFRAME: ') print(self.selected_df.shape) outlayers = self.generate_outlayers(categorical_column) kwargs = { 'categorical_column': categorical_column, 'variable_column': variable_column, 'df': self.selected_df } output_generators = OrderedDict({ 'chart1': { 'function': self.get_boxplot, 'kwargs': kwargs }, 'chart2': { 'function': self.get_histogram, 'kwargs': kwargs }, 'correlation_chart': { 'function': self.generate_correlation_chart, 'kwargs': kwargs }, 'get_null_map': { 'function': self.get_null_map, 'kwargs': { 'df': self.df, 'columns_numeric': self.columns_numeric } }, 'table1': { 'function': generate_table_simple, 'kwargs': { 'dataframe': self.df.describe().round(1) } }, 'outlayers_table': { 'function': generate_table_simple, 'kwargs': { 'dataframe': outlayers } }, 'full_Table': { 'function': generate_table_simple, 'kwargs': { 'dataframe': self.df } }, }) return [ values['function'](**values['kwargs']) for out_name, values in output_generators.items() if out_name in self.charts + self.tables ] @try_catch def get_app(self): return self.app
class JupyterDash: def __init__(self, name, gav=None, width=800, height=600, add_bootstrap_links=False, serve_locally=None): self.dd = DjangoDash(name, serve_locally=serve_locally, add_bootstrap_links=add_bootstrap_links) self.gav = gav and gav or get_global_av() self.gav.add_application(self, name) self.width = width self.height = height self.frame = False self.add_external_link = True self.session_state = dict() self.app_state = dict() self.local_uuid = str(uuid.uuid4()).replace('-', '') self.use_nbproxy = False def as_dash_instance(self, specific_identifier=None, base_pathname=None): if base_pathname is None: base_pathname = self.get_base_pathname(specific_identifier) specific_identifier = self.session_id() else: if base_pathname[0] != '/': base_pathname = "/%s" % base_pathname if base_pathname[-1] != '/': base_pathname = "%s/" % base_pathname # TODO perhaps cache this. If so, need to ensure updated if self.app_state changes return self.dd.form_dash_instance(replacements=self.app_state, ndid=specific_identifier, base_pathname=base_pathname) def get_session_state(self): return self.session_state def set_session_state(self, state): self.session_state = state def handle_current_state(self): # Do nothing, at least for the moment... pass def update_current_state(self, wid, name, value): wd = self.app_state.get(wid, None) if wd is None: wd = dict() self.app_state[wid] = wd wd[name] = value def have_current_state_entry(self, wid, name): wd = self.app_state.get(wid, {}) entry = wd.get(name, None) return entry is not None def get_base_pathname(self, specific_identifier): the_id = specific_identifier and specific_identifier or self.session_id( ) if self.use_nbproxy: # TODO this should be the_id not the uid? return '/proxy/%i/%s/' % (self.gav.port, self.dd._uid) return "/app/endpoints/%s/" % the_id def session_id(self): return self.local_uuid def get_app_root_url(self): # Local (not binder use) determined by presence of server prefix jh_serv_pref = os.environ.get('JUPYTERHUB_SERVICE_PREFIX', None) if jh_serv_pref is None: # Local use, root is given by proxy flag return self.get_base_pathname(self.session_id()) # Running on a binder or similar # TODO restrict use of use_nbproxy here return "%s%s" % (jh_serv_pref, self.get_base_pathname( self.session_id())[1:]) def __html__(self): return self._repr_html_() def _repr_html_(self): url = self.get_app_root_url() da_id = self.session_id() comm = locate_jpd_comm(da_id, self, url[1:-1]) external = self.add_external_link and '<hr/><a href="{url}" target="_new">Open in new window</a> for {url}'.format( url=url) or "" fb = 'frameborder="%i"' % (self.frame and 1 or 0) iframe = '''<div> <iframe src="%(url)s" width=%(width)s height=%(height)s %(frame)s></iframe> %(external)s </div>''' % { 'url': url, 'da_id': da_id, 'external': external, 'width': self.width, 'height': self.height, 'frame': fb, } return iframe def callback(self, *args, **kwargs): return self.dd.callback(*args, **kwargs) def expanded_callback(self, *args, **kwargs): return self.dd.expanded_callback(*args, **kwargs) def _get_layout(self): return self.dd.layout def _set_layout(self, layout): self.dd.layout = layout layout = property(_get_layout, _set_layout) def process_view(self, view_name, args, app_path): if view_name == None: view_name = '' view_name_parts = view_name.split('/') view_name = view_name_parts[0] view_name = view_name.replace('-', '_') func = getattr(self, 'rv_%s' % view_name, None) if func is not None: # TODO process app_path if needed resp, rmt = func(args, app_path, view_name_parts) return (resp, rmt) return ( "<html><body>Unable to understand view name of %s with args %s and app path %s</body></html>" % (view_name, args, app_path), "text/html") def rv_(self, args, app_path, view_name_parts): mFunc = self.as_dash_instance( base_pathname=app_path).locate_endpoint_function() response = mFunc() return (response, "text/html") def rv__dash_layout(self, args, app_path, view_name_parts): dapp = self.as_dash_instance(base_pathname=app_path) with dapp.app_context(): mFunc = dapp.locate_endpoint_function('dash-layout') resp = mFunc() body, mimetype = dapp.augment_initial_layout(resp) return (body, mimetype) def rv__dash_dependencies(self, args, app_path, view_name_parts): dapp = self.as_dash_instance(base_pathname=app_path) with dapp.app_context(): mFunc = dapp.locate_endpoint_function('dash-dependencies') resp = mFunc() return (resp.data.decode('utf-8'), resp.mimetype) def rv__dash_update_component(self, args, app_path, view_name_parts): dapp = self.as_dash_instance(base_pathname=app_path) if dapp.use_dash_dispatch(): mFunc = dapp.locate_endpoint_function('dash-update-component') import flask with dapp.test_request_context(): flask.request._cached_json = (args, flask.request._cached_json[True]) resp = mFunc() else: # Use direct dispatch with extra arguments in the argMap app_state = self.get_session_state() app_state['call_count'] = app_state.get('call_count', 0) + 1 argMap = {} argMap = { 'dash_app_id': self.local_uuid, 'dash_app': self, 'user': None, 'session_state': app_state } resp = dapp.dispatch_with_args(args, argMap) self.set_session_state(app_state) self.handle_current_state() try: rdata = resp.data rtype = resp.mimetype except: rdata = resp rtype = "application/json" return (rdata, rtype) def rv__dash_component_suites(self, args, app_path, view_name_parts): dapp = self.as_dash_instance(base_pathname=app_path) with dapp.app_context(): # Force recalc of dependencies in case the instance is a fresh one dapp.index() # Endpoint is dash-component-suites/package_name/path_in_package try: mFunc = dapp.locate_endpoint_function( 'dash-component-suites/<string:package_name>/<path:path_in_package_dist>' ) except Exception as e: return ( "<html><body>Requested %s at %s with %s and failed with %s</body></html>" % (args, app_path, view_name_parts, e), "text/html") # Need two arguments here: package_name and path_in_package_dist package_name = view_name_parts[1] path_in_package_dist = "/".join(view_name_parts[2:]) resp = mFunc(package_name=package_name, path_in_package_dist=path_in_package_dist) return (resp.data.decode('utf-8'), resp.mimetype)
class JupyterDash: def __init__(self, name, gav=None, width=800, height=600): self.dd = DjangoDash(name) self.gav = gav and gav or get_global_av() self.gav.add_app(self, name) self.width = width self.height = height self.add_external_link = True self.session_state = dict() self.app_state = dict() def as_dash_instance(self): # TODO perhaps cache this. If so, need to ensure updated if self.app_state changes return self.dd.form_dash_instance(replacements=self.app_state) def get_session_state(self): return self.session_state def set_session_state(self, state): self.session_state = state def handle_current_state(self): # Do nothing, at least for the moment... pass def update_current_state(self, wid, name, value): wd = self.app_state.get(wid,None) if wd is None: wd = dict() self.app_state[wid] = wd wd[name] = value def have_current_state_entry(self, wid, name): wd = self.app_state.get(wid,{}) entry = wd.get(name,None) return entry is not None def get_base_pathname(self, specific_identifier): return '/%s/' % specific_identifier def get_app_root_url(self): return 'http://localhost:%i%s' % (self.gav.port, self.get_base_pathname(self.dd._uid)) def _repr_html_(self): url = self.get_app_root_url() external = self.add_external_link and '<hr/><a href="{url}" target="_new">Open in new window</a>'.format(url=url) or "" iframe = '''<div> <iframe src="{url}" width={width} height={height}></iframe> {external} </div>'''.format(url = url, external = external, width = self.width, height = self.height) return iframe def callback(self, *args, **kwargs): return self.dd.callback(*args,**kwargs) def expanded_callback(self, *args, **kwargs): return self.dd.expanded_callback(*args,**kwargs) def _get_layout(self): return self.dd.layout def _set_layout(self, layout): self.dd.layout = layout layout = property(_get_layout, _set_layout)