class AnalysisApp:
def __init__(self,
dfs_path=os.path.join('static', 'dfs'),
title='Analysis Dashboard',
app_name='app',
callback_side='backend'):
"""
callback_side='client' is not implemented for multiple outputs yet in dash
"""
if DEBUG: print('AnalysisApp init')
self.dfs_path = dfs_path
self.title = title
self.callback_side = callback_side
self.app_name = app_name
self.template = self.CHART_FONT = self.opacity = None
self.get_style()
# Create Dahs App
self.app = DjangoDash('analysis_app',
add_bootstrap_links=True,
suppress_callback_exceptions=True)
# self.app.css.append_css({'external_url': 'https://codepen.io/amyoshino/pen/jzXypZ.css'})
# external_js = ["https://code.jquery.com/jquery-3.2.1.min.js", "https://codepen.io/bcd/pen/YaXojL.js"]
# for js in external_js: self.app.scripts.append_script({"external_url": js})
# Instantiate components empty
self.input_components, self.div_charts, self.div_tables, self.function_outputs = [], [], [], []
self.main_inputs, self.inputs_as_output, self.function_inputs, self.parameter_inputs = [], [], [], []
# Read Data
self.read_data()
if DEBUG: print(' -> Data Read')
# Initialize Components
self.read_input_configuration()
self.create_input_components()
self.create_output_components()
self.update_inputs()
# self.update_output()
# Create App Layout
self.app.layout = html.Div([
html.H2(self.title),
dcc.Dropdown(id='test',
options=[{
'label': 'chart1',
'value': 'chart1'
}, {
'label': 'chart2',
'value': 'chart2'
}, {
'label': 'San Francisco',
'value': 'SF'
}],
placeholder="Select a city",
multi=True),
self.input_components,
self.div_charts,
self.div_tables,
],
className="principal")
# self.app.config.suppress_callback_exceptions = True
# Cache
# self.cache = Cache(self.app.server, config={
# # try 'filesystem' if you don't want to setup redis
# 'CACHE_TYPE': 'filesystem',
# 'CACHE_DIR': 'cache-directory'
# })
# self.cache.memoize(timeout=5)(self.update_output)
# Associate callbacks
if self.callback_side == 'backend':
self.app.callback(
inputs=self.main_inputs,
output=self.inputs_as_output,
)(self.update_inputs)
self.app.callback(
inputs=self.function_inputs +
[Input('correlation_chart', 'selectedData')],
output=self.function_outputs,
)(self.update_output)
self.app.callback(output=[
Output('chart1', component_property='style'),
Output('chart2', component_property='style')
],
inputs=[Input('test',
'value')])(self.hide_charts)
elif self.callback_side == 'client':
ip = ','.join(self.parameter_inputs)
self.app.clientside_callback(
f"""
function({ip}) {{
return update_inputs({ip});
}}
""",
output=self.inputs_as_output,
inputs=self.main_inputs,
) # (self.update_inputs)
self.app.clientside_callback(
f"""
function({ip}) {{
return update_output({ip});
}}
""",
output=self.function_outputs,
inputs=self.function_inputs,
) # (self.update_output)
if DEBUG: print(' -> Layout & Callbacks Ready')
def hide_charts(self, show_charts):
print('@@@@@@@@@@@@@@@@')
print(show_charts)
retorno = []
for chart_name in ['chart1', 'chart2']:
if chart_name in show_charts:
retorno.append({'display': 'block'})
else:
retorno.append({'display': 'none'})
return retorno
def get_style(self):
conf_path = os.path.join('static', 'analysis_app', 'conf_files',
f'{self.app_name}.txt')
with open(conf_path) as json_file:
json_conf = json.load(json_file, encoding='cp1252')
self.charts = json_conf['charts']
self.tables = json_conf['tables']
self.template = json_conf['style']['template']
self.CHART_FONT = json_conf['style']['chart_font']
self.opacity = json_conf['style']['opacity']
if DEBUG: print(' -> Style Ready', self.template)
@try_catch
def get_csv(self, df_name='test_df.csv', path='static/dfs/'):
"""
Returns a pandas dataframe from a csv
"""
self.df = pd.read_csv(os.path.join(path, df_name))
@try_catch
def read_data(self, dataframe_name=None):
# Read data
self.dataframes = [df_name for df_name in os.listdir(self.dfs_path)]
if dataframe_name is None: dataframe_name = self.dataframes[0]
self.get_csv(dataframe_name)
self.columns_str = self.df.select_dtypes(include='object').columns
self.columns_numeric = self.df.select_dtypes(
include=['float64', 'int']).columns
@try_catch
def get_component(self, i, v):
"""
Generates de dcc component based on the attributes passed
"""
if v['control_type'] == 'dropdown':
return html.P(
dcc.Dropdown(
options=[{
'label': x,
'value': x
} for x in v['data']],
value=v['data'][0],
className=v['className'],
id=f'{i}',
persistence=True,
persistence_type='local', # local|memory
clearable=True,
searchable=True,
placeholder=f"Select a {i}",
disabled=False,
))
elif v['control_type'] == 'slider':
return html.P(
dcc.RangeSlider(
id=f'{i}',
min=v['data']['min'],
max=v['data']['max'],
step=v['data']['step'],
value=v['data']['value'],
), )
@try_catch
def read_input_configuration(self,
input_f_path=''): # TODO: read from file
# Create the Inputs from Configuration
self.inputs_conf = {
'dataframe': {
'property': 'value',
'control_type': 'dropdown',
'data': self.dataframes,
'className': 'col-6',
'main_control': True
},
'categorical': {
'property': 'value',
'control_type': 'dropdown',
'data': self.columns_str,
'className': 'col-6',
'main_control': False
},
'numerical': {
'property': 'value',
'control_type': 'dropdown',
'data': self.columns_numeric,
'className': 'col-6',
'main_control': False
},
}
if DEBUG: print(' -> Input Configuration Ready')
@try_catch
def save_input_config(self, input_f_path=''): # TODO:
pass
@try_catch
def create_input_components(self):
self.input_components = html.Div(className='row',
id='controls_div',
children=[])
self.main_inputs, self.inputs_as_output, self.function_inputs = [], [], [
] # Restart Components
self.parameter_inputs = []
for i, v in self.inputs_conf.items():
self.input_components.children.append(
html.Div(className='col', children=[self.get_component(i,
v)]), )
if v['main_control']:
self.main_inputs.append(Input(i, v['property']))
else:
self.parameter_inputs.append(i)
self.parameter_inputs.append(f'{i}_selected_data')
self.function_inputs.append(Input(i, v['property']))
# self.function_inputs.append(Input(i, 'selectedData'))
self.inputs_as_output.append(Output(i, 'options'))
self.inputs_as_output.append(Output(i, v['property']))
if DEBUG: print(' -> Input Components Ready')
@try_catch
def create_output_components(self):
self.function_outputs = []
self.div_charts = html.Div(className='row',
id='charts_div',
children=[])
new_row = "row"
for chart_name in self.charts:
self.div_charts.children.append(
html.Div([
html.Div([
dcc.Graph(id=chart_name, style={}),
],
className="col card",
id=f'{chart_name}_div'),
],
className=f"{new_row}"))
self.function_outputs.append(Output(chart_name, 'figure'))
self.div_tables = html.Div(className='row',
id='tables_div',
children=[])
for table_name in self.tables:
self.div_tables.children.append(
html.Div(id=f'{table_name}', className='col card'))
self.function_outputs.append(Output(table_name, 'children'))
if DEBUG: print(' -> Output Components Ready')
@try_catch
def update_inputs(self,
dataframe_name='test_df.csv'): # self.parameter_inputs
""" Update the values of the Input Controls """
self.read_data(dataframe_name=dataframe_name)
return [{'label': col, 'value': col} for col in self.columns_str], self.columns_str[0], \
[{'label': col, 'value': col} for col in self.columns_numeric], self.columns_numeric[0]
@try_catch
def get_boxplot(self, categorical_column, variable_column, df):
try:
names = self.df[categorical_column].unique()
data = [
go.Box( # marker=dict(color=COLORS[provider],
opacity=self.opacity,
name=name,
x=df[df[categorical_column] == name][categorical_column],
y=df[df[categorical_column] == name][variable_column])
for name in names
]
layout = {
'legend_orientation':
'h',
'title':
go.layout.Title(text=f"Distribution of {variable_column}", ),
'template':
self.template
}
except Exception as e:
print(str(e))
data, layout = [], {}
return go.Figure(data=data, layout=layout)
@try_catch
def get_histogram(self,
categorical_column,
variable_column,
df,
x_name='sepal_length',
y_name='petal_length'):
'''try:
names = df[categorical_column].unique()
data = [
dict(
type='scatter',
mode='markers',
x=name_df[x_name],
y=name_df[y_name],
name=name,
) for name_df, name in [(df[df[categorical_column] == name], name) for name in names]
]
layout = {
'title': go.layout.Title(text=f"{y_name} vs {x_name}", font=CHART_FONT),
'xaxis': go.layout.XAxis(
title=go.layout.xaxis.Title(text=x_name, font=CHART_FONT)),
'yaxis': go.layout.YAxis(
title=go.layout.yaxis.Title(text=y_name, font=CHART_FONT)),
'template': template,
}
except: data, layout = [], {}
return go.Figure(data=data, layout=layout)'''
return px.histogram(
df,
x=variable_column,
y=variable_column,
color=categorical_column,
marginal="box",
# or violin, rug
hover_data=self.df.columns,
template=self.template)
@try_catch
def get_null_map(self, df, columns_numeric):
layout = {
'legend_orientation': 'h',
'title': go.layout.Title(text=f"Null Distribution", ),
'template': self.template
}
return go.Figure(data=go.Heatmap(
z=self.df[columns_numeric].isnull().astype(int).to_numpy()),
layout=layout)
@try_catch
def generate_correlation_chart(self, categorical_column, **kwargs):
index_vals = self.df[categorical_column].astype('category').cat.codes
fig = go.Figure(data=go.Splom(
dimensions=[
dict(label=c, values=self.df[c]) for c in self.columns_numeric
],
text=self.df[categorical_column],
marker=dict(
color=index_vals,
showscale=False, # colors encode categorical variables
line_color='white',
line_width=0.5)))
fig.update_layout(title='Correlations',
template=self.template) # ,width=600, height=600,
return fig
@try_catch
def generate_outlayers(self, categorical_column):
# def get_outlayers(self, param='Total Unsuccessful', date=None, dataframe=None):
D = []
for category in self.df[categorical_column].unique():
specific_df = self.df[self.df[categorical_column] == category]
for feature in self.columns_numeric:
specific_df['measuring'] = specific_df[feature]
qv1 = specific_df[feature].quantile(0.25)
qv3 = specific_df[feature].quantile(0.75)
qv_limit = 1.5 * (qv3 - qv1)
un_outliers_mask = (specific_df[feature] > qv3 + qv_limit) | (
specific_df[feature] < qv1 - qv_limit)
un_outliers_data = specific_df[feature][un_outliers_mask]
un_outliers_name = specific_df[un_outliers_mask]
if un_outliers_data.shape[0] > 0:
for i in [{
'feature': feature,
'category': category,
'value': val
} for val in un_outliers_data]:
D.append(i)
return pd.DataFrame(D)
@try_catch # categorical_column, variable_column ####
def update_output(self, categorical_column, variable_column,
selected_data):
if selected_data is None:
self.selected_df = self.df
else:
selected_points = [
p['pointNumber'] for p in selected_data['points']
]
self.selected_df = self.df[self.df.index.isin(selected_points)]
print(f'LENGTH OF DATAFRAME: ')
print(self.selected_df.shape)
outlayers = self.generate_outlayers(categorical_column)
kwargs = {
'categorical_column': categorical_column,
'variable_column': variable_column,
'df': self.selected_df
}
output_generators = OrderedDict({
'chart1': {
'function': self.get_boxplot,
'kwargs': kwargs
},
'chart2': {
'function': self.get_histogram,
'kwargs': kwargs
},
'correlation_chart': {
'function': self.generate_correlation_chart,
'kwargs': kwargs
},
'get_null_map': {
'function': self.get_null_map,
'kwargs': {
'df': self.df,
'columns_numeric': self.columns_numeric
}
},
'table1': {
'function': generate_table_simple,
'kwargs': {
'dataframe': self.df.describe().round(1)
}
},
'outlayers_table': {
'function': generate_table_simple,
'kwargs': {
'dataframe': outlayers
}
},
'full_Table': {
'function': generate_table_simple,
'kwargs': {
'dataframe': self.df
}
},
})
return [
values['function'](**values['kwargs'])
for out_name, values in output_generators.items()
if out_name in self.charts + self.tables
]
@try_catch
def get_app(self):
return self.app
class JupyterDash:
def __init__(self,
name,
gav=None,
width=800,
height=600,
add_bootstrap_links=False,
serve_locally=None):
self.dd = DjangoDash(name,
serve_locally=serve_locally,
add_bootstrap_links=add_bootstrap_links)
self.gav = gav and gav or get_global_av()
self.gav.add_application(self, name)
self.width = width
self.height = height
self.frame = False
self.add_external_link = True
self.session_state = dict()
self.app_state = dict()
self.local_uuid = str(uuid.uuid4()).replace('-', '')
self.use_nbproxy = False
def as_dash_instance(self, specific_identifier=None, base_pathname=None):
if base_pathname is None:
base_pathname = self.get_base_pathname(specific_identifier)
specific_identifier = self.session_id()
else:
if base_pathname[0] != '/':
base_pathname = "/%s" % base_pathname
if base_pathname[-1] != '/':
base_pathname = "%s/" % base_pathname
# TODO perhaps cache this. If so, need to ensure updated if self.app_state changes
return self.dd.form_dash_instance(replacements=self.app_state,
ndid=specific_identifier,
base_pathname=base_pathname)
def get_session_state(self):
return self.session_state
def set_session_state(self, state):
self.session_state = state
def handle_current_state(self):
# Do nothing, at least for the moment...
pass
def update_current_state(self, wid, name, value):
wd = self.app_state.get(wid, None)
if wd is None:
wd = dict()
self.app_state[wid] = wd
wd[name] = value
def have_current_state_entry(self, wid, name):
wd = self.app_state.get(wid, {})
entry = wd.get(name, None)
return entry is not None
def get_base_pathname(self, specific_identifier):
the_id = specific_identifier and specific_identifier or self.session_id(
)
if self.use_nbproxy:
# TODO this should be the_id not the uid?
return '/proxy/%i/%s/' % (self.gav.port, self.dd._uid)
return "/app/endpoints/%s/" % the_id
def session_id(self):
return self.local_uuid
def get_app_root_url(self):
# Local (not binder use) determined by presence of server prefix
jh_serv_pref = os.environ.get('JUPYTERHUB_SERVICE_PREFIX', None)
if jh_serv_pref is None:
# Local use, root is given by proxy flag
return self.get_base_pathname(self.session_id())
# Running on a binder or similar
# TODO restrict use of use_nbproxy here
return "%s%s" % (jh_serv_pref, self.get_base_pathname(
self.session_id())[1:])
def __html__(self):
return self._repr_html_()
def _repr_html_(self):
url = self.get_app_root_url()
da_id = self.session_id()
comm = locate_jpd_comm(da_id, self, url[1:-1])
external = self.add_external_link and '<hr/><a href="{url}" target="_new">Open in new window</a> for {url}'.format(
url=url) or ""
fb = 'frameborder="%i"' % (self.frame and 1 or 0)
iframe = '''<div>
<iframe src="%(url)s" width=%(width)s height=%(height)s %(frame)s></iframe>
%(external)s
</div>''' % {
'url': url,
'da_id': da_id,
'external': external,
'width': self.width,
'height': self.height,
'frame': fb,
}
return iframe
def callback(self, *args, **kwargs):
return self.dd.callback(*args, **kwargs)
def expanded_callback(self, *args, **kwargs):
return self.dd.expanded_callback(*args, **kwargs)
def _get_layout(self):
return self.dd.layout
def _set_layout(self, layout):
self.dd.layout = layout
layout = property(_get_layout, _set_layout)
def process_view(self, view_name, args, app_path):
if view_name == None:
view_name = ''
view_name_parts = view_name.split('/')
view_name = view_name_parts[0]
view_name = view_name.replace('-', '_')
func = getattr(self, 'rv_%s' % view_name, None)
if func is not None:
# TODO process app_path if needed
resp, rmt = func(args, app_path, view_name_parts)
return (resp, rmt)
return (
"<html><body>Unable to understand view name of %s with args %s and app path %s</body></html>"
% (view_name, args, app_path), "text/html")
def rv_(self, args, app_path, view_name_parts):
mFunc = self.as_dash_instance(
base_pathname=app_path).locate_endpoint_function()
response = mFunc()
return (response, "text/html")
def rv__dash_layout(self, args, app_path, view_name_parts):
dapp = self.as_dash_instance(base_pathname=app_path)
with dapp.app_context():
mFunc = dapp.locate_endpoint_function('dash-layout')
resp = mFunc()
body, mimetype = dapp.augment_initial_layout(resp)
return (body, mimetype)
def rv__dash_dependencies(self, args, app_path, view_name_parts):
dapp = self.as_dash_instance(base_pathname=app_path)
with dapp.app_context():
mFunc = dapp.locate_endpoint_function('dash-dependencies')
resp = mFunc()
return (resp.data.decode('utf-8'), resp.mimetype)
def rv__dash_update_component(self, args, app_path, view_name_parts):
dapp = self.as_dash_instance(base_pathname=app_path)
if dapp.use_dash_dispatch():
mFunc = dapp.locate_endpoint_function('dash-update-component')
import flask
with dapp.test_request_context():
flask.request._cached_json = (args,
flask.request._cached_json[True])
resp = mFunc()
else:
# Use direct dispatch with extra arguments in the argMap
app_state = self.get_session_state()
app_state['call_count'] = app_state.get('call_count', 0) + 1
argMap = {}
argMap = {
'dash_app_id': self.local_uuid,
'dash_app': self,
'user': None,
'session_state': app_state
}
resp = dapp.dispatch_with_args(args, argMap)
self.set_session_state(app_state)
self.handle_current_state()
try:
rdata = resp.data
rtype = resp.mimetype
except:
rdata = resp
rtype = "application/json"
return (rdata, rtype)
def rv__dash_component_suites(self, args, app_path, view_name_parts):
dapp = self.as_dash_instance(base_pathname=app_path)
with dapp.app_context():
# Force recalc of dependencies in case the instance is a fresh one
dapp.index()
# Endpoint is dash-component-suites/package_name/path_in_package
try:
mFunc = dapp.locate_endpoint_function(
'dash-component-suites/<string:package_name>/<path:path_in_package_dist>'
)
except Exception as e:
return (
"<html><body>Requested %s at %s with %s and failed with %s</body></html>"
% (args, app_path, view_name_parts, e), "text/html")
# Need two arguments here: package_name and path_in_package_dist
package_name = view_name_parts[1]
path_in_package_dist = "/".join(view_name_parts[2:])
resp = mFunc(package_name=package_name,
path_in_package_dist=path_in_package_dist)
return (resp.data.decode('utf-8'), resp.mimetype)
class JupyterDash:
def __init__(self, name, gav=None, width=800, height=600):
self.dd = DjangoDash(name)
self.gav = gav and gav or get_global_av()
self.gav.add_app(self, name)
self.width = width
self.height = height
self.add_external_link = True
self.session_state = dict()
self.app_state = dict()
def as_dash_instance(self):
# TODO perhaps cache this. If so, need to ensure updated if self.app_state changes
return self.dd.form_dash_instance(replacements=self.app_state)
def get_session_state(self):
return self.session_state
def set_session_state(self, state):
self.session_state = state
def handle_current_state(self):
# Do nothing, at least for the moment...
pass
def update_current_state(self, wid, name, value):
wd = self.app_state.get(wid,None)
if wd is None:
wd = dict()
self.app_state[wid] = wd
wd[name] = value
def have_current_state_entry(self, wid, name):
wd = self.app_state.get(wid,{})
entry = wd.get(name,None)
return entry is not None
def get_base_pathname(self, specific_identifier):
return '/%s/' % specific_identifier
def get_app_root_url(self):
return 'http://localhost:%i%s' % (self.gav.port, self.get_base_pathname(self.dd._uid))
def _repr_html_(self):
url = self.get_app_root_url()
external = self.add_external_link and '<hr/><a href="{url}" target="_new">Open in new window</a>'.format(url=url) or ""
iframe = '''<div>
<iframe src="{url}" width={width} height={height}></iframe>
{external}
</div>'''.format(url = url,
external = external,
width = self.width,
height = self.height)
return iframe
def callback(self, *args, **kwargs):
return self.dd.callback(*args,**kwargs)
def expanded_callback(self, *args, **kwargs):
return self.dd.expanded_callback(*args,**kwargs)
def _get_layout(self):
return self.dd.layout
def _set_layout(self, layout):
self.dd.layout = layout
layout = property(_get_layout, _set_layout)
