def upload_data(contents, filename):
"""Upload Files and Regenerate the file list."""
if contents:
for i in range(len(filename)):
FileUtils.upload(filename[i], contents[i])
return common.success_msg('File Uploaded Successfully: ' + str(filename))
return ""
def lr_save_model(n_clicks):
model_name = db.get('cl.model_name')
if model_name is None or model_name == '':
return ("", "")
file = db.get("lr.file")
model = db.get("lr.model")
params = db.get("lr.params")
anova = db.get("lr.anova")
error_mean = db.get('lr.error_mean')
summary = db.get("lr.summary")
x_col = db.get("lr.x_col")
y_col = db.get("lr.y_col")
m = {
'file': file,
'tag': db.get('tags')[file],
'type': 'linear',
'name': model_name,
'model': model,
'params': params,
'anova': anova,
'summary': summary,
'x_col': x_col,
'y_col': y_col,
'error_mean': error_mean
}
models = db.get('models')
if models is None:
models = {}
db.put('models', models)
models[model_name] = m
return (common.success_msg('Model "' + model_name +
'" Saved Successfully.'), "")
def hor_save_model(n_clicks):
model_name = db.get('hor.model_name')
if model_name is None or model_name == '':
return ("", "")
file = db.get("hor.file")
model = db.get("hor.model")
order = db.get("hor.order")
params = db.get("hor.params")
anova = db.get("hor.anova")
error_mean = db.get('hor.error_mean')
summary = db.get("hor.summary")
x_col = db.get("hor.x_col")
y_col = db.get("hor.y_col")
m = {
'file': file,
'tag': db.get('tags')[file],
'type': 'Higher Order Polynomial',
'name': model_name,
'model': model,
'order': order,
'params': params,
'no_of_coeff': len(params),
'anova': anova,
'summary': summary,
'x_col': x_col,
'y_col': y_col,
'error_mean': error_mean
}
models = db.get('models')
if models is None:
models = {}
db.put('models', models)
models[model_name] = m
return (common.success_msg('Model "' + model_name +
'" Saved Successfully.'), "")
def ts_do_time_series_analysis(n_clicks):
msg = 'Starting Time Series Analysis ...'
print(msg)
ts_thread_1 = TSThread(n_clicks, "Thread-" + str(n_clicks), 1)
ts_thread_1.start()
msg = 'Time Series Analysis Started.'
print(msg)
return common.success_msg(msg)
def hor_predict_data(n_clicks):
predict_data = db.get('hor.predict_data')
y_col = db.get("hor.y_col")
if predict_data is None:
return ""
params = db.get("hor.params")
predict_data = [float(predict_data)]
predicted = Predict_final_values(predict_data, params)
return common.success_msg('Predicted Dependent Variable (' + y_col +
') = ' + str(predicted))
def lr_predict_data(n_clicks):
predict_data = db.get('cl.predict_data')
if predict_data is None:
return ""
predict_data = get_predict_data_list(predict_data)
model = db.get("lr.model")
params = db.get("lr.params")
if len(predict_data) != len(params) - 1:
return common.error_msg('Pass Valid InDependent Variables!!')
predicted = model.predict(predict_data)
return common.success_msg('Predicted Dependent Variable = ' +
str(predicted))
def clean_save_file(n):
## Team 2 API Integration
df = db.get("raw_data")
file = db.get("file")
sheet = db.get("sheet")
tags = db.get('tags')
div = None
if (not n is None) and (not df is None):
try:
df, cleaned_df, defective_df, stats = data_cleaning(df)
if not sheet is None:
file = FileUtils.append_file_name(file, sheet)
file = file.split('.')[0]
path = FileUtils.path('clean', file)
cleaned_df.to_csv(path, index=False)
### Tag the cleaned data ###
if file in tags:
tags[file] = tags[file] + 1
else:
tags[file] = 1
col_df = pd.DataFrame(columns=stats['col_name'])
col_df.loc[0] = stats['col_type']
stat_df = pd.DataFrame(columns=[
'Tag', 'Total no of Records', 'Cleaned no of Records',
'Defective no of Records'
])
stat_df.loc[0] = [
'Tag ' + str(tags[file]), stats['row_total'],
stats['row_cleaned'], stats['row_defect']
]
div = html.Div([
common.success_msg("File is Cleaned & Saved Successfully!!"),
html.H2('Cleaned Data Statistic'),
dbc.Table.from_dataframe(stat_df,
striped=True,
bordered=True,
hover=True,
style=common.table_style),
html.H2('Cleaned Data Schema'),
dbc.Table.from_dataframe(col_df,
striped=True,
bordered=True,
hover=True,
style=common.table_style)
],
style={'margin': '10px'})
except Exception as e:
return common.error_msg("Data Cleansing API Error: " + str(e))
return div
def stats_table_and_linear_regression(json_ordered_data):
if json_ordered_data is None:
return (common.msg(None), generate_table(pd.DataFrame(columns=[])),
generate_table(pd.DataFrame(columns=[])), y_ycap_fig,
error_fig, "", "")
dff = pd.read_json(json_ordered_data, orient='split')
col = list(dff.columns)
y = list(dff[col[-1]])
data = []
x_col = col[:-1]
y_col = col[-1]
data = [[] for i in range(len(x_col))]
for i in range(len(x_col)):
x = dff[x_col[i]].values.tolist()
data[i] = x
##Team 3 API Integration
try:
model = LinearRegression()
db.put("lr.model", model)
db.put("lr.x_col", x_col)
db.put("lr.y_col", y_col)
(summary, params, ycap) = model.fit(data, y)
db.put("lr.summary", summary)
db.put("lr.params", params)
db.put("lr.ycap", ycap)
error_mean = model.model_stats()['mean']
db.put("lr.error_mean", error_mean)
except (Exception, ValueError) as e:
return (common.error_msg("Linear Regression API Error: " + str(e)),
generate_table(pd.DataFrame(columns=[])),
generate_table(pd.DataFrame(columns=[])), y_ycap_fig,
error_fig, "", "")
df_stats = common.get_stats_df(summary, x_col, y_col)
table1 = dbc.Table.from_dataframe(df_stats,
striped=True,
bordered=True,
hover=True,
style=common.table_style)
df_coeff = common.get_coeff_df(params, x_col)
table2 = dbc.Table.from_dataframe(df_coeff,
striped=True,
bordered=True,
hover=True,
style=common.table_style)
trace_1 = go.Scatter(x=list(range(len(y))),
y=ycap,
name='Y Predicted (ŷ)',
line=dict(width=2, color='rgb(229, 151, 50)'))
trace_2 = go.Scatter(x=list(range(len(y))),
y=y,
name='Y Actual',
line=dict(width=2, color='rgb(106, 181, 135)'))
ydiff = [y[i] - ycap[i] for i in range(len(y))]
trace_3 = go.Scatter(x=list(range(len(y))),
y=ydiff,
line=dict(width=2, color='rgb(236, 10, 15)'))
fig1 = go.Figure(data=[trace_1, trace_2], layout=y_ycap_title)
fig2 = go.Figure(data=[trace_3], layout=error_title)
error_mean = html.H2('Error Mean = ' +
str(round(db.get('lr.error_mean'), 4)))
##Team 5 API Integration
anova = get_anova(y, ycap, len(params))
db.put('lr.anova', anova)
anova_div = common.get_anova_div(anova)
return (
common.success_msg("Linear Regression API Exceuted Successfully!!"),
table1, table2, fig1, fig2, error_mean, anova_div)
def stats_table_and_hor_regression(json_ordered_data, hor_order):
if json_ordered_data is None or hor_order is None:
return (common.msg(None), generate_table(pd.DataFrame(columns=[])),
generate_table(pd.DataFrame(columns=[])), y_ycap_fig,
error_fig, "", "", "")
dff = pd.read_json(json_ordered_data, orient='split')
col = list(dff.columns)
x_col = [col[0]]
y_col = col[1]
dff = dff.sort_values(by=x_col)
x = list(dff[col[0]])
y = list(dff[col[1]])
##Team 4 API Integration
try:
db.put("hor.x_col", x_col)
db.put("hor.y_col", y_col)
(ycap, params) = Building_model_equation(x, y, hor_order)
print(params)
params = params.tolist()
print(params)
db.put("hor.params", params)
db.put("hor.ycap", ycap)
db.put("hor.order", hor_order)
error = 10.0
for i in range(len(y)):
error += y[i] - ycap[i]
error_mean = error / len(y)
db.put("hor.error_mean", error_mean)
db.put("hor.error_mean", error_mean)
## Team 3 API call for Summary Statistics
model = LinearRegression()
(summary, params_ignore, ycap_ignore) = model.fit([x], y)
db.put("hor.summary", summary)
except (Exception, ValueError) as e:
return (common.error_msg("Higher Order Regression API Error: " +
str(e)),
generate_table(pd.DataFrame(columns=[])),
generate_table(pd.DataFrame(columns=[])), y_ycap_fig,
error_fig, "", "", "")
df_stats = common.get_stats_df(summary, x_col, y_col)
table1 = dbc.Table.from_dataframe(df_stats,
striped=True,
bordered=True,
hover=True,
style=common.table_style)
df_coeff = common.hor_get_coeff_df(params)
table2 = dbc.Table.from_dataframe(df_coeff,
striped=True,
bordered=True,
hover=True,
style=common.table_style)
trace_actual = go.Scatter(x=x,
y=y,
name='Y Actual',
mode='markers',
marker=dict(color='rgb(106, 181, 135)'))
trace_predict = go.Scatter(x=x,
y=ycap,
name='Y Predicted (ŷ)',
line=dict(width=2, color='rgb(229, 151, 50)'))
ydiff = [y[i] - ycap[i] for i in range(len(y))]
trace_error = go.Scatter(x=x,
y=ydiff,
line=dict(width=2, color='rgb(236, 10, 15)'))
x_title = "x (" + str(x_col[0]) + ")"
y_title = "y,ŷ(" + str(y_col) + ")"
y_ycap_title = go.Layout(title='Actual vs Predicted Y Plot',
hovermode='closest',
xaxis={'title': x_title},
yaxis={'title': y_title})
error_title = go.Layout(title='Error Plot',
hovermode='closest',
xaxis={'title': x_title},
yaxis={'title': 'Error = y - ŷ'})
fig1 = go.Figure(data=[trace_predict, trace_actual], layout=y_ycap_title)
fig2 = go.Figure(data=[trace_error], layout=error_title)
error_mean = html.H2('Error Mean = ' +
str(round(db.get('hor.error_mean'), 4)))
##Team 5 API Integration
anova = get_anova(y, ycap, len(params))
db.put('hor.anova', anova)
anova_div = common.get_anova_div(anova)
independent_var = ','.join(x_col)
return (common.success_msg(
"Higher Order Regression API Exceuted Successfully!!"), table1, table2,
fig1, fig2, error_mean, anova_div, html.H2(independent_var))
def stats_table_and_linear_regression(json_ordered_data):
if json_ordered_data is None:
return (common.msg(None), generate_table(pd.DataFrame(columns=[])),
generate_table(pd.DataFrame(columns=[])), y_ycap_fig,
error_fig, "", "", "")
dff = pd.read_json(json_ordered_data, orient='split')
col = list(dff.columns)
x_col = col[:-1]
y_col = col[-1]
if len(x_col) == 1:
dff = dff.sort_values(by=x_col)
data = [[] for i in range(len(x_col))]
for i in range(len(x_col)):
x = dff[x_col[i]].values.tolist()
data[i] = x
y = list(dff[col[-1]])
##Team 3 API Integration
try:
model = LinearRegression()
db.put("lr.model", model)
db.put("lr.x_col", x_col)
db.put("lr.y_col", y_col)
(summary, params, ycap) = model.fit(data, y)
db.put("lr.summary", summary)
db.put("lr.params", params)
db.put("lr.ycap", ycap)
error_mean = model.model_stats()['mean']
db.put("lr.error_mean", error_mean)
except (Exception, ValueError) as e:
return (common.error_msg("Linear Regression API Error: " + str(e)),
generate_table(pd.DataFrame(columns=[])),
generate_table(pd.DataFrame(columns=[])), y_ycap_fig,
error_fig, "", "", "")
df_stats = common.get_stats_df(summary, x_col, y_col)
table1 = dbc.Table.from_dataframe(df_stats,
striped=True,
bordered=True,
hover=True,
style=common.table_style)
df_coeff = common.get_coeff_df(params, x_col)
table2 = dbc.Table.from_dataframe(df_coeff,
striped=True,
bordered=True,
hover=True,
style=common.table_style)
if len(data) == 1:
trace_x = data[0]
x_title = "x (" + str(x_col[0]) + ")"
trace_actual = go.Scatter(x=trace_x,
y=y,
name='Y Actual',
mode='markers',
marker=dict(color='rgb(106, 181, 135)'))
else:
trace_x = list(range(len(y)))
x_title = 'Sequence of data points'
trace_actual = go.Scatter(x=trace_x,
y=y,
name='Y Actual',
line=dict(width=2,
color='rgb(106, 181, 135)'))
trace_predict = go.Scatter(x=trace_x,
y=ycap,
name='Y Predicted (ŷ)',
line=dict(width=2, color='rgb(229, 151, 50)'))
ydiff = [y[i] - ycap[i] for i in range(len(y))]
trace_error = go.Scatter(x=trace_x,
y=ydiff,
line=dict(width=2, color='rgb(236, 10, 15)'))
y_title = "y,ŷ(" + str(y_col) + ")"
y_ycap_title = go.Layout(title='Actual vs Predicted Y Plot',
hovermode='closest',
xaxis={'title': x_title},
yaxis={'title': y_title})
error_title = go.Layout(title='Error Plot',
hovermode='closest',
xaxis={'title': x_title},
yaxis={'title': 'Error = y - ŷ'})
fig1 = go.Figure(data=[trace_predict, trace_actual], layout=y_ycap_title)
fig2 = go.Figure(data=[trace_error], layout=error_title)
error_mean = html.H2('Error Mean = ' +
str(round(db.get('lr.error_mean'), 4)))
##Team 5 API Integration
anova = get_anova(y, ycap, len(params))
db.put('lr.anova', anova)
anova_div = common.get_anova_div(anova)
independent_var = ','.join(x_col)
return (
common.success_msg("Linear Regression API Exceuted Successfully!!"),
table1, table2, fig1, fig2, error_mean, anova_div,
html.H2(independent_var))