def download_labels():
"""
Generate CSV file for label assignment
Parameters
----------
job_id: str
task_id: str
converted to int in this function
Returns
-------
text/csv file
"""
job_id = request.args.get('job_id')
if job_id is None:
return None
task_id = int(request.args.get('task_id'))
if task_id is None:
return None
task = mongo_get_task(job_id, task_id)
if task is None:
return None
covar_type = task['covar_type']
covar_tied = task['covar_tied']
k = task['k']
export_filename = '{}_{}_{}_{}.csv'.format(job_id, covar_type, covar_tied, k)
job = mongo_get_job(job_id)
s3_file_key = job['s3_file_key']
data = s3_to_df(s3_file_key)
data = data.assign(Label=task['labels'])
response = make_response(data.to_csv(index=False))
response.headers["Content-Disposition"] = "attachment; filename={}".format(export_filename)
response.headers["Content-Type"] = "text/csv"
return response
def plot_cluster():
"""
Generate the Cluster plot as a PNG
Parameters
----------
job_id: str
x_axis: str
Name of column from user dataset to be used for the x axis of the plot
y_axis: str
Name of column from user dataset to be used for the y axis of the plot
Returns
-------
image/png
"""
job_id = request.args.get('job_id')
x_axis = request.args.get('x_axis')
y_axis = request.args.get('y_axis')
show_ticks = request.args.get('show_ticks', 'True') == 'True'
min_members = int(request.args.get('min_members', None))
if job_id is None or x_axis is None or y_axis is None:
return None
job = mongo_get_job(job_id)
tasks = mongo_get_tasks(job_id)
if min_members is not None:
tasks = filter_by_min_members(tasks, min_members)
covar_types, covar_tieds, ks, labels, bics, task_ids = tasks_to_best_results(tasks)
s3_file_key = job['s3_file_key']
viz_columns = [x_axis, y_axis]
data = s3_to_df(s3_file_key)
fig = plot_cluster_fig(data, viz_columns, zip(covar_types, covar_tieds, labels, ks, bics), show_ticks)
cluster_plot = fig_to_png(fig)
response = make_response(cluster_plot.getvalue())
response.mimetype = 'image/png'
return response
def status(job_id=None):
"""
Pull information on all tasks for a job from MongoDB and render as a table
Parameters
----------
job_id: str
Returns
-------
html
"""
if request.method == 'POST':
job_id = request.form['job_id']
if job_id:
return redirect(url_for('status', job_id=job_id))
else:
flash("Invalid job ID!", 'danger')
return render_template('index.html')
if job_id is None:
job_id = request.args.get('job_id', None)
if job_id is None:
flash('Job ID invalid!'.format(job_id), category='danger')
return render_template('index.html')
else:
if not mongo_job_id_exists(job_id):
flash('Job ID {} not found!'.format(job_id), category='danger')
return render_template('index.html')
# job_id is valid
job = mongo_get_job(job_id)
tasks = mongo_get_tasks(job_id)
stats = task_stats(job['n_tasks'], tasks)
start_time_date, start_time_clock = format_date_time(job['start_time'])
return render_template('status.html', job_id=job_id, stats=stats, tasks=tasks, job=job,
start_time_date=start_time_date, start_time_clock=start_time_clock)
def plot_correlation():
"""
Generate the Correlation heat map as a PNG
Parameters
----------
job_id: str
Returns
-------
image/png
"""
job_id = request.args.get('job_id')
if job_id is None:
return None
job = mongo_get_job(job_id)
s3_file_key = job['s3_file_key']
data = s3_to_df(s3_file_key)
fig = plot_correlation_fig(data)
correlation_plot = fig_to_png(fig)
response = make_response(correlation_plot.getvalue())
response.mimetype = 'image/png'
return response
def report(job_id=None):
"""
Generate report for a job
Parameters
----------
job_id: str
x_axis: str
Name of column from user dataset to be used for the x axis of the plot
y_axis: str
Name of column from user dataset to be used for the y axis of the plot
min_members: int, optional
Minimum number of members required in all clusters in an experiment to consider the experiment for the report.
Returns
-------
html
"""
if request.method == 'POST':
job_id = request.form.get('job_id')
x_axis = request.form.get('x_axis', None)
y_axis = request.form.get('y_axis', None)
min_members = request.form.get('min_members', None)
elif request.method == 'GET':
if job_id is None:
job_id = request.args.get('job_id')
x_axis = request.args.get('x_axis', None)
y_axis = request.args.get('y_axis', None)
min_members = request.args.get('min_members', None)
if job_id is None:
flash('Job ID invalid!'.format(job_id), category='danger')
return render_template('index.html')
if not mongo_job_id_exists(job_id):
flash('Job ID {} not found!'.format(job_id), category='danger')
return render_template('index.html')
# job_id is valid
job = mongo_get_job(job_id)
n_tasks = job['n_tasks']
tasks = mongo_get_tasks(job_id)
n_tasks_done = len([x for x in tasks if x['task_status'] == 'done'])
if n_tasks != n_tasks_done:
flash('All tasks not completed yet for job ID: {}'.format(job_id), category='danger')
return redirect(url_for('status', job_id=job_id))
# all tasks are done
if min_members is None:
min_members = 10
else:
min_members = int(min_members)
tasks = filter_by_min_members(tasks, min_members=min_members)
start_time_date, start_time_clock = format_date_time(job['start_time'])
covar_types, covar_tieds, ks, labels, bics, task_ids = tasks_to_best_results(tasks)
if x_axis is None or y_axis is None:
# Visualize the first two columns that are not on the exclude list
viz_columns = [c for c in job['columns'] if c.lower().strip() not in EXCLUDE_COLUMNS][:2]
else:
viz_columns = [x_axis, y_axis]
data = s3_to_df(job['s3_file_key'])
columns = list(data.columns)
spatial_columns = [c for c in columns if c.lower() in SPATIAL_COLUMNS][:2]
# recommendations for all covariance types
covar_type_tied_k = {}
for covar_type in covar_types:
covar_type_tied_k[covar_type.capitalize()] = {}
for covar_type, covar_tied, k in zip(covar_types, covar_tieds, ks):
covar_type_tied_k[covar_type.capitalize()][['Untied', 'Tied'][covar_tied]] = k
# task_id for all recommended assignments
covar_type_tied_task_id = {}
for covar_type in covar_types:
covar_type_tied_task_id[covar_type.capitalize()] = {}
for covar_type, covar_tied, task_id in zip(covar_types, covar_tieds, task_ids):
covar_type_tied_task_id[covar_type.capitalize()][['Untied', 'Tied'][covar_tied]] = task_id
return render_template('report.html', job_id=job_id, job=job, min_members=min_members,
covar_type_tied_k=covar_type_tied_k, covar_type_tied_task_id=covar_type_tied_task_id,
columns=columns, viz_columns=viz_columns, spatial_columns=spatial_columns,
start_time_date=start_time_date, start_time_clock=start_time_clock)