def _update_analysis_reports_and_datastore(tnode_, task_):
assert (len(tnode_.meta_task.output_file_display_names) ==
len(tnode_.meta_task.output_file_descriptions) ==
len(tnode_.meta_task.output_types) == len(task_.output_files))
for i_file, (file_type_, path_, name, description) in enumerate(zip(
tnode_.meta_task.output_types, task_.output_files,
tnode_.meta_task.output_file_display_names,
tnode_.meta_task.output_file_descriptions)):
source_id = "{t}-out-{i}".format(t=task_.task_id, i=i_file)
if tnode_.meta_task.datastore_source_id is not None:
source_id = tnode_.meta_task.datastore_source_id
ds_uuid = _get_or_create_uuid_from_file(path_, file_type_)
is_chunked_ = _is_chunked_task_node_type(tnode_)
ds_file_ = DataStoreFile(ds_uuid, source_id, file_type_.file_type_id, path_, is_chunked=is_chunked_, name=name, description=description)
ds.add(ds_file_)
ds.write_update_json(job_resources.datastore_json)
# Update Services
services_add_datastore_file(ds_file_)
dsr = DU.datastore_to_report(ds)
R.write_report_to_html(dsr, os.path.join(job_resources.html, 'datastore.html'))
if file_type_ == FileTypes.REPORT:
T.write_task_report(job_resources, task_.task_id, path_, DU._get_images_in_dir(task_.output_dir))
update_analysis_file_links(tnode_.idx, path_)
def write_main_workflow_report(job_id, job_resources, workflow_opts, task_opts, bg_, state_, was_successful_, run_time_sec):
"""
Write the main workflow level report.
:type job_resources: JobResources
:type workflow_opts: WorkflowLevelOptions
:type bg_: BindingsGraph
:type was_successful_: bool
:type run_time_sec: float
:param job_id:
:param job_resources:
:param workflow_opts:
:param task_opts:
:param bg_:
:param state_:
:param was_successful_:
:param run_time_sec:
:return:
"""
# workflow_json = os.path.join(job_resources.workflow, 'workflow.json')
# with open(workflow_json, 'w+') as f:
# f.write(json.dumps(json_graph.node_link_data(bg_)))
report_path = os.path.join(job_resources.workflow, 'report-tasks.json')
report_ = _to_report(bg_, job_resources.root, job_id, state_, was_successful_, run_time_sec)
report_.write_json(report_path)
R.write_report_with_html_extras(report_, os.path.join(job_resources.root, 'index.html'), job_resources.html)
setting_report = _to_workflow_settings_report(bg_, workflow_opts, task_opts, state_, was_successful_)
R.write_report_to_html(setting_report, os.path.join(job_resources.html, 'settings.html'))
setting_report = _to_workflow_report(job_resources, bg_, workflow_opts, task_opts, state_, was_successful_, _get_images_in_dir(job_resources.workflow, formats=(".svg",)))
R.write_report_to_html(setting_report, os.path.join(job_resources.html, 'workflow.html'))
def _update_analysis_reports_and_datastore(tnode_, task_):
assert (len(tnode_.meta_task.output_file_display_names) ==
len(tnode_.meta_task.output_file_descriptions) ==
len(tnode_.meta_task.output_types) == len(task_.output_files))
for i_file, (file_type_, path_, name, description) in enumerate(zip(
tnode_.meta_task.output_types, task_.output_files,
tnode_.meta_task.output_file_display_names,
tnode_.meta_task.output_file_descriptions)):
source_id = "{t}-out-{i}".format(t=task_.task_id, i=i_file)
if tnode_.meta_task.datastore_source_id is not None:
source_id = tnode_.meta_task.datastore_source_id
ds_uuid = _get_or_create_uuid_from_file(path_, file_type_)
is_chunked_ = _is_chunked_task_node_type(tnode_)
ds_file_ = DataStoreFile(ds_uuid, source_id, file_type_.file_type_id, path_, is_chunked=is_chunked_, name=name, description=description)
ds.add(ds_file_)
ds.write_update_json(job_resources.datastore_json)
# Update Services
services_add_datastore_file(ds_file_)
dsr = DU.datastore_to_report(ds)
R.write_report_to_html(dsr, os.path.join(job_resources.html, 'datastore.html'))
if file_type_ == FileTypes.REPORT:
T.write_task_report(job_resources, task_.task_id, path_, DU._get_images_in_dir(task_.output_dir))
update_analysis_file_links(tnode_.idx, path_)
def write_update_main_workflow_report(job_id,
job_resources,
bg_,
state_,
was_successful_,
run_time_sec,
report_uuid=None,
error_message=None):
"""
This will only update the index.html with the current state of each task
"""
report_path = os.path.join(job_resources.workflow, 'report-tasks.json')
report_ = _to_report(bg_,
job_resources.root,
job_id,
state_,
was_successful_,
run_time_sec,
report_uuid=report_uuid,
error_message=error_message)
report_.write_json(report_path)
R.write_report_to_html(report_,
os.path.join(job_resources.html, 'index.html'))
return True
def write_main_workflow_report(job_id, job_resources, workflow_opts, task_opts, bg_, state_, was_successful_, run_time_sec):
"""
Write the main workflow level report.
:type job_resources: JobResources
:type workflow_opts: WorkflowLevelOptions
:type bg_: BindingsGraph
:type was_successful_: bool
:type run_time_sec: float
:param job_id:
:param job_resources:
:param workflow_opts:
:param task_opts:
:param bg_:
:param state_:
:param was_successful_:
:param run_time_sec:
:return:
"""
# workflow_json = os.path.join(job_resources.workflow, 'workflow.json')
# with open(workflow_json, 'w+') as f:
# f.write(json.dumps(json_graph.node_link_data(bg_)))
report_path = os.path.join(job_resources.workflow, 'report-tasks.json')
report_ = _to_report(bg_, job_resources.root, job_id, state_, was_successful_, run_time_sec)
report_.write_json(report_path)
R.write_report_with_html_extras(report_, os.path.join(job_resources.root, 'index.html'), job_resources.html)
setting_report = _to_workflow_settings_report(bg_, workflow_opts, task_opts, state_, was_successful_)
R.write_report_to_html(setting_report, os.path.join(job_resources.html, 'settings.html'))
setting_report = _to_workflow_report(job_resources, bg_, workflow_opts, task_opts, state_, was_successful_, _get_images_in_dir(job_resources.workflow, formats=(".svg",)))
R.write_report_to_html(setting_report, os.path.join(job_resources.html, 'workflow.html'))
def write_update_main_workflow_report(job_id, job_resources, bg_, state_, was_successful_, run_time_sec):
"""
This will only update the index.html with the current state of each task
"""
report_path = os.path.join(job_resources.workflow, 'report-tasks.json')
report_ = _to_report(bg_, job_resources.root, job_id, state_, was_successful_, run_time_sec)
report_.write_json(report_path)
R.write_report_to_html(report_, os.path.join(job_resources.root, 'index.html'))
return True
def _update_analysis_reports_and_datastore(tnode_, task_):
for file_type_, path_ in zip(tnode_.meta_task.output_types, task_.output_files):
source_id = "{t}-{f}".format(t=task_.task_id, f=file_type_.file_type_id)
ds_uuid = _get_dataset_uuid_or_create_uuid(path_)
ds_file_ = DataStoreFile(ds_uuid, source_id, file_type_.file_type_id, path_)
ds.add(ds_file_)
ds.write_update_json(job_resources.datastore_json)
# Update Services
services_add_datastore_file(ds_file_)
dsr = DU.datastore_to_report(ds)
R.write_report_to_html(dsr, os.path.join(job_resources.html, 'datastore.html'))
if file_type_ == FileTypes.REPORT:
T.write_task_report(job_resources, task_.task_id, path_, DU._get_images_in_dir(task_.output_dir))
update_analysis_file_links(tnode_.idx, path_)
def _update_analysis_reports_and_datastore(tnode_, task_):
for file_type_, path_ in zip(tnode_.meta_task.output_types, task_.output_files):
source_id = "{t}-{f}".format(t=task_.task_id, f=file_type_.file_type_id)
ds_uuid = _get_dataset_uuid_or_create_uuid(path_)
ds_file_ = DataStoreFile(ds_uuid, source_id, file_type_.file_type_id, path_)
ds.add(ds_file_)
ds.write_update_json(job_resources.datastore_json)
# Update Services
services_add_datastore_file(ds_file_)
dsr = DU.datastore_to_report(ds)
R.write_report_to_html(dsr, os.path.join(job_resources.html, 'datastore.html'))
if file_type_ == FileTypes.REPORT:
T.write_task_report(job_resources, task_.task_id, path_, DU._get_images_in_dir(task_.output_dir))
update_analysis_file_links(tnode_.idx, path_)
def write_task_summary_report(bg_):
task_summary_report = DU.to_task_summary_report(bg_)
p = os.path.join(job_resources.html, 'task_summary.html')
R.write_report_to_html(task_summary_report, p)
def write_task_summary_report(bg_):
task_summary_report = DU.to_task_summary_report(bg_)
p = os.path.join(job_resources.html, 'task_summary.html')
R.write_report_to_html(task_summary_report, p)
