def compare(info):
"""Tool to compare pipeline runs based on defined metrics"""
notice('Downloading evaluation metrics and tensorboard logs for all run '
'pipelines in the workspace. This might take some time if there '
'are many pipelines.\nYour patience is '
'much appreciated!')
temp_info = {
constants.ACTIVE_USER: info[constants.ACTIVE_USER],
info[constants.ACTIVE_USER]: info[info[constants.ACTIVE_USER]]
}
# generate notebook
nb = nbf.v4.new_notebook()
nb['cells'] = [
nbf.v4.new_code_cell(import_block()),
nbf.v4.new_code_cell(info_block(temp_info)),
nbf.v4.new_code_cell(application_block()),
nbf.v4.new_code_cell(interface_block()),
]
# write notebook
final_out_path = os.path.join(click.get_app_dir(constants.APP_NAME),
constants.COMPARISON_NOTEBOOK)
s = nbf.writes(nb)
if isinstance(s, bytes):
s = s.decode('utf8')
with open(final_out_path, 'w') as f:
f.write(s)
# serve notebook
os.system('panel serve "{}" --show'.format(final_out_path))
def evaluate(info, pipeline_):
"""Tool for the in-depth evaluation of a pipeline run"""
p_uuid, r_uuid = resolve_pipeline_runs(info, pipeline_)
notice('Downloading evaluation metrics and tensorboard logs for '
'pipeline ID {} and run ID {}. This might take some time if the '
'model resources are significantly large in size.\nYour patience '
'is much appreciated!'.format(format_uuid(p_uuid),
format_uuid(r_uuid)))
log_dir = get_log_dir(p_uuid, r_uuid, info)
eval_dir = get_eval_dir(p_uuid, r_uuid, info)
# generate notebook
nb = nbf.v4.new_notebook()
nb['cells'] = [
nbf.v4.new_code_cell(get_model_block(log_dir)),
nbf.v4.new_code_cell(get_eval_block(eval_dir)),
]
# write notebook
final_out_path = (Path(click.get_app_dir(constants.APP_NAME)) /
constants.EVALUATION_NOTEBOOK)
s = nbf.writes(nb)
if isinstance(s, bytes):
s = s.decode('utf8')
# only import tfx when needed
with open(final_out_path, 'w') as f:
f.write(s)
os.system('jupyter notebook "{}"'.format(final_out_path))
def list_pipelines(info, pipeline_id, ignore_empty):
"""List of registered pipelines"""
utils.notice('Fetching pipeline(s). This might take a few seconds... \n')
active_user = info[constants.ACTIVE_USER]
ws = info[active_user][constants.ACTIVE_WORKSPACE]
ws_api = ce_api.WorkspacesApi(utils.api_client(info))
p_api = ce_api.PipelinesApi(utils.api_client(info))
d_api = ce_api.DatasourcesApi(utils.api_client(info))
pipelines = utils.api_call(
ws_api.get_workspaces_pipelines_api_v1_workspaces_workspace_id_pipelines_get,
ws)
if pipeline_id is not None:
pipeline_id = utils.find_closest_uuid(pipeline_id, pipelines)
pipelines.sort(key=lambda x: x.created_at)
for p in pipelines:
write_check = (len(p.pipeline_runs) > 0 or not ignore_empty) and \
(pipeline_id is None or pipeline_id == p.id)
if write_check:
# THIS WHOLE THING IS HERE FOR A REASON!!!!!!
title = 'PIPELINE NAME: {} PIPELINE ID: {}'.format(
p.name, utils.format_uuid(p.id))
utils.declare(title)
utils.declare('-' * len(title))
if len(p.pipeline_runs) == 0:
click.echo('No runs for this pipeline yet!')
else:
table = []
for r in p.pipeline_runs:
author = utils.api_call(
p_api.get_pipeline_run_user_api_v1_pipelines_pipeline_id_runs_pipeline_run_id_user_get,
p.id,
r.id)
# Resolve datasource
ds_commit = utils.api_call(
d_api.get_single_commit_api_v1_datasources_commits_commit_id_get,
r.datasource_commit_id)
ds = utils.api_call(
d_api.get_datasource_api_v1_datasources_ds_id_get,
ds_commit.datasource_id)
table.append({
'RUN ID': utils.format_uuid(r.id),
'TYPE': r.pipeline_run_type,
'CPUs PER WORKER': r.cpus_per_worker,
'WORKERS': r.workers,
'DATASOURCE': '{}_{}'.format(
ds.name,
utils.format_uuid(r.datasource_commit_id)),
'AUTHOR': author.email,
'CREATED AT': utils.format_date(r.start_time),
})
click.echo(tabulate(table, headers='keys', tablefmt='plain'))
click.echo('\n')
def logs_pipeline(info, source_id):
"""Get link to the logs of a pipeline"""
p_uuid, r_uuid = utils.resolve_pipeline_runs(info, source_id)
utils.notice(
'Generating logs url for the pipeline run ID {}. Please visit the '
'url for all your logs.'.format(utils.format_uuid(r_uuid)))
api = ce_api.PipelinesApi(utils.api_client(info))
logs_url = utils.api_call(
api.get_pipeline_logs_api_v1_pipelines_pipeline_id_runs_pipeline_run_id_logs_get,
pipeline_id=p_uuid,
pipeline_run_id=r_uuid
)
click.echo(logs_url)
def statistics_pipeline(info, pipeline_):
"""Serve the statistics of a pipeline run"""
p_uuid, r_uuid = utils.resolve_pipeline_runs(info,
pipeline_,
run_type=PipelineRunTypes.training.name)
utils.notice('Generating statistics for the pipeline run ID {}. If your '
'browser opens up to a blank window, please refresh '
'the page once.'.format(utils.format_uuid(r_uuid)))
api = ce_api.PipelinesApi(utils.api_client(info))
stat_artifact = utils.api_call(
api.get_pipeline_artifacts_api_v1_pipelines_pipeline_id_runs_pipeline_run_id_artifacts_component_type_get,
pipeline_id=p_uuid,
pipeline_run_id=r_uuid,
component_type=GDPComponent.SplitStatistics.name)
ws_id = info[info[constants.ACTIVE_USER]][constants.ACTIVE_WORKSPACE]
path = Path(click.get_app_dir(constants.APP_NAME),
'statistics',
str(ws_id),
p_uuid,
r_uuid)
utils.download_artifact(artifact_json=stat_artifact[0].to_dict(),
path=path)
import tensorflow as tf
from tensorflow_metadata.proto.v0 import statistics_pb2
import panel as pn
result = {}
for split in os.listdir(path):
stats_path = os.path.join(path, split, 'stats_tfrecord')
serialized_stats = next(tf.compat.v1.io.tf_record_iterator(stats_path))
stats = statistics_pb2.DatasetFeatureStatisticsList()
stats.ParseFromString(serialized_stats)
dataset_list = statistics_pb2.DatasetFeatureStatisticsList()
for i, d in enumerate(stats.datasets):
d.name = split
dataset_list.datasets.append(d)
result[split] = dataset_list
h = utils.get_statistics_html(result)
pn.serve(panels=pn.pane.HTML(h, width=1200), show=True)
def model_pipeline(info, pipeline_, output_path):
"""Download the trained model to a specified location"""
if os.path.exists(output_path) and os.path.isdir(output_path):
if not [f for f in os.listdir(output_path) if
not f.startswith('.')] == []:
utils.error("Output path must be an empty directory!")
if os.path.exists(output_path) and not os.path.isdir(output_path):
utils.error("Output path must be an empty directory!")
if not os.path.exists(output_path):
"Creating directory {}..".format(output_path)
p_uuid, r_uuid = utils.resolve_pipeline_runs(info, pipeline_)
utils.notice('Downloading the trained model from pipeline run '
'ID {}. This might take some time if the model '
'resources are significantly large in size.\nYour patience '
'is much appreciated!'.format(utils.format_uuid(r_uuid)))
api = ce_api.PipelinesApi(utils.api_client(info))
artifact = utils.api_call(
api.get_pipeline_artifacts_api_v1_pipelines_pipeline_id_runs_pipeline_run_id_artifacts_component_type_get,
pipeline_id=p_uuid,
pipeline_run_id=r_uuid,
component_type=GDPComponent.Deployer.name)
spin = utils.Spinner()
spin.start()
if len(artifact) == 1:
utils.download_artifact(artifact_json=artifact[0].to_dict(),
path=output_path)
spin.stop()
else:
utils.error('Something unexpected happened! Please contact '
'[email protected] to get further information.')
utils.declare('Model downloaded to: {}'.format(output_path))
# TODO: [LOW] Make the Tensorflow version more dynamic
utils.declare('Please note that the model is saved as a SavedModel '
'Tensorflow artifact, trained on Tensoflow 2.1.0.')
def pull_function_version(info, function_id, version_id, output_path):
"""Download a version of a given custom function"""
api = ce_api.FunctionsApi(api_client(info))
# Infer the function uuid and name
f_list = api_call(api.get_functions_api_v1_functions_get)
f_uuid = find_closest_uuid(function_id, f_list)
f_name = [f.name for f in f_list if f.id == f_uuid][0]
# Infer the version uuid
v_list = api_call(
api.get_function_versions_api_v1_functions_function_id_versions_get,
f_uuid)
v_uuid = find_closest_uuid(version_id, v_list)
notice('Downloading the function with the following parameters: \n'
'Name: {f_name}\n'
'function_id: {f_id}\n'
'version_id: {v_id}\n'.format(f_name=f_name,
f_id=format_uuid(f_uuid),
v_id=format_uuid(v_uuid)))
# Get the file and write it to the output path
encoded_file = api_call(
api.get_function_version_api_v1_functions_function_id_versions_version_id_get,
f_uuid,
v_uuid)
# Derive the output path and download
if output_path is None:
output_path = os.path.join(os.getcwd(), '{}@{}.py'.format(f_name,
v_uuid))
with open(output_path, 'wb') as f:
f.write(base64.b64decode(encoded_file.file_contents))
declare('File downloaded to {}'.format(output_path))
def get_pipeline_status(info, pipeline_id):
"""Get status of started pipelines"""
utils.notice('Fetching pipeline(s). This might take a few seconds.. \n')
active_user = info[constants.ACTIVE_USER]
ws = info[active_user][constants.ACTIVE_WORKSPACE]
ws_api = ce_api.WorkspacesApi(utils.api_client(info))
p_api = ce_api.PipelinesApi(utils.api_client(info))
d_api = ce_api.DatasourcesApi(utils.api_client(info))
pipelines = utils.api_call(
ws_api.get_workspaces_pipelines_api_v1_workspaces_workspace_id_pipelines_get,
ws)
if pipeline_id is not None:
pipeline_id = utils.find_closest_uuid(pipeline_id, pipelines)
pipelines.sort(key=lambda x: x.created_at)
for p in pipelines:
write_check = (len(p.pipeline_runs) > 0) and \
(pipeline_id is None or pipeline_id == p.id)
if write_check:
title = 'PIPELINE NAME: {} PIPELINE ID: {}'.format(
p.name, utils.format_uuid(p.id))
utils.declare(title)
utils.declare('-' * len(title))
table = []
for r in p.pipeline_runs:
run = utils.api_call(
p_api.get_pipeline_run_api_v1_pipelines_pipeline_id_runs_pipeline_run_id_get,
p.id,
r.id)
# Resolve datasource
ds_commit = utils.api_call(
d_api.get_single_commit_api_v1_datasources_commits_commit_id_get,
r.datasource_commit_id)
ds = utils.api_call(
d_api.get_datasource_api_v1_datasources_ds_id_get,
ds_commit.datasource_id)
if run.end_time:
td = run.end_time - run.start_time
else:
td = datetime.now(timezone.utc) - run.start_time
# # Resolve component status
# stage = utils.get_run_stage(run.pipeline_components)
table.append({
'RUN ID': utils.format_uuid(run.id),
'TYPE': run.pipeline_run_type,
'STATUS': run.status,
# 'STAGE': stage,
'DATASOURCE': '{}_{}'.format(
ds.name, utils.format_uuid(run.datasource_commit_id)),
'DATAPOINTS': '{}'.format(ds_commit.n_datapoints),
# 'RUNNING STAGE': stage,
'START TIME': utils.format_date(run.start_time),
'DURATION': utils.format_timedelta(td),
})
click.echo(tabulate(table, headers='keys', tablefmt='plain'))
click.echo('\n')