def _display_experiments(experiments: List[ApiExperiment],
output_format: OutputFormat):
headers = ['Experiment ID', 'Name', 'Created at']
data = [
[exp.id, exp.name, exp.created_at.isoformat()] for exp in experiments
]
print_output(data, headers, output_format, table_format='grid')
def create_version(ctx: click.Context,
package_file: str,
pipeline_version: str,
pipeline_id: Optional[str] = None,
pipeline_name: Optional[str] = None,
description: Optional[str] = None):
"""Upload a version of a pipeline."""
client_obj: client.Client = ctx.obj['client']
output_format = ctx.obj['output']
if bool(pipeline_id) == bool(pipeline_name):
raise ValueError(either_option_required)
if pipeline_name is not None:
pipeline_id = client_obj.get_pipeline_id(name=pipeline_name)
if pipeline_id is None:
raise ValueError(
f"Can't find a pipeline with name: {pipeline_name}")
version = client_obj.upload_pipeline_version(
pipeline_package_path=package_file,
pipeline_version_name=pipeline_version,
pipeline_id=pipeline_id,
pipeline_name=pipeline_name,
description=description)
output.print_output(
version,
output.ModelType.PIPELINE,
output_format,
)
def _print_pipeline_versions(versions, output_format):
headers = ["Version ID", "Version name", "Uploaded at"]
data = [[
version.id,
version.name,
version.created_at.isoformat()
] for version in versions]
print_output(data, headers, output_format, table_format="grid")
def _print_pipelines(pipelines, output_format):
headers = ["Pipeline ID", "Name", "Uploaded at"]
data = [[
pipeline.id,
pipeline.name,
pipeline.created_at.isoformat()
] for pipeline in pipelines]
print_output(data, headers, output_format, table_format="grid")
def _print_pipeline_versions(versions: List[kfp_server_api.ApiPipelineVersion],
output_format: OutputFormat):
headers = ['Version ID', 'Version name', 'Uploaded at', 'Pipeline ID']
data = [[
version.id, version.name,
version.created_at.isoformat(),
next(rr for rr in version.resource_references
if rr.key.type == kfp_server_api.ApiResourceType.PIPELINE).key.id
] for version in versions]
print_output(data, headers, output_format, table_format='grid')
def unarchive(ctx: click.Context, run_id: str):
"""Unarchive a pipeline run."""
client_obj: client.Client = ctx.obj['client']
output_format = ctx.obj['output']
client_obj.unarchive_run(run_id=run_id)
run = client_obj.get_run(run_id=run_id)
output.print_output(
run.run,
output.ModelType.RUN,
output_format,
)
def get(ctx: click.Context, job_id: str):
"""Get information about a recurring run."""
client_obj: client.Client = ctx.obj['client']
output_format = ctx.obj['output']
recurring_run = client_obj.get_recurring_run(job_id)
output.print_output(
recurring_run,
output.ModelType.JOB,
output_format,
)
def get(ctx: click.Context, experiment_id: str):
"""Get information about an experiment."""
client_obj: client.Client = ctx.obj['client']
output_format = ctx.obj['output']
experiment = client_obj.get_experiment(experiment_id)
output.print_output(
experiment,
output.ModelType.EXPERIMENT,
output_format,
)
def create(ctx: click.Context, description: str, name: str):
"""Create an experiment."""
client_obj: client.Client = ctx.obj['client']
output_format = ctx.obj['output']
experiment = client_obj.create_experiment(name, description=description)
output.print_output(
experiment,
output.ModelType.EXPERIMENT,
output_format,
)
def _print_runs(runs: List[kfp_server_api.ApiRun], output_format: OutputFormat):
headers = ['run id', 'name', 'status', 'created at', 'experiment id']
data = [[
run.id, run.name, run.status,
run.created_at.isoformat(),
next(rr
for rr in run.resource_references
if rr.key.type == kfp_server_api.ApiResourceType.EXPERIMENT).key.id
]
for run in runs]
print_output(data, headers, output_format, table_format='grid')
def get(ctx: click.Context, pipeline_id: str):
"""Get information about a pipeline."""
client_obj: client.Client = ctx.obj['client']
output_format = ctx.obj['output']
pipeline = client_obj.get_pipeline(pipeline_id)
output.print_output(
pipeline,
output.ModelType.PIPELINE,
output_format,
)
def get_version(ctx: click.Context, version_id: str):
"""Get information about a version of a pipeline."""
client_obj: client.Client = ctx.obj['client']
output_format = ctx.obj['output']
version = client_obj.get_pipeline_version(version_id=version_id)
output.print_output(
version,
output.ModelType.PIPELINE,
output_format,
)
def disable(ctx: click.Context, job_id: str):
"""Disable a recurring run."""
client_obj: client.Client = ctx.obj['client']
output_format = ctx.obj['output']
client_obj.disable_job(job_id=job_id)
# TODO: add wait option, since disable takes time to complete
recurring_run = client_obj.get_recurring_run(job_id=job_id)
output.print_output(
recurring_run,
output.ModelType.JOB,
output_format,
)
def _display_pipeline(pipeline, output_format):
# Pipeline information
table = [
["ID", pipeline.id],
["Name", pipeline.name],
["Description", pipeline.description],
["Uploaded at", pipeline.created_at.isoformat()],
]
# Pipeline parameter details
headers = ["Parameter Name", "Default Value"]
data = []
if pipeline.parameters is not None:
data = [[param.name, param.value] for param in pipeline.parameters]
if output_format == OutputFormat.table.name:
print_output([], ["Pipeline Details"], output_format)
print_output(table, [], output_format, table_format="plain")
print_output(data, headers, output_format, table_format="grid")
elif output_format == OutputFormat.json.name:
output = dict()
output["Pipeline Details"] = dict(table)
params = []
for item in data:
params.append(dict(zip(headers, item)))
output["Pipeline Parameters"] = params
print_output(output, [], output_format)
def create(ctx: click.Context,
pipeline_name: str,
package_file: str,
description: str = None):
"""Upload a pipeline."""
client_obj: client.Client = ctx.obj['client']
output_format = ctx.obj['output']
pipeline = client_obj.upload_pipeline(package_file, pipeline_name,
description)
output.print_output(
pipeline,
output.ModelType.PIPELINE,
output_format,
)
def list(ctx: click.Context, page_token: str, max_size: int, sort_by: str,
filter: str):
"""List pipelines."""
client_obj: client.Client = ctx.obj['client']
output_format = ctx.obj['output']
response = client_obj.list_pipelines(page_token=page_token,
page_size=max_size,
sort_by=sort_by,
filter=filter)
output.print_output(
response.pipelines or [],
output.ModelType.PIPELINE,
output_format,
)
def list(ctx: click.Context, page_token: str, max_size: int, sort_by: str,
filter: str):
"""List experiments."""
client_obj: client.Client = ctx.obj['client']
output_format = ctx.obj['output']
response = client_obj.list_experiments(page_token=page_token,
page_size=max_size,
sort_by=sort_by,
filter=filter)
output.print_output(
response.experiments or [],
output.ModelType.EXPERIMENT,
output_format,
)
def list(ctx: click.Context, experiment_id: str, page_token: str, max_size: int,
sort_by: str, filter: str):
"""List pipeline runs."""
client_obj: client.Client = ctx.obj['client']
output_format = ctx.obj['output']
response = client_obj.list_runs(
experiment_id=experiment_id,
page_token=page_token,
page_size=max_size,
sort_by=sort_by,
filter=filter)
output.print_output(
response.runs or [],
output.ModelType.RUN,
output_format,
)
def display_run(client: client.Client, namespace: str, run_id: str, watch: bool,
output_format: str):
run = client.get_run(run_id).run
output.print_output(
run,
output.ModelType.RUN,
output_format,
)
if not watch:
return
argo_path = shutil.which('argo')
if not argo_path:
raise RuntimeError(
"argo isn't found in $PATH. It's necessary for watch. "
"Please make sure it's installed and available. "
'Installation instructions be found here - '
'https://github.com/argoproj/argo-workflows/releases')
argo_workflow_name = None
while True:
time.sleep(1)
run_detail = client.get_run(run_id)
run = run_detail.run
if run_detail.pipeline_runtime and run_detail.pipeline_runtime.workflow_manifest:
manifest = json.loads(run_detail.pipeline_runtime.workflow_manifest)
if manifest['metadata'] and manifest['metadata']['name']:
argo_workflow_name = manifest['metadata']['name']
break
if run_detail.run.status in ['Succeeded', 'Skipped', 'Failed', 'Error']:
click.echo(f'Run is finished with status {run_detail.run.status}.')
return
if argo_workflow_name:
subprocess.run(
[argo_path, 'watch', argo_workflow_name, '-n', namespace])
output.print_output(
run,
output.ModelType.RUN,
output_format,
)
def unarchive(ctx: click.Context, experiment_id: str, experiment_name: str):
"""Unarchive an experiment."""
client_obj: client.Client = ctx.obj['client']
output_format = ctx.obj['output']
if (experiment_id is None) == (experiment_name is None):
raise ValueError(either_option_required)
if not experiment_id:
experiment = client_obj.get_experiment(experiment_name=experiment_name)
experiment_id = experiment.id
client_obj.unarchive_experiment(experiment_id=experiment_id)
if experiment_id:
experiment = client_obj.get_experiment(experiment_id=experiment_id)
else:
experiment = client_obj.get_experiment(experiment_name=experiment_name)
output.print_output(
experiment,
output.ModelType.EXPERIMENT,
output_format,
)
def create(ctx: click.Context, experiment_name: str, run_name: str,
package_file: str, pipeline_id: str, pipeline_name: str, watch: bool,
timeout: int, version: str, args: List[str]):
"""Submit a pipeline run."""
client_obj: client.Client = ctx.obj['client']
namespace = ctx.obj['namespace']
output_format = ctx.obj['output']
if not run_name:
run_name = experiment_name
if not pipeline_id and pipeline_name:
pipeline_id = client_obj.get_pipeline_id(name=pipeline_name)
if not package_file and not pipeline_id and not version:
click.echo(
'You must provide one of [package_file, pipeline_id, version].',
err=True)
sys.exit(1)
arg_dict = dict(arg.split('=', maxsplit=1) for arg in args)
experiment = client_obj.create_experiment(experiment_name)
run = client_obj.run_pipeline(
experiment_id=experiment.id,
job_name=run_name,
pipeline_package_path=package_file,
params=arg_dict,
pipeline_id=pipeline_id,
version_id=version)
if timeout > 0:
run_detail = client_obj.wait_for_run_completion(run.id, timeout)
output.print_output(
run_detail.run,
output.ModelType.RUN,
output_format,
)
else:
display_run(client_obj, namespace, run.id, watch, output_format)
def _display_recurring_run(recurring_run: kfp_server_api.ApiJob,
output_format: OutputFormat):
table = [
["Recurring Run ID", recurring_run.id],
["Name", recurring_run.name],
]
if output_format == OutputFormat.table.name:
print_output([], ["Recurring Run Details"], output_format)
print_output(table, [], output_format, table_format="plain")
elif output_format == OutputFormat.json.name:
print_output(dict(table), [], output_format)
def _display_experiment(exp: kfp_server_api.ApiExperiment,
output_format: OutputFormat):
table = [
['ID', exp.id],
['Name', exp.name],
['Description', exp.description],
['Created at', exp.created_at.isoformat()],
]
if output_format == OutputFormat.table.name:
print_output([], ['Experiment Details'], output_format)
print_output(table, [], output_format, table_format='plain')
elif output_format == OutputFormat.json.name:
print_output(dict(table), [], output_format)
def _display_pipeline_version(version, output_format):
pipeline_id = version.resource_references[0].key.id
table = [
["Pipeline ID", pipeline_id],
["Version Name", version.name],
["Uploaded at", version.created_at.isoformat()],
]
if output_format == OutputFormat.table.name:
print_output([], ["Pipeline Version Details"], output_format)
print_output(table, [], output_format, table_format="plain")
elif output_format == OutputFormat.json.name:
print_output(dict(table), [], output_format)
def _display_experiment(exp: kfp_server_api.ApiExperiment,
output_format: OutputFormat):
table = [
["ID", exp.id],
["Name", exp.name],
["Description", exp.description],
["Created at", exp.created_at.isoformat()],
]
if output_format == OutputFormat.table.name:
print_output([], ["Experiment Details"], output_format)
print_output(table, [], output_format, table_format="plain")
elif output_format == OutputFormat.json.name:
print_output(dict(table), [], output_format)
def _display_pipeline_version(version: kfp_server_api.ApiPipelineVersion,
output_format: OutputFormat):
pipeline_id = next(
rr for rr in version.resource_references
if rr.key.type == kfp_server_api.ApiResourceType.PIPELINE).key.id
table = [['Pipeline ID', pipeline_id], ['Version name', version.name],
['Uploaded at', version.created_at.isoformat()],
['Version ID', version.id]]
if output_format == OutputFormat.table.name:
print_output([], ['Pipeline Version Details'], output_format)
print_output(table, [], output_format, table_format='plain')
elif output_format == OutputFormat.json.name:
print_output(dict(table), [], output_format)
def _display_pipeline(pipeline: kfp_server_api.ApiPipeline,
output_format: OutputFormat):
# Pipeline information
table = [["Pipeline ID", pipeline.id], ["Name", pipeline.name],
["Description", pipeline.description],
["Uploaded at", pipeline.created_at.isoformat()],
["Version ID", pipeline.default_version.id]]
# Pipeline parameter details
headers = ["Parameter Name", "Default Value"]
data = []
if pipeline.parameters is not None:
data = [[param.name, param.value] for param in pipeline.parameters]
if output_format == OutputFormat.table.name:
print_output([], ["Pipeline Details"], output_format)
print_output(table, [], output_format, table_format="plain")
print_output(data, headers, output_format, table_format="grid")
elif output_format == OutputFormat.json.name:
OutputType = Dict[str, Union[Dict[str, str], List[Dict[str, Any]]]]
output: OutputType = {"Pipeline Details": dict(table)}
params = [dict(zip(headers, item)) for item in data]
output["Pipeline Parameters"] = params
print_output(output, [], output_format)
def _display_pipeline(pipeline: kfp_server_api.ApiPipeline,
output_format: OutputFormat):
# Pipeline information
table = [['Pipeline ID', pipeline.id], ['Name', pipeline.name],
['Description', pipeline.description],
['Uploaded at', pipeline.created_at.isoformat()],
['Version ID', pipeline.default_version.id]]
# Pipeline parameter details
headers = ['Parameter Name', 'Default Value']
data = []
if pipeline.parameters is not None:
data = [[param.name, param.value] for param in pipeline.parameters]
if output_format == OutputFormat.table.name:
print_output([], ['Pipeline Details'], output_format)
print_output(table, [], output_format, table_format='plain')
print_output(data, headers, output_format, table_format='grid')
elif output_format == OutputFormat.json.name:
OutputType = Dict[str, Union[Dict[str, str], List[Dict[str, Any]]]]
output: OutputType = {'Pipeline Details': dict(table)}
params = [dict(zip(headers, item)) for item in data]
output['Pipeline Parameters'] = params
print_output(output, [], output_format)
def _display_experiments(experiments: List[ApiExperiment],
output_format: OutputFormat):
headers = ["Experiment ID", "Name", "Created at"]
data = [[exp.id, exp.name, exp.created_at.isoformat()]
for exp in experiments]
print_output(data, headers, output_format, table_format="grid")
def create(ctx: click.Context,
job_name: str,
experiment_id: Optional[str] = None,
experiment_name: Optional[str] = None,
catchup: Optional[bool] = None,
cron_expression: Optional[str] = None,
enabled: Optional[bool] = None,
description: Optional[str] = None,
enable_caching: Optional[bool] = None,
end_time: Optional[str] = None,
interval_second: Optional[int] = None,
max_concurrency: Optional[int] = None,
pipeline_package_path: Optional[str] = None,
pipeline_id: Optional[str] = None,
start_time: Optional[str] = None,
version_id: Optional[str] = None,
args: Optional[List[str]] = None):
"""Create a recurring run."""
client_obj: client.Client = ctx.obj['client']
output_format = ctx.obj['output']
if enable_caching is not None:
click.echo(
'--enable-caching and --disable-caching options are not yet supported.'
)
enable_caching = None
if (interval_second is None) == (cron_expression is None):
raise ValueError(
'Either of --interval-second or --cron-expression options is required.'
)
if (experiment_id is None) == (experiment_name is None):
raise ValueError(either_option_required)
if not experiment_id:
experiment = client_obj.create_experiment(experiment_name)
experiment_id = experiment.id
# Ensure we only split on the first equals char so the value can contain
# equals signs too.
split_args: List = [arg.split('=', 1) for arg in args or []]
params: Dict[str, Any] = dict(split_args)
recurring_run = client_obj.create_recurring_run(
cron_expression=cron_expression,
description=description,
enabled=enabled,
enable_caching=enable_caching,
end_time=end_time,
experiment_id=experiment_id,
interval_second=interval_second,
job_name=job_name,
max_concurrency=max_concurrency,
no_catchup=not catchup,
params=params,
pipeline_package_path=pipeline_package_path,
pipeline_id=pipeline_id,
start_time=start_time,
version_id=version_id)
output.print_output(
recurring_run,
output.ModelType.JOB,
output_format,
)
