def _display_experiments(experiments: List[ApiExperiment], output_format: OutputFormat): headers = ['Experiment ID', 'Name', 'Created at'] data = [ [exp.id, exp.name, exp.created_at.isoformat()] for exp in experiments ] print_output(data, headers, output_format, table_format='grid')
def create_version(ctx: click.Context, package_file: str, pipeline_version: str, pipeline_id: Optional[str] = None, pipeline_name: Optional[str] = None, description: Optional[str] = None): """Upload a version of a pipeline.""" client_obj: client.Client = ctx.obj['client'] output_format = ctx.obj['output'] if bool(pipeline_id) == bool(pipeline_name): raise ValueError(either_option_required) if pipeline_name is not None: pipeline_id = client_obj.get_pipeline_id(name=pipeline_name) if pipeline_id is None: raise ValueError( f"Can't find a pipeline with name: {pipeline_name}") version = client_obj.upload_pipeline_version( pipeline_package_path=package_file, pipeline_version_name=pipeline_version, pipeline_id=pipeline_id, pipeline_name=pipeline_name, description=description) output.print_output( version, output.ModelType.PIPELINE, output_format, )
def _print_pipeline_versions(versions, output_format): headers = ["Version ID", "Version name", "Uploaded at"] data = [[ version.id, version.name, version.created_at.isoformat() ] for version in versions] print_output(data, headers, output_format, table_format="grid")
def _print_pipelines(pipelines, output_format): headers = ["Pipeline ID", "Name", "Uploaded at"] data = [[ pipeline.id, pipeline.name, pipeline.created_at.isoformat() ] for pipeline in pipelines] print_output(data, headers, output_format, table_format="grid")
def _print_pipeline_versions(versions: List[kfp_server_api.ApiPipelineVersion], output_format: OutputFormat): headers = ['Version ID', 'Version name', 'Uploaded at', 'Pipeline ID'] data = [[ version.id, version.name, version.created_at.isoformat(), next(rr for rr in version.resource_references if rr.key.type == kfp_server_api.ApiResourceType.PIPELINE).key.id ] for version in versions] print_output(data, headers, output_format, table_format='grid')
def unarchive(ctx: click.Context, run_id: str): """Unarchive a pipeline run.""" client_obj: client.Client = ctx.obj['client'] output_format = ctx.obj['output'] client_obj.unarchive_run(run_id=run_id) run = client_obj.get_run(run_id=run_id) output.print_output( run.run, output.ModelType.RUN, output_format, )
def get(ctx: click.Context, job_id: str): """Get information about a recurring run.""" client_obj: client.Client = ctx.obj['client'] output_format = ctx.obj['output'] recurring_run = client_obj.get_recurring_run(job_id) output.print_output( recurring_run, output.ModelType.JOB, output_format, )
def get(ctx: click.Context, experiment_id: str): """Get information about an experiment.""" client_obj: client.Client = ctx.obj['client'] output_format = ctx.obj['output'] experiment = client_obj.get_experiment(experiment_id) output.print_output( experiment, output.ModelType.EXPERIMENT, output_format, )
def create(ctx: click.Context, description: str, name: str): """Create an experiment.""" client_obj: client.Client = ctx.obj['client'] output_format = ctx.obj['output'] experiment = client_obj.create_experiment(name, description=description) output.print_output( experiment, output.ModelType.EXPERIMENT, output_format, )
def _print_runs(runs: List[kfp_server_api.ApiRun], output_format: OutputFormat): headers = ['run id', 'name', 'status', 'created at', 'experiment id'] data = [[ run.id, run.name, run.status, run.created_at.isoformat(), next(rr for rr in run.resource_references if rr.key.type == kfp_server_api.ApiResourceType.EXPERIMENT).key.id ] for run in runs] print_output(data, headers, output_format, table_format='grid')
def get(ctx: click.Context, pipeline_id: str): """Get information about a pipeline.""" client_obj: client.Client = ctx.obj['client'] output_format = ctx.obj['output'] pipeline = client_obj.get_pipeline(pipeline_id) output.print_output( pipeline, output.ModelType.PIPELINE, output_format, )
def get_version(ctx: click.Context, version_id: str): """Get information about a version of a pipeline.""" client_obj: client.Client = ctx.obj['client'] output_format = ctx.obj['output'] version = client_obj.get_pipeline_version(version_id=version_id) output.print_output( version, output.ModelType.PIPELINE, output_format, )
def disable(ctx: click.Context, job_id: str): """Disable a recurring run.""" client_obj: client.Client = ctx.obj['client'] output_format = ctx.obj['output'] client_obj.disable_job(job_id=job_id) # TODO: add wait option, since disable takes time to complete recurring_run = client_obj.get_recurring_run(job_id=job_id) output.print_output( recurring_run, output.ModelType.JOB, output_format, )
def _display_pipeline(pipeline, output_format): # Pipeline information table = [ ["ID", pipeline.id], ["Name", pipeline.name], ["Description", pipeline.description], ["Uploaded at", pipeline.created_at.isoformat()], ] # Pipeline parameter details headers = ["Parameter Name", "Default Value"] data = [] if pipeline.parameters is not None: data = [[param.name, param.value] for param in pipeline.parameters] if output_format == OutputFormat.table.name: print_output([], ["Pipeline Details"], output_format) print_output(table, [], output_format, table_format="plain") print_output(data, headers, output_format, table_format="grid") elif output_format == OutputFormat.json.name: output = dict() output["Pipeline Details"] = dict(table) params = [] for item in data: params.append(dict(zip(headers, item))) output["Pipeline Parameters"] = params print_output(output, [], output_format)
def create(ctx: click.Context, pipeline_name: str, package_file: str, description: str = None): """Upload a pipeline.""" client_obj: client.Client = ctx.obj['client'] output_format = ctx.obj['output'] pipeline = client_obj.upload_pipeline(package_file, pipeline_name, description) output.print_output( pipeline, output.ModelType.PIPELINE, output_format, )
def list(ctx: click.Context, page_token: str, max_size: int, sort_by: str, filter: str): """List pipelines.""" client_obj: client.Client = ctx.obj['client'] output_format = ctx.obj['output'] response = client_obj.list_pipelines(page_token=page_token, page_size=max_size, sort_by=sort_by, filter=filter) output.print_output( response.pipelines or [], output.ModelType.PIPELINE, output_format, )
def list(ctx: click.Context, page_token: str, max_size: int, sort_by: str, filter: str): """List experiments.""" client_obj: client.Client = ctx.obj['client'] output_format = ctx.obj['output'] response = client_obj.list_experiments(page_token=page_token, page_size=max_size, sort_by=sort_by, filter=filter) output.print_output( response.experiments or [], output.ModelType.EXPERIMENT, output_format, )
def list(ctx: click.Context, experiment_id: str, page_token: str, max_size: int, sort_by: str, filter: str): """List pipeline runs.""" client_obj: client.Client = ctx.obj['client'] output_format = ctx.obj['output'] response = client_obj.list_runs( experiment_id=experiment_id, page_token=page_token, page_size=max_size, sort_by=sort_by, filter=filter) output.print_output( response.runs or [], output.ModelType.RUN, output_format, )
def display_run(client: client.Client, namespace: str, run_id: str, watch: bool, output_format: str): run = client.get_run(run_id).run output.print_output( run, output.ModelType.RUN, output_format, ) if not watch: return argo_path = shutil.which('argo') if not argo_path: raise RuntimeError( "argo isn't found in $PATH. It's necessary for watch. " "Please make sure it's installed and available. " 'Installation instructions be found here - ' 'https://github.com/argoproj/argo-workflows/releases') argo_workflow_name = None while True: time.sleep(1) run_detail = client.get_run(run_id) run = run_detail.run if run_detail.pipeline_runtime and run_detail.pipeline_runtime.workflow_manifest: manifest = json.loads(run_detail.pipeline_runtime.workflow_manifest) if manifest['metadata'] and manifest['metadata']['name']: argo_workflow_name = manifest['metadata']['name'] break if run_detail.run.status in ['Succeeded', 'Skipped', 'Failed', 'Error']: click.echo(f'Run is finished with status {run_detail.run.status}.') return if argo_workflow_name: subprocess.run( [argo_path, 'watch', argo_workflow_name, '-n', namespace]) output.print_output( run, output.ModelType.RUN, output_format, )
def unarchive(ctx: click.Context, experiment_id: str, experiment_name: str): """Unarchive an experiment.""" client_obj: client.Client = ctx.obj['client'] output_format = ctx.obj['output'] if (experiment_id is None) == (experiment_name is None): raise ValueError(either_option_required) if not experiment_id: experiment = client_obj.get_experiment(experiment_name=experiment_name) experiment_id = experiment.id client_obj.unarchive_experiment(experiment_id=experiment_id) if experiment_id: experiment = client_obj.get_experiment(experiment_id=experiment_id) else: experiment = client_obj.get_experiment(experiment_name=experiment_name) output.print_output( experiment, output.ModelType.EXPERIMENT, output_format, )
def create(ctx: click.Context, experiment_name: str, run_name: str, package_file: str, pipeline_id: str, pipeline_name: str, watch: bool, timeout: int, version: str, args: List[str]): """Submit a pipeline run.""" client_obj: client.Client = ctx.obj['client'] namespace = ctx.obj['namespace'] output_format = ctx.obj['output'] if not run_name: run_name = experiment_name if not pipeline_id and pipeline_name: pipeline_id = client_obj.get_pipeline_id(name=pipeline_name) if not package_file and not pipeline_id and not version: click.echo( 'You must provide one of [package_file, pipeline_id, version].', err=True) sys.exit(1) arg_dict = dict(arg.split('=', maxsplit=1) for arg in args) experiment = client_obj.create_experiment(experiment_name) run = client_obj.run_pipeline( experiment_id=experiment.id, job_name=run_name, pipeline_package_path=package_file, params=arg_dict, pipeline_id=pipeline_id, version_id=version) if timeout > 0: run_detail = client_obj.wait_for_run_completion(run.id, timeout) output.print_output( run_detail.run, output.ModelType.RUN, output_format, ) else: display_run(client_obj, namespace, run.id, watch, output_format)
def _display_recurring_run(recurring_run: kfp_server_api.ApiJob, output_format: OutputFormat): table = [ ["Recurring Run ID", recurring_run.id], ["Name", recurring_run.name], ] if output_format == OutputFormat.table.name: print_output([], ["Recurring Run Details"], output_format) print_output(table, [], output_format, table_format="plain") elif output_format == OutputFormat.json.name: print_output(dict(table), [], output_format)
def _display_experiment(exp: kfp_server_api.ApiExperiment, output_format: OutputFormat): table = [ ['ID', exp.id], ['Name', exp.name], ['Description', exp.description], ['Created at', exp.created_at.isoformat()], ] if output_format == OutputFormat.table.name: print_output([], ['Experiment Details'], output_format) print_output(table, [], output_format, table_format='plain') elif output_format == OutputFormat.json.name: print_output(dict(table), [], output_format)
def _display_pipeline_version(version, output_format): pipeline_id = version.resource_references[0].key.id table = [ ["Pipeline ID", pipeline_id], ["Version Name", version.name], ["Uploaded at", version.created_at.isoformat()], ] if output_format == OutputFormat.table.name: print_output([], ["Pipeline Version Details"], output_format) print_output(table, [], output_format, table_format="plain") elif output_format == OutputFormat.json.name: print_output(dict(table), [], output_format)
def _display_experiment(exp: kfp_server_api.ApiExperiment, output_format: OutputFormat): table = [ ["ID", exp.id], ["Name", exp.name], ["Description", exp.description], ["Created at", exp.created_at.isoformat()], ] if output_format == OutputFormat.table.name: print_output([], ["Experiment Details"], output_format) print_output(table, [], output_format, table_format="plain") elif output_format == OutputFormat.json.name: print_output(dict(table), [], output_format)
def _display_pipeline_version(version: kfp_server_api.ApiPipelineVersion, output_format: OutputFormat): pipeline_id = next( rr for rr in version.resource_references if rr.key.type == kfp_server_api.ApiResourceType.PIPELINE).key.id table = [['Pipeline ID', pipeline_id], ['Version name', version.name], ['Uploaded at', version.created_at.isoformat()], ['Version ID', version.id]] if output_format == OutputFormat.table.name: print_output([], ['Pipeline Version Details'], output_format) print_output(table, [], output_format, table_format='plain') elif output_format == OutputFormat.json.name: print_output(dict(table), [], output_format)
def _display_pipeline(pipeline: kfp_server_api.ApiPipeline, output_format: OutputFormat): # Pipeline information table = [["Pipeline ID", pipeline.id], ["Name", pipeline.name], ["Description", pipeline.description], ["Uploaded at", pipeline.created_at.isoformat()], ["Version ID", pipeline.default_version.id]] # Pipeline parameter details headers = ["Parameter Name", "Default Value"] data = [] if pipeline.parameters is not None: data = [[param.name, param.value] for param in pipeline.parameters] if output_format == OutputFormat.table.name: print_output([], ["Pipeline Details"], output_format) print_output(table, [], output_format, table_format="plain") print_output(data, headers, output_format, table_format="grid") elif output_format == OutputFormat.json.name: OutputType = Dict[str, Union[Dict[str, str], List[Dict[str, Any]]]] output: OutputType = {"Pipeline Details": dict(table)} params = [dict(zip(headers, item)) for item in data] output["Pipeline Parameters"] = params print_output(output, [], output_format)
def _display_pipeline(pipeline: kfp_server_api.ApiPipeline, output_format: OutputFormat): # Pipeline information table = [['Pipeline ID', pipeline.id], ['Name', pipeline.name], ['Description', pipeline.description], ['Uploaded at', pipeline.created_at.isoformat()], ['Version ID', pipeline.default_version.id]] # Pipeline parameter details headers = ['Parameter Name', 'Default Value'] data = [] if pipeline.parameters is not None: data = [[param.name, param.value] for param in pipeline.parameters] if output_format == OutputFormat.table.name: print_output([], ['Pipeline Details'], output_format) print_output(table, [], output_format, table_format='plain') print_output(data, headers, output_format, table_format='grid') elif output_format == OutputFormat.json.name: OutputType = Dict[str, Union[Dict[str, str], List[Dict[str, Any]]]] output: OutputType = {'Pipeline Details': dict(table)} params = [dict(zip(headers, item)) for item in data] output['Pipeline Parameters'] = params print_output(output, [], output_format)
def _display_experiments(experiments: List[ApiExperiment], output_format: OutputFormat): headers = ["Experiment ID", "Name", "Created at"] data = [[exp.id, exp.name, exp.created_at.isoformat()] for exp in experiments] print_output(data, headers, output_format, table_format="grid")
def create(ctx: click.Context, job_name: str, experiment_id: Optional[str] = None, experiment_name: Optional[str] = None, catchup: Optional[bool] = None, cron_expression: Optional[str] = None, enabled: Optional[bool] = None, description: Optional[str] = None, enable_caching: Optional[bool] = None, end_time: Optional[str] = None, interval_second: Optional[int] = None, max_concurrency: Optional[int] = None, pipeline_package_path: Optional[str] = None, pipeline_id: Optional[str] = None, start_time: Optional[str] = None, version_id: Optional[str] = None, args: Optional[List[str]] = None): """Create a recurring run.""" client_obj: client.Client = ctx.obj['client'] output_format = ctx.obj['output'] if enable_caching is not None: click.echo( '--enable-caching and --disable-caching options are not yet supported.' ) enable_caching = None if (interval_second is None) == (cron_expression is None): raise ValueError( 'Either of --interval-second or --cron-expression options is required.' ) if (experiment_id is None) == (experiment_name is None): raise ValueError(either_option_required) if not experiment_id: experiment = client_obj.create_experiment(experiment_name) experiment_id = experiment.id # Ensure we only split on the first equals char so the value can contain # equals signs too. split_args: List = [arg.split('=', 1) for arg in args or []] params: Dict[str, Any] = dict(split_args) recurring_run = client_obj.create_recurring_run( cron_expression=cron_expression, description=description, enabled=enabled, enable_caching=enable_caching, end_time=end_time, experiment_id=experiment_id, interval_second=interval_second, job_name=job_name, max_concurrency=max_concurrency, no_catchup=not catchup, params=params, pipeline_package_path=pipeline_package_path, pipeline_id=pipeline_id, start_time=start_time, version_id=version_id) output.print_output( recurring_run, output.ModelType.JOB, output_format, )