def list_pipelines(filter_dict: dict = {},
sort_by: str = None) -> [ApiPipeline]:
api_client = get_swagger_client()
api_instance = swagger_client.PipelineServiceApi(api_client=api_client)
try:
filter_str = json.dumps(filter_dict) if filter_dict else None
api_response: ApiListPipelinesResponse = api_instance.list_pipelines(
filter=filter_str, sort_by=sort_by)
for c in api_response.pipelines:
print("%s %s %s" %
(c.id, c.created_at.strftime("%Y-%m-%d %H:%M:%S"), c.name))
return api_response.pipelines
except ApiException as e:
print(
"Exception when calling PipelineServiceApi -> list_pipelines: %s\n"
% e,
file=stderr)
return []
def upload_pipeline_template(uploadfile_name,
name: str = None,
description: str = None,
annotations: str = "") -> str:
api_client = get_swagger_client()
api_instance = swagger_client.PipelineServiceApi(api_client=api_client)
try:
pipeline: ApiPipeline = api_instance.upload_pipeline(
uploadfile=uploadfile_name,
name=name,
description=description,
annotations=annotations)
print(f"Uploaded '{pipeline.name}': {pipeline.id}")
return pipeline.id
except ApiException as e:
print(
"Exception when calling PipelineServiceApi -> upload_pipeline: %s\n"
% e,
file=stderr)
raise e
return None
def verify_pipeline_download(pipeline_id: str) -> bool:
api_client = get_swagger_client()
api_instance = swagger_client.PipelineServiceApi(api_client=api_client)
try:
response: HTTPResponse = api_instance.download_pipeline_files(
pipeline_id, _preload_content=False)
tgz_file = BytesIO(response.read())
with tarfile.open(fileobj=tgz_file) as tar:
file_contents = {
m.name: tar.extractfile(m).read().decode("utf-8")
for m in tar.getmembers()
}
template_response: ApiGetTemplateResponse = api_instance.get_template(
pipeline_id)
template_text_from_api = template_response.template
assert template_text_from_api == file_contents.get("pipeline.yaml")
print("downloaded files match")
return True
except ApiException as e:
print(
"Exception when calling PipelineServiceApi -> download_pipeline_files: %s\n"
% e,
file=stderr)
return False
def get_template(template_id: str) -> str:
api_client = get_swagger_client()
api_instance = swagger_client.PipelineServiceApi(api_client=api_client)
try:
template_response: ApiGetTemplateResponse = api_instance.get_template(
template_id)
print(template_response.template)
# yaml_dict = yaml.load(template_response.template, Loader=yaml.FullLoader)
# pipeline_name = yaml_dict.get("name") or f"template_{template_id}"
# template_file = os.path.join("files", pipeline_name) + ".yaml"
# with open(template_file, "w") as f:
# f.write(template_response.template)
return template_response.template
except ApiException as e:
print(
"Exception when calling PipelineServiceApi -> get_pipeline_template: %s\n"
% e,
file=stderr)
return None
def delete_assets(upload_assets_response: ApiCatalogUploadResponse = None):
api_client = get_swagger_client()
delete_methods = {
"components":
swagger_client.ComponentServiceApi(api_client).delete_component,
"datasets":
swagger_client.DatasetServiceApi(api_client).delete_dataset,
"models": swagger_client.ModelServiceApi(api_client).delete_model,
"notebooks":
swagger_client.NotebookServiceApi(api_client).delete_notebook,
"pipelines":
swagger_client.PipelineServiceApi(api_client).delete_pipeline
}
try:
for asset_type, delete_method in delete_methods.items():
if upload_assets_response:
asset_list = upload_assets_response.__getattribute__(
asset_type)
for asset in asset_list:
delete_method(asset.id)
else:
delete_method("*")
except ApiException as e:
print(f"Exception when calling {delete_method}: {e}\n", file=stderr)
def download_pipeline_tgz(pipeline_id) -> str:
api_client = get_swagger_client()
api_instance = swagger_client.PipelineServiceApi(api_client=api_client)
try:
response: HTTPResponse = \
api_instance.download_pipeline_files(pipeline_id, _preload_content=False)
attachment_header = response.info().get(
"Content-Disposition", f"attachment; filename={pipeline_id}.tgz")
download_filename = re.sub("attachment; filename=", "",
attachment_header)
download_dir = os.path.join(tempfile.gettempdir(), "download",
"pipelines")
os.makedirs(download_dir, exist_ok=True)
tarfile_path = os.path.join(download_dir, download_filename)
with open(tarfile_path, 'wb') as f:
f.write(response.read())
print(tarfile_path)
return tarfile_path
except ApiException as e:
print(
"Exception when calling PipelineServiceApi -> download_pipeline_files: %s\n"
% e,
file=stderr)
return "Download failed?"
def run_custom_pipeline(pipeline_template: dict, parameters: list = None, run_name: str = None) -> str:
api_client = get_swagger_client()
api_instance = swagger_client.PipelineServiceApi(api_client=api_client)
try:
# TODO: cleanup pipeline parameter, we should not support KFP Argo YAML
if pipeline_template.get("spec", {}).get("templates"):
# pipeline_dag = [t for t in pipeline_template["spec"]["templates"] if "dag" in t][0]
pipeline_dag = list(filter(lambda t: "dag" in t, pipeline_template["spec"]["templates"]))[0]
elif "dag" in pipeline_template:
pipeline_dag = pipeline_template
# custom_pipeline = ApiPipelineCustom.from_dict(pipeline_dag)
mock_response = custom_obj(data=json.dumps(pipeline_dag))
custom_pipeline = api_client.deserialize(response=mock_response, response_type=ApiPipelineCustom)
run_custom_pipeline_payload = ApiPipelineCustomRunPayload(custom_pipeline=custom_pipeline,
run_parameters=parameters)
run_code_response: ApiRunCodeResponse = \
api_instance.run_custom_pipeline(run_custom_pipeline_payload=run_custom_pipeline_payload,
run_name=run_name)
print(run_code_response.run_url)
return run_code_response.run_url
except ApiException as e:
print("Exception while calling PipelineServiceApi -> run_pipeline: %s\n" % e, file=stderr)
return None
def delete_pipeline(pipeline_id: str):
api_client = get_swagger_client()
api_instance = swagger_client.PipelineServiceApi(api_client=api_client)
try:
api_instance.delete_pipeline(pipeline_id)
except ApiException as e:
print("Exception when calling PipelineServiceApi -> delete_pipeline: %s\n" % e, file=stderr)
def set_featured_pipelines(pipeline_ids: [str]):
api_client = get_swagger_client()
api_instance = swagger_client.PipelineServiceApi(api_client=api_client)
try:
api_response = api_instance.set_featured_pipelines(pipeline_ids)
except ApiException as e:
print("Exception when calling PipelineServiceApi -> set_featured_pipelines: %s\n" % e, file=stderr)
return None
def approve_pipelines_for_publishing(pipeline_ids: [str]):
api_client = get_swagger_client()
api_instance = swagger_client.PipelineServiceApi(api_client=api_client)
try:
api_response = api_instance.approve_pipelines_for_publishing(pipeline_ids)
except ApiException as e:
print("Exception when calling PipelineServiceApi -> approve_pipelines_for_publishing: %s\n" % e, file=stderr)
return None
def upload_pipeline_file(pipeline_id, file_path):
api_client = get_swagger_client()
api_instance = swagger_client.PipelineServiceApi(api_client=api_client)
try:
response = api_instance.upload_pipeline_file(id=pipeline_id, uploadfile=file_path)
print(f"Upload file '{file_path}' to pipeline with ID '{pipeline_id}'")
except ApiException as e:
print("Exception when calling PipelineServiceApi -> upload_pipeline_file: %s\n" % e, file=stderr)
raise e
def get_pipeline(pipeline_id: str) -> ApiPipelineExtended:
api_client = get_swagger_client()
api_instance = swagger_client.PipelineServiceApi(api_client=api_client)
try:
pipeline_meta: ApiPipelineExtended = api_instance.get_pipeline(pipeline_id)
pprint(pipeline_meta, indent=2)
return pipeline_meta
except ApiException as e:
print("Exception when calling PipelineServiceApi -> get_pipeline: %s\n" % e, file=stderr)
return None
def run_pipeline(pipeline_id: str, parameters: dict = None, run_name: str = None) -> str:
api_client = get_swagger_client()
api_instance = swagger_client.PipelineServiceApi(api_client=api_client)
try:
run_code_response: ApiRunCodeResponse = api_instance.run_pipeline(pipeline_id, parameters=parameters,
run_name=run_name)
print(run_code_response.run_url)
return run_code_response.run_url
except ApiException as e:
print("Exception while calling PipelineServiceApi -> run_pipeline: %s\n" % e, file=stderr)
return None
