def generate_dataset_code(id): # noqa: E501
"""generate_dataset_code
Generate sample code to use dataset in a pipeline.
:param id:
:type id: str
:rtype: ApiGenerateCodeResponse
"""
api_datasets: [ApiDataset] = load_data(ApiDataset, filter_dict={"id": id})
if not api_datasets:
return f"Dataset with id '{id}' does not exist", 404
api_dataset = api_datasets[0]
# TODO: could there be multiple pipeline DSL scripts, how to choose
source_code = retrieve_file_content(bucket_name="mlpipeline",
prefix=f"datasets/{id}/",
file_extensions=[".py"])
if not source_code:
api_template, _ = get_dataset_template(id)
source_code = generate_dataset_run_script(api_dataset,
api_template.url)
if source_code:
generate_code_response = ApiGenerateCodeResponse(script=source_code)
return generate_code_response, 200
return f"Could not generate source code for dataset {id}", 500
def generate_component_code(id): # noqa: E501
"""generate_component_code
:param id:
:type id: str
:rtype: ApiGenerateCodeResponse
"""
api_components: [ApiComponent] = load_data(ApiComponent,
filter_dict={"id": id})
if not api_components:
return f"Component with id '{id}' does not exist", 404
api_component = api_components[0]
# TODO: re-enable check for uploaded script, until then save time by not doing Minio lookup
# source_code = retrieve_file_content(bucket_name="mlpipeline", prefix=f"components/{id}/",
# file_extensions=[".py"])
source_code = None
if not source_code:
api_template, _ = get_component_template(id)
source_code = generate_component_run_script(api_component,
api_template.url)
if source_code:
generate_code_response = ApiGenerateCodeResponse(script=source_code)
return generate_code_response, 200
else:
return f"Could not generate source code for component {id}", 500
def generate_notebook_code(id): # noqa: E501
"""generate_notebook_code
:param id:
:type id: str
:rtype: ApiGenerateCodeResponse
"""
api_notebooks: [ApiNotebook] = load_data(ApiNotebook,
filter_dict={"id": id})
if not api_notebooks:
return f"Notebook with id '{id}' does not exist", 404
api_notebook = api_notebooks[0]
source_code = retrieve_file_content(bucket_name="mlpipeline",
prefix=f"notebooks/{id}/",
file_extensions=[".py"])
if not source_code:
source_code = generate_notebook_run_script(api_notebook)
if source_code:
generate_code_response = ApiGenerateCodeResponse(script=source_code)
return generate_code_response, 200
else:
return f"Could not generate source code for notebook {id}", 500
def get_dataset(id): # noqa: E501
"""get_dataset
:param id:
:type id: str
:rtype: ApiDataset
"""
api_datasets: [ApiDataset] = load_data(ApiDataset, filter_dict={"id": id})
if not api_datasets:
return "Not found", 404
return api_datasets[0], 200
def get_model(id): # noqa: E501
"""get_model
:param id:
:type id: str
:rtype: ApiModel
"""
api_models: [ApiModel] = load_data(ApiModel, filter_dict={"id": id})
if not api_models:
return "Not found", 404
return api_models[0], 200
def get_notebook(id): # noqa: E501
"""get_notebook
:param id:
:type id: str
:rtype: ApiNotebook
"""
api_notebooks: [ApiNotebook] = load_data(ApiNotebook,
filter_dict={"id": id})
if not api_notebooks:
return "Not found", 404
return api_notebooks[0], 200
def generate_model_code(id): # noqa: E501
"""generate_model_code
# noqa: E501
:param id:
:type id: str
:rtype: ApiGenerateModelCodeResponse
"""
api_models: [ApiModel] = load_data(ApiModel, filter_dict={"id": id})
if not api_models:
return f"Model with id '{id}' does not exist", 404
api_model = api_models[0]
generate_code_response = ApiGenerateModelCodeResponse(scripts=[])
source_combinations = []
if api_model.trainable:
source_combinations.extend([
("train", p) for p in api_model.trainable_tested_platforms
])
if api_model.servable:
source_combinations.extend([
("serve", p) for p in api_model.servable_tested_platforms
])
for stage, platform in source_combinations:
# TODO: re-enable check for uploaded script, until then save time by not doing Minio lookup
# source_code = retrieve_file_content(bucket_name="mlpipeline", prefix=f"models/{id}/",
# file_extensions=[".py"],
# file_name_filter=f"{stage}_{platform}")
source_code = None
if not source_code:
source_code = generate_model_run_script(api_model, stage, platform)
api_model_script = ApiModelScript(pipeline_stage=stage,
execution_platform=platform,
script_code=source_code)
generate_code_response.scripts.append(api_model_script)
return generate_code_response, 200
def get_pipeline(id): # noqa: E501
"""get_pipeline
:param id:
:type id: str
:rtype: ApiPipelineExtended
"""
api_pipelines: [ApiPipelineExtended] = load_data(ApiPipelineExtended, filter_dict={"id": id})
if not api_pipelines:
return "Not found", 404
api_pipeline = api_pipelines[0]
return api_pipeline, 200
def list_credentials(page_token=None,
page_size=None,
sort_by=None,
filter=None): # noqa: E501
"""list_credentials
:param page_token:
:type page_token: str
:param page_size:
:type page_size: int
:param sort_by: Can be format of \"field_name\", \"field_name asc\" or \"field_name des\" Ascending by default.
:type sort_by: str
:param filter: A string-serialized JSON dictionary containing key-value pairs with name of the object property to apply filter on and the value of the respective property.
:type filter: str
:rtype: ApiListCredentialsResponse
"""
if page_size == 0:
return {}, 200
# TODO: do not misuse page_token as MySQL result offset
offset = int(page_token) if page_token and page_token.isdigit() else 0
filter_dict = json.loads(filter) if filter else None
api_credentials: [ApiCredential] = load_data(ApiCredential,
filter_dict=filter_dict,
sort_by=sort_by,
count=page_size,
offset=offset)
next_page_token = offset + page_size if len(
api_credentials) == page_size else None
total_size = num_rows(ApiCredential)
if total_size == next_page_token:
next_page_token = None
secrets = list_secrets(name_prefix=secret_name_prefix)
# TODO: consolidate kubernetes secrets with MLX registered credentials (i.e. add status field?)
comp_list = ApiListCredentialsResponse(credentials=api_credentials,
total_size=total_size,
next_page_token=next_page_token)
return comp_list, 200
def get_component(id): # noqa: E501
"""get_component
:param id:
:type id: str
:rtype: ApiComponent
"""
api_components: [ApiComponent] = load_data(ApiComponent,
filter_dict={"id": id})
if not api_components:
return "Not found", 404
api_component = api_components[0]
return api_component, 200
def get_credential(id): # noqa: E501
"""get_credential
:param id:
:type id: str
:rtype: ApiComponent
"""
api_credentials: [ApiCredential] = load_data(ApiCredential, filter_dict={"id": id})
if not api_credentials:
return "Not found", 404
api_credential = api_credentials[0]
secret = get_secret(id)
return api_credential, 200
def list_notebooks(page_token=None,
page_size=None,
sort_by=None,
filter=None): # noqa: E501
"""list_notebooks
:param page_token:
:type page_token: str
:param page_size:
:type page_size: int
:param sort_by: Can be format of \"field_name\", \"field_name asc\" or \"field_name des\" Ascending by default.
:type sort_by: str
:param filter: A string-serialized JSON dictionary containing key-value pairs with name of the object property to apply filter on and the value of the respective property.
:type filter: str
:rtype: ApiListNotebooksResponse
"""
if page_size == 0:
return {}, 200
# TODO: do not misuse page_token as MySQL result offset
offset = int(page_token) if page_token and page_token.isdigit() else 0
filter_dict = json.loads(filter) if filter else None
api_notebooks: [ApiNotebook] = load_data(ApiNotebook,
filter_dict=filter_dict,
sort_by=sort_by,
count=page_size,
offset=offset)
next_page_token = offset + page_size if len(
api_notebooks) == page_size else None
total_size = num_rows(ApiNotebook)
if total_size == next_page_token:
next_page_token = None
notebooks = ApiListNotebooksResponse(notebooks=api_notebooks,
total_size=total_size,
next_page_token=next_page_token)
return notebooks, 200
def list_pipelines(page_token=None, page_size=None, sort_by=None, filter=None): # noqa: E501
"""list_pipelines
:param page_token:
:type page_token: str
:param page_size:
:type page_size: int
:param sort_by: Can be format of \"field_name\", \"field_name asc\" or \"field_name des\" Ascending by default.
:type sort_by: str
:param filter: A string-serialized JSON dictionary containing key-value pairs with name of the object property to apply filter on and the value of the respective property.
:type filter: str
:rtype: ApiListPipelinesResponse
"""
if page_size == 0:
return {}, 200
# TODO: do not misuse page_token as MySQL result offset
offset = int(page_token) if page_token and page_token.isdigit() else 0
filter_dict = json.loads(filter) if filter else {}
# TODO: add filter_categories to ApiPipelineExtension (and give users a way
# to add category labels to pipelines) until then remove categories from filter
if "filter_categories" in filter_dict:
del filter_dict["filter_categories"]
api_pipelines: [ApiPipeline] = load_data(ApiPipelineExtended, filter_dict=filter_dict,
sort_by=sort_by, count=page_size, offset=offset)
next_page_token = offset + page_size if len(api_pipelines) == page_size else None
total_size = num_rows(ApiPipeline)
if total_size == next_page_token:
next_page_token = None
pipeline_list = ApiListPipelinesResponse(pipelines=api_pipelines, total_size=total_size,
next_page_token=next_page_token)
return pipeline_list, 200
def approve_pipelines_for_publishing(pipeline_ids): # noqa: E501
"""approve_pipelines_for_publishing
:param pipeline_ids: Array of pipeline IDs to be approved for publishing.
:type pipeline_ids: List[str]
:rtype: None
"""
pipe_exts: [ApiPipelineExtension] = load_data(ApiPipelineExtension)
pipe_ext_ids = {p.id for p in pipe_exts}
missing_pipe_ext_ids = set(pipeline_ids) - pipe_ext_ids
for id in missing_pipe_ext_ids:
store_data(ApiPipelineExtension(id=id))
update_multiple(ApiPipelineExtension, [], "publish_approved", False)
if pipeline_ids:
update_multiple(ApiPipelineExtension, pipeline_ids, "publish_approved", True)
return None, 200
def set_featured_pipelines(pipeline_ids): # noqa: E501
"""set_featured_pipelines
:param pipeline_ids: Array of pipeline IDs to be featured.
:type pipeline_ids: List[str]
:rtype: None
"""
pipe_exts: [ApiPipelineExtension] = load_data(ApiPipelineExtension)
pipe_ext_ids = {p.id for p in pipe_exts}
missing_pipe_ext_ids = set(pipeline_ids) - pipe_ext_ids
for id in missing_pipe_ext_ids:
store_data(ApiPipelineExtension(id=id))
update_multiple(ApiPipelineExtension, [], "featured", False)
if pipeline_ids:
update_multiple(ApiPipelineExtension, pipeline_ids, "featured", True)
return None, 200
