def generate_dataset_code(id): # noqa: E501 """generate_dataset_code Generate sample code to use dataset in a pipeline. :param id: :type id: str :rtype: ApiGenerateCodeResponse """ api_datasets: [ApiDataset] = load_data(ApiDataset, filter_dict={"id": id}) if not api_datasets: return f"Dataset with id '{id}' does not exist", 404 api_dataset = api_datasets[0] # TODO: could there be multiple pipeline DSL scripts, how to choose source_code = retrieve_file_content(bucket_name="mlpipeline", prefix=f"datasets/{id}/", file_extensions=[".py"]) if not source_code: api_template, _ = get_dataset_template(id) source_code = generate_dataset_run_script(api_dataset, api_template.url) if source_code: generate_code_response = ApiGenerateCodeResponse(script=source_code) return generate_code_response, 200 return f"Could not generate source code for dataset {id}", 500
def generate_component_code(id): # noqa: E501 """generate_component_code :param id: :type id: str :rtype: ApiGenerateCodeResponse """ api_components: [ApiComponent] = load_data(ApiComponent, filter_dict={"id": id}) if not api_components: return f"Component with id '{id}' does not exist", 404 api_component = api_components[0] # TODO: re-enable check for uploaded script, until then save time by not doing Minio lookup # source_code = retrieve_file_content(bucket_name="mlpipeline", prefix=f"components/{id}/", # file_extensions=[".py"]) source_code = None if not source_code: api_template, _ = get_component_template(id) source_code = generate_component_run_script(api_component, api_template.url) if source_code: generate_code_response = ApiGenerateCodeResponse(script=source_code) return generate_code_response, 200 else: return f"Could not generate source code for component {id}", 500
def generate_notebook_code(id): # noqa: E501 """generate_notebook_code :param id: :type id: str :rtype: ApiGenerateCodeResponse """ api_notebooks: [ApiNotebook] = load_data(ApiNotebook, filter_dict={"id": id}) if not api_notebooks: return f"Notebook with id '{id}' does not exist", 404 api_notebook = api_notebooks[0] source_code = retrieve_file_content(bucket_name="mlpipeline", prefix=f"notebooks/{id}/", file_extensions=[".py"]) if not source_code: source_code = generate_notebook_run_script(api_notebook) if source_code: generate_code_response = ApiGenerateCodeResponse(script=source_code) return generate_code_response, 200 else: return f"Could not generate source code for notebook {id}", 500
def get_dataset(id): # noqa: E501 """get_dataset :param id: :type id: str :rtype: ApiDataset """ api_datasets: [ApiDataset] = load_data(ApiDataset, filter_dict={"id": id}) if not api_datasets: return "Not found", 404 return api_datasets[0], 200
def get_model(id): # noqa: E501 """get_model :param id: :type id: str :rtype: ApiModel """ api_models: [ApiModel] = load_data(ApiModel, filter_dict={"id": id}) if not api_models: return "Not found", 404 return api_models[0], 200
def get_notebook(id): # noqa: E501 """get_notebook :param id: :type id: str :rtype: ApiNotebook """ api_notebooks: [ApiNotebook] = load_data(ApiNotebook, filter_dict={"id": id}) if not api_notebooks: return "Not found", 404 return api_notebooks[0], 200
def generate_model_code(id): # noqa: E501 """generate_model_code # noqa: E501 :param id: :type id: str :rtype: ApiGenerateModelCodeResponse """ api_models: [ApiModel] = load_data(ApiModel, filter_dict={"id": id}) if not api_models: return f"Model with id '{id}' does not exist", 404 api_model = api_models[0] generate_code_response = ApiGenerateModelCodeResponse(scripts=[]) source_combinations = [] if api_model.trainable: source_combinations.extend([ ("train", p) for p in api_model.trainable_tested_platforms ]) if api_model.servable: source_combinations.extend([ ("serve", p) for p in api_model.servable_tested_platforms ]) for stage, platform in source_combinations: # TODO: re-enable check for uploaded script, until then save time by not doing Minio lookup # source_code = retrieve_file_content(bucket_name="mlpipeline", prefix=f"models/{id}/", # file_extensions=[".py"], # file_name_filter=f"{stage}_{platform}") source_code = None if not source_code: source_code = generate_model_run_script(api_model, stage, platform) api_model_script = ApiModelScript(pipeline_stage=stage, execution_platform=platform, script_code=source_code) generate_code_response.scripts.append(api_model_script) return generate_code_response, 200
def get_pipeline(id): # noqa: E501 """get_pipeline :param id: :type id: str :rtype: ApiPipelineExtended """ api_pipelines: [ApiPipelineExtended] = load_data(ApiPipelineExtended, filter_dict={"id": id}) if not api_pipelines: return "Not found", 404 api_pipeline = api_pipelines[0] return api_pipeline, 200
def list_credentials(page_token=None, page_size=None, sort_by=None, filter=None): # noqa: E501 """list_credentials :param page_token: :type page_token: str :param page_size: :type page_size: int :param sort_by: Can be format of \"field_name\", \"field_name asc\" or \"field_name des\" Ascending by default. :type sort_by: str :param filter: A string-serialized JSON dictionary containing key-value pairs with name of the object property to apply filter on and the value of the respective property. :type filter: str :rtype: ApiListCredentialsResponse """ if page_size == 0: return {}, 200 # TODO: do not misuse page_token as MySQL result offset offset = int(page_token) if page_token and page_token.isdigit() else 0 filter_dict = json.loads(filter) if filter else None api_credentials: [ApiCredential] = load_data(ApiCredential, filter_dict=filter_dict, sort_by=sort_by, count=page_size, offset=offset) next_page_token = offset + page_size if len( api_credentials) == page_size else None total_size = num_rows(ApiCredential) if total_size == next_page_token: next_page_token = None secrets = list_secrets(name_prefix=secret_name_prefix) # TODO: consolidate kubernetes secrets with MLX registered credentials (i.e. add status field?) comp_list = ApiListCredentialsResponse(credentials=api_credentials, total_size=total_size, next_page_token=next_page_token) return comp_list, 200
def get_component(id): # noqa: E501 """get_component :param id: :type id: str :rtype: ApiComponent """ api_components: [ApiComponent] = load_data(ApiComponent, filter_dict={"id": id}) if not api_components: return "Not found", 404 api_component = api_components[0] return api_component, 200
def get_credential(id): # noqa: E501 """get_credential :param id: :type id: str :rtype: ApiComponent """ api_credentials: [ApiCredential] = load_data(ApiCredential, filter_dict={"id": id}) if not api_credentials: return "Not found", 404 api_credential = api_credentials[0] secret = get_secret(id) return api_credential, 200
def list_notebooks(page_token=None, page_size=None, sort_by=None, filter=None): # noqa: E501 """list_notebooks :param page_token: :type page_token: str :param page_size: :type page_size: int :param sort_by: Can be format of \"field_name\", \"field_name asc\" or \"field_name des\" Ascending by default. :type sort_by: str :param filter: A string-serialized JSON dictionary containing key-value pairs with name of the object property to apply filter on and the value of the respective property. :type filter: str :rtype: ApiListNotebooksResponse """ if page_size == 0: return {}, 200 # TODO: do not misuse page_token as MySQL result offset offset = int(page_token) if page_token and page_token.isdigit() else 0 filter_dict = json.loads(filter) if filter else None api_notebooks: [ApiNotebook] = load_data(ApiNotebook, filter_dict=filter_dict, sort_by=sort_by, count=page_size, offset=offset) next_page_token = offset + page_size if len( api_notebooks) == page_size else None total_size = num_rows(ApiNotebook) if total_size == next_page_token: next_page_token = None notebooks = ApiListNotebooksResponse(notebooks=api_notebooks, total_size=total_size, next_page_token=next_page_token) return notebooks, 200
def list_pipelines(page_token=None, page_size=None, sort_by=None, filter=None): # noqa: E501 """list_pipelines :param page_token: :type page_token: str :param page_size: :type page_size: int :param sort_by: Can be format of \"field_name\", \"field_name asc\" or \"field_name des\" Ascending by default. :type sort_by: str :param filter: A string-serialized JSON dictionary containing key-value pairs with name of the object property to apply filter on and the value of the respective property. :type filter: str :rtype: ApiListPipelinesResponse """ if page_size == 0: return {}, 200 # TODO: do not misuse page_token as MySQL result offset offset = int(page_token) if page_token and page_token.isdigit() else 0 filter_dict = json.loads(filter) if filter else {} # TODO: add filter_categories to ApiPipelineExtension (and give users a way # to add category labels to pipelines) until then remove categories from filter if "filter_categories" in filter_dict: del filter_dict["filter_categories"] api_pipelines: [ApiPipeline] = load_data(ApiPipelineExtended, filter_dict=filter_dict, sort_by=sort_by, count=page_size, offset=offset) next_page_token = offset + page_size if len(api_pipelines) == page_size else None total_size = num_rows(ApiPipeline) if total_size == next_page_token: next_page_token = None pipeline_list = ApiListPipelinesResponse(pipelines=api_pipelines, total_size=total_size, next_page_token=next_page_token) return pipeline_list, 200
def approve_pipelines_for_publishing(pipeline_ids): # noqa: E501 """approve_pipelines_for_publishing :param pipeline_ids: Array of pipeline IDs to be approved for publishing. :type pipeline_ids: List[str] :rtype: None """ pipe_exts: [ApiPipelineExtension] = load_data(ApiPipelineExtension) pipe_ext_ids = {p.id for p in pipe_exts} missing_pipe_ext_ids = set(pipeline_ids) - pipe_ext_ids for id in missing_pipe_ext_ids: store_data(ApiPipelineExtension(id=id)) update_multiple(ApiPipelineExtension, [], "publish_approved", False) if pipeline_ids: update_multiple(ApiPipelineExtension, pipeline_ids, "publish_approved", True) return None, 200
def set_featured_pipelines(pipeline_ids): # noqa: E501 """set_featured_pipelines :param pipeline_ids: Array of pipeline IDs to be featured. :type pipeline_ids: List[str] :rtype: None """ pipe_exts: [ApiPipelineExtension] = load_data(ApiPipelineExtension) pipe_ext_ids = {p.id for p in pipe_exts} missing_pipe_ext_ids = set(pipeline_ids) - pipe_ext_ids for id in missing_pipe_ext_ids: store_data(ApiPipelineExtension(id=id)) update_multiple(ApiPipelineExtension, [], "featured", False) if pipeline_ids: update_multiple(ApiPipelineExtension, pipeline_ids, "featured", True) return None, 200