def upload_component_from_url(url, name=None, access_token=None): # noqa: E501 """upload_component_from_url :param url: URL pointing to the component YAML file. :type url: str :param name: Optional, the name of the component to be created overriding the name in the YAML file. :type name: str :param access_token: Optional, the Bearer token to access the 'url'. :type access_token: str :rtype: ApiComponent """ yaml_file_content = download_file_content_from_url(url, access_token) return _upload_component_yaml(yaml_file_content, name)
def upload_catalog_from_url(url, access_token=None): # noqa: E501 """upload_catalog_from_url # noqa: E501 :param url: URL pointing to the catalog YAML file. :type url: str :param access_token: Optional, the Bearer token to access the 'url'. :type access_token: str :rtype: ApiCatalogUploadResponse """ json_file_content = download_file_content_from_url(url, access_token) catalog_dict = json.loads(json_file_content) catalog = ApiCatalogUpload.from_dict(catalog_dict) return _upload_multiple_assets(catalog)
def _upload_notebook_yaml(yaml_file_content: AnyStr, name=None, access_token=None, existing_id=None): yaml_dict = yaml.load(yaml_file_content, Loader=yaml.FullLoader) template_metadata = yaml_dict.get("metadata") or dict() notebook_id = existing_id or generate_id(name=name or yaml_dict["name"]) created_at = datetime.now() name = name or yaml_dict["name"] description = yaml_dict["description"].strip() url = yaml_dict["implementation"]["github"]["source"] requirements = yaml_dict["implementation"]["github"].get("requirements") metadata = ApiMetadata(annotations=template_metadata.get("annotations"), labels=template_metadata.get("labels"), tags=template_metadata.get("tags")) notebook_content = _download_notebook( url, enterprise_github_api_token=access_token) # parameters = _extract_notebook_parameters(notebook_content) # TODO: not using Papermill any longer, notebook parameters no longer valid? # kfp-notebook has inputs and outputs ? parameters = dict() api_notebook = ApiNotebook(id=notebook_id, created_at=created_at, name=name, description=description, url=url, metadata=metadata, parameters=parameters) uuid = store_data(api_notebook) api_notebook.id = uuid store_file(bucket_name="mlpipeline", prefix=f"notebooks/{notebook_id}/", file_name="template.yaml", file_content=yaml_file_content) s3_url = store_file(bucket_name="mlpipeline", prefix=f"notebooks/{notebook_id}/", file_name=url.split("/")[-1].split("?")[0], file_content=json.dumps(notebook_content).encode()) if requirements: if _is_url(requirements): requirements_url = requirements requirements_txt = download_file_content_from_url( requirements_url).decode() else: requirements_txt = "\n".join(requirements.split(",")) # TODO: remove this after fixing the Elyra-AI/KFP-Notebook runner so that # Elyra should install its own requirements in addition to the provided requirements requirements_elyra_url = "https://github.com/elyra-ai/kfp-notebook/blob/master/etc/requirements-elyra.txt" requirements_elyra_txt = download_file_content_from_url( requirements_elyra_url).decode() requirements_elyra = "\n".join([ line for line in requirements_elyra_txt.split("\n") if not line.startswith("#") ]) requirements_all = f"# Required packages for {api_notebook.name}:\n" \ f"{requirements_txt}\n" \ f"# Requirements from {requirements_elyra_url}:\n" \ f"{requirements_elyra}" store_file(bucket_name="mlpipeline", prefix=f"notebooks/{notebook_id}/", file_name="requirements.txt", file_content=requirements_all.encode()) # if the url included an access token, replace the original url with the s3 url if "?token=" in url or "github.ibm.com" in url: api_notebook.url = s3_url update_multiple(ApiNotebook, [notebook_id], "url", s3_url) enable_anonymous_read_access(bucket_name="mlpipeline", prefix="notebooks/*") return api_notebook, 201