def sync_summaries(last_check: Optional[datetime], run_uuid: str, client: RunClient): events_path = CONTEXT_MOUNT_RUN_EVENTS_FORMAT.format(run_uuid) # check if there's a path to sync if not os.path.exists(events_path): return # crawl dirs summaries = [] last_values = {} connection = get_artifacts_connection() connection_name = connection.name if connection else None for events_kind in get_dirs_under_path(events_path): _summaries, _last_values = sync_events_summaries( events_path=events_path, events_kind=events_kind, last_check=last_check, connection_name=connection_name, ) summaries += _summaries last_values.update(_last_values) if summaries: client.log_artifact_lineage(summaries) if last_values: client.log_outputs(**last_values)
def create_code_repo( repo_path: str, url: str, revision: str, connection: str = None, flags: List[str] = None, ): try: clone_url = get_clone_url(url) except Exception as e: raise PolyaxonContainerException( "Error parsing url: {}.".format(url)) from e if flags and "--experimental-fetch" in flags: flags.remove("--experimental-fetch") fetch_git_repo(repo_path=repo_path, clone_url=clone_url, revision=revision, flags=flags) else: clone_and_checkout_git_repo(repo_path=repo_path, clone_url=clone_url, revision=revision, flags=flags) # Update remote set_remote(repo_path=repo_path, url=url) if settings.CLIENT_CONFIG.no_api: return try: run_client = RunClient() except PolyaxonClientException as e: raise PolyaxonContainerException(e) code_ref = get_code_reference(path=repo_path, url=url) artifact_run = V1RunArtifact( name=code_ref.get("commit"), kind=V1ArtifactKind.CODEREF, connection=connection, summary=code_ref, is_input=True, ) run_client.log_artifact_lineage(artifact_run)
def create_dockerfile_lineage(dockerfile_path: str, summary: Dict): if not dockerfile_path: return filename = os.path.basename(dockerfile_path) if settings.CLIENT_CONFIG.no_api: return try: run_client = RunClient() except PolyaxonClientException as e: raise PolyaxonContainerException(e) artifact_run = V1RunArtifact( name=filename, kind=V1ArtifactKind.DOCKERFILE, path=get_rel_asset_path(dockerfile_path), summary=summary, is_input=True, ) run_client.log_artifact_lineage(artifact_run)
def create_file_lineage(filepath: str, summary: Dict, kind: str): kind = kind or V1ArtifactKind.FILE if not filepath: return filename = os.path.basename(filepath) if settings.CLIENT_CONFIG.no_api: return try: run_client = RunClient() except PolyaxonClientException as e: raise PolyaxonContainerException(e) artifact_run = V1RunArtifact( name=get_base_filename(filename), kind=kind, path=get_rel_asset_path(filepath), summary=summary, is_input=True, ) run_client.log_artifact_lineage(artifact_run)