def sync_summaries(last_check: Optional[datetime], run_uuid: str, client: RunClient):
events_path = CONTEXT_MOUNT_RUN_EVENTS_FORMAT.format(run_uuid)
# check if there's a path to sync
if not os.path.exists(events_path):
return
# crawl dirs
summaries = []
last_values = {}
connection = get_artifacts_connection()
connection_name = connection.name if connection else None
for events_kind in get_dirs_under_path(events_path):
_summaries, _last_values = sync_events_summaries(
events_path=events_path,
events_kind=events_kind,
last_check=last_check,
connection_name=connection_name,
)
summaries += _summaries
last_values.update(_last_values)
if summaries:
client.log_artifact_lineage(summaries)
if last_values:
client.log_outputs(**last_values)
def create_code_repo(
repo_path: str,
url: str,
revision: str,
connection: str = None,
flags: List[str] = None,
):
try:
clone_url = get_clone_url(url)
except Exception as e:
raise PolyaxonContainerException(
"Error parsing url: {}.".format(url)) from e
if flags and "--experimental-fetch" in flags:
flags.remove("--experimental-fetch")
fetch_git_repo(repo_path=repo_path,
clone_url=clone_url,
revision=revision,
flags=flags)
else:
clone_and_checkout_git_repo(repo_path=repo_path,
clone_url=clone_url,
revision=revision,
flags=flags)
# Update remote
set_remote(repo_path=repo_path, url=url)
if settings.CLIENT_CONFIG.no_api:
return
try:
run_client = RunClient()
except PolyaxonClientException as e:
raise PolyaxonContainerException(e)
code_ref = get_code_reference(path=repo_path, url=url)
artifact_run = V1RunArtifact(
name=code_ref.get("commit"),
kind=V1ArtifactKind.CODEREF,
connection=connection,
summary=code_ref,
is_input=True,
)
run_client.log_artifact_lineage(artifact_run)
def create_dockerfile_lineage(dockerfile_path: str, summary: Dict):
if not dockerfile_path:
return
filename = os.path.basename(dockerfile_path)
if settings.CLIENT_CONFIG.no_api:
return
try:
run_client = RunClient()
except PolyaxonClientException as e:
raise PolyaxonContainerException(e)
artifact_run = V1RunArtifact(
name=filename,
kind=V1ArtifactKind.DOCKERFILE,
path=get_rel_asset_path(dockerfile_path),
summary=summary,
is_input=True,
)
run_client.log_artifact_lineage(artifact_run)
def create_file_lineage(filepath: str, summary: Dict, kind: str):
kind = kind or V1ArtifactKind.FILE
if not filepath:
return
filename = os.path.basename(filepath)
if settings.CLIENT_CONFIG.no_api:
return
try:
run_client = RunClient()
except PolyaxonClientException as e:
raise PolyaxonContainerException(e)
artifact_run = V1RunArtifact(
name=get_base_filename(filename),
kind=kind,
path=get_rel_asset_path(filepath),
summary=summary,
is_input=True,
)
run_client.log_artifact_lineage(artifact_run)
