def create_auth_context():
try:
run_client = RunClient()
except PolyaxonClientException as e:
raise PolyaxonContainerException(e)
retry = 1
done = False
exp = None
while not done and retry <= 3:
try:
impersonate(
owner=run_client.owner,
project=run_client.project,
run_uuid=run_client.run_uuid,
client=run_client.client,
)
print("Auth context initialized.")
return
except PolyaxonClientException as e:
retry += 1
print("Could not establish connection, retrying ...")
exp = "Polyaxon auth initialized failed authenticating the operation: {}\n{}".format(
repr(e), traceback.format_exc())
time.sleep(retry)
run_client.log_failed("Could not create an auth context.", traceback=exp)
raise PolyaxonContainerException(
"Init job did not succeed authenticating job.")
def upload(ctx, project, uid, path_from, path_to, is_file, sync_failure):
"""Upload runs' artifacts.
Uses /docs/core/cli/#caching
Examples:
\b
$ polyaxon ops upload -uid=8aac02e3a62a4f0aaa257c59da5eab80
\b
$ polyaxon ops upload -uid=8aac02e3a62a4f0aaa257c59da5eab80 path_from="path/to/upload"
\b
$ polyaxon ops upload -uid=8aac02e3a62a4f0aaa257c59da5eab80 path_to="path/to/upload/to"
"""
owner, project_name, run_uuid = get_project_run_or_local(
project or ctx.obj.get("project"),
uid or ctx.obj.get("run_uuid"),
is_cli=True,
)
try:
client = RunClient(owner=owner,
project=project_name,
run_uuid=run_uuid)
if is_file:
response = client.upload_artifact(filepath=path_from,
path=path_to,
overwrite=True)
else:
response = client.upload_artifacts_dir(
dirpath=path_from,
path=path_to,
overwrite=True,
relative_to=path_from,
)
except (
ApiException,
HTTPError,
PolyaxonHTTPError,
PolyaxonShouldExitError,
PolyaxonClientException,
) as e:
handle_cli_error(
e,
message="Could not upload artifacts for run `{}`".format(run_uuid))
sys.exit(1)
if response.status_code == 200:
Printer.print_success("Artifacts uploaded")
else:
if sync_failure:
client.log_failed(reason="OperationCli",
message="Operation failed uploading artifacts")
Printer.print_error(
"Error uploading artifacts. "
"Status: {}. Error: {}.".format(response.status_code,
response.content),
sys_exit=True,
)
def handle_iteration(
client: RunClient,
iteration: int = None,
suggestions: List[Dict] = None,
summary: Dict = None,
name: str = None,
):
summary = summary or {}
summary.update({
"iteration": iteration,
"suggestions": [sanitize_dict(s) for s in suggestions],
})
def handler():
if suggestions:
artifact_run = V1RunArtifact(
name=name or "out-iteration-{}".format(iteration),
kind=V1ArtifactKind.ITERATION,
summary=summary,
is_input=False,
)
client.log_artifact_lineage(artifact_run)
Printer.print_success("Tuner generated new suggestions.")
else:
client.log_succeeded(message="Iterative operation has succeeded")
Printer.print_success("Iterative optimization succeeded")
try:
handler()
except Exception as e:
exp = "Polyaxon tuner failed fetching iteration definition: {}\n{}".format(
repr(e), traceback.format_exc())
client.log_failed(message="PolyaxonTunerFailed", traceback=exp)
logger.warning(e)
def get_iteration_definition(
client: RunClient,
iteration: int,
search: V1ParamSearch,
optimization_metric: str,
name: str = None,
):
def handler():
runs = (
client.list(
query=search.query,
sort=search.sort,
limit=search.limit,
offset=search.offset,
).results
or []
)
configs = []
metrics = []
run_uuids = []
for run in runs:
if optimization_metric in run.outputs:
run_uuids.append(run.uuid)
configs.append(run.inputs)
metrics.append(run.outputs[optimization_metric])
if configs or metrics or run_uuids:
artifact_run = V1RunArtifact(
name=name or "in-iteration-{}".format(iteration),
kind=V1ArtifactKind.ITERATION,
summary={
"iteration": iteration,
"configs": [sanitize_dict(s) for s in configs],
"metrics": [sanitize_np_types(s) for s in metrics],
"uuid": run_uuids,
},
is_input=True,
)
client.log_artifact_lineage(artifact_run)
return run_uuids, configs, metrics
try:
return handler()
except Exception as e:
exp = "Polyaxon tuner failed fetching iteration definition: {}\n{}".format(
repr(e), traceback.format_exc()
)
client.log_failed(message="PolyaxonTunerFailed", traceback=exp)
logger.warning(e)
def handle_iteration(
client: RunClient,
suggestions: List[Dict] = None,
):
if not suggestions:
logger.warning("No new suggestions were created")
return
try:
logger.info("Generated new {} suggestions".format(len(suggestions)))
client.log_outputs(suggestions=[sanitize_dict(s) for s in suggestions],
async_req=False)
except Exception as e:
exp = "Polyaxon tuner failed logging iteration definition: {}\n{}".format(
repr(e), traceback.format_exc())
client.log_failed(reason="PolyaxonTunerIteration", message=exp)
logger.warning(e)
def handle_iteration_failure(client: RunClient, exp: Exception):
exp = "Polyaxon tuner failed creating suggestions : {}\n{}".format(
repr(exp), traceback.format_exc()
)
client.log_failed(message="PolyaxonTunerCreatingSuggestions", traceback=exp)
