def create_auth_context(): try: run_client = RunClient() except PolyaxonClientException as e: raise PolyaxonContainerException(e) retry = 1 done = False exp = None while not done and retry <= 3: try: impersonate( owner=run_client.owner, project=run_client.project, run_uuid=run_client.run_uuid, client=run_client.client, ) print("Auth context initialized.") return except PolyaxonClientException as e: retry += 1 print("Could not establish connection, retrying ...") exp = "Polyaxon auth initialized failed authenticating the operation: {}\n{}".format( repr(e), traceback.format_exc()) time.sleep(retry) run_client.log_failed("Could not create an auth context.", traceback=exp) raise PolyaxonContainerException( "Init job did not succeed authenticating job.")
def upload(ctx, project, uid, path_from, path_to, is_file, sync_failure): """Upload runs' artifacts. Uses /docs/core/cli/#caching Examples: \b $ polyaxon ops upload -uid=8aac02e3a62a4f0aaa257c59da5eab80 \b $ polyaxon ops upload -uid=8aac02e3a62a4f0aaa257c59da5eab80 path_from="path/to/upload" \b $ polyaxon ops upload -uid=8aac02e3a62a4f0aaa257c59da5eab80 path_to="path/to/upload/to" """ owner, project_name, run_uuid = get_project_run_or_local( project or ctx.obj.get("project"), uid or ctx.obj.get("run_uuid"), is_cli=True, ) try: client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) if is_file: response = client.upload_artifact(filepath=path_from, path=path_to, overwrite=True) else: response = client.upload_artifacts_dir( dirpath=path_from, path=path_to, overwrite=True, relative_to=path_from, ) except ( ApiException, HTTPError, PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException, ) as e: handle_cli_error( e, message="Could not upload artifacts for run `{}`".format(run_uuid)) sys.exit(1) if response.status_code == 200: Printer.print_success("Artifacts uploaded") else: if sync_failure: client.log_failed(reason="OperationCli", message="Operation failed uploading artifacts") Printer.print_error( "Error uploading artifacts. " "Status: {}. Error: {}.".format(response.status_code, response.content), sys_exit=True, )
def handle_iteration( client: RunClient, iteration: int = None, suggestions: List[Dict] = None, summary: Dict = None, name: str = None, ): summary = summary or {} summary.update({ "iteration": iteration, "suggestions": [sanitize_dict(s) for s in suggestions], }) def handler(): if suggestions: artifact_run = V1RunArtifact( name=name or "out-iteration-{}".format(iteration), kind=V1ArtifactKind.ITERATION, summary=summary, is_input=False, ) client.log_artifact_lineage(artifact_run) Printer.print_success("Tuner generated new suggestions.") else: client.log_succeeded(message="Iterative operation has succeeded") Printer.print_success("Iterative optimization succeeded") try: handler() except Exception as e: exp = "Polyaxon tuner failed fetching iteration definition: {}\n{}".format( repr(e), traceback.format_exc()) client.log_failed(message="PolyaxonTunerFailed", traceback=exp) logger.warning(e)
def get_iteration_definition( client: RunClient, iteration: int, search: V1ParamSearch, optimization_metric: str, name: str = None, ): def handler(): runs = ( client.list( query=search.query, sort=search.sort, limit=search.limit, offset=search.offset, ).results or [] ) configs = [] metrics = [] run_uuids = [] for run in runs: if optimization_metric in run.outputs: run_uuids.append(run.uuid) configs.append(run.inputs) metrics.append(run.outputs[optimization_metric]) if configs or metrics or run_uuids: artifact_run = V1RunArtifact( name=name or "in-iteration-{}".format(iteration), kind=V1ArtifactKind.ITERATION, summary={ "iteration": iteration, "configs": [sanitize_dict(s) for s in configs], "metrics": [sanitize_np_types(s) for s in metrics], "uuid": run_uuids, }, is_input=True, ) client.log_artifact_lineage(artifact_run) return run_uuids, configs, metrics try: return handler() except Exception as e: exp = "Polyaxon tuner failed fetching iteration definition: {}\n{}".format( repr(e), traceback.format_exc() ) client.log_failed(message="PolyaxonTunerFailed", traceback=exp) logger.warning(e)
def handle_iteration( client: RunClient, suggestions: List[Dict] = None, ): if not suggestions: logger.warning("No new suggestions were created") return try: logger.info("Generated new {} suggestions".format(len(suggestions))) client.log_outputs(suggestions=[sanitize_dict(s) for s in suggestions], async_req=False) except Exception as e: exp = "Polyaxon tuner failed logging iteration definition: {}\n{}".format( repr(e), traceback.format_exc()) client.log_failed(reason="PolyaxonTunerIteration", message=exp) logger.warning(e)
def handle_iteration_failure(client: RunClient, exp: Exception): exp = "Polyaxon tuner failed creating suggestions : {}\n{}".format( repr(exp), traceback.format_exc() ) client.log_failed(message="PolyaxonTunerCreatingSuggestions", traceback=exp)