def handle_iteration(
client: RunClient,
iteration: int = None,
suggestions: List[Dict] = None,
summary: Dict = None,
name: str = None,
):
summary = summary or {}
summary.update({
"iteration": iteration,
"suggestions": [sanitize_dict(s) for s in suggestions],
})
def handler():
if suggestions:
artifact_run = V1RunArtifact(
name=name or "out-iteration-{}".format(iteration),
kind=V1ArtifactKind.ITERATION,
summary=summary,
is_input=False,
)
client.log_artifact_lineage(artifact_run)
Printer.print_success("Tuner generated new suggestions.")
else:
client.log_succeeded(message="Iterative operation has succeeded")
Printer.print_success("Iterative optimization succeeded")
try:
handler()
except Exception as e:
exp = "Polyaxon tuner failed fetching iteration definition: {}\n{}".format(
repr(e), traceback.format_exc())
client.log_failed(message="PolyaxonTunerFailed", traceback=exp)
logger.warning(e)
def handler():
runs = (
client.list(
query=search.query,
sort=search.sort,
limit=search.limit,
offset=search.offset,
).results
or []
)
configs = []
metrics = []
run_uuids = []
for run in runs:
if optimization_metric in run.outputs:
run_uuids.append(run.uuid)
configs.append(run.inputs)
metrics.append(run.outputs[optimization_metric])
if configs or metrics or run_uuids:
artifact_run = V1RunArtifact(
name=name or "in-iteration-{}".format(iteration),
kind=V1ArtifactKind.ITERATION,
summary={
"iteration": iteration,
"configs": [sanitize_dict(s) for s in configs],
"metrics": [sanitize_np_types(s) for s in metrics],
"uuid": run_uuids,
},
is_input=True,
)
client.log_artifact_lineage(artifact_run)
return run_uuids, configs, metrics
def handle_iteration(
client: RunClient,
suggestions: List[Dict] = None,
):
if not suggestions:
logger.warning("No new suggestions were created")
return
try:
logger.info("Generated new {} suggestions".format(len(suggestions)))
client.log_outputs(suggestions=[sanitize_dict(s) for s in suggestions],
async_req=False)
except Exception as e:
exp = "Polyaxon tuner failed logging iteration definition: {}\n{}".format(
repr(e), traceback.format_exc())
client.log_failed(reason="PolyaxonTunerIteration", message=exp)
logger.warning(e)
def log_classifier(classifier, X_test, y_test, nrows=1000, run=None):
assert is_classifier(
classifier), "classifier should be sklearn classifier."
run = tracking.get_or_create_run(run)
run.log_inputs(**sanitize_dict(classifier.get_params()))
_log_test_preds_proba(run, classifier, X_test, nrows=nrows)
y_pred = classifier.predict(X_test)
results = {}
for metric_name, values in zip(
["precision", "recall", "fbeta_score", "support"],
precision_recall_fscore_support(y_test, y_pred),
):
for i, value in enumerate(values):
results["{}_class_{}_test".format(metric_name, i)] = value
results["accuracy"] = accuracy_score(y_test, y_pred)
results["f1"] = f1_score(y_pred, y_pred, average="weighted")
run.log_metrics(**results)
_log_test_predictions(run, y_test, y_pred=y_pred, nrows=nrows)