def on_train_begin(self, logs=None): # pylint: disable=unused-argument
if not self.run:
return
params = dict(
num_layers=len(self.model.layers),
optimizer_name=type(self.model.optimizer).__name__,
)
if hasattr(self.model.optimizer, "lr"):
params["learning_rate"] = sanitize_np_types(
self.model.optimizer.lr if type(self.model.optimizer.lr) is
float else keras.backend.eval(self.model.optimizer.lr))
if hasattr(self.model.optimizer, "epsilon"):
params["epsilon"] = sanitize_np_types(
self.model.optimizer.epsilon
if type(self.model.optimizer.epsilon) is float else keras.
backend.eval(self.model.optimizer.epsilon))
self.run.log_inputs(**params)
sum_list = []
self.model.summary(print_fn=sum_list.append)
summary = "\n".join(sum_list)
rel_path = self.run.get_outputs_path("model_summary.txt")
with open(rel_path, "w") as f:
f.write(summary)
self.run.log_file_ref(path=rel_path)
def _get_step_summary(self) -> Optional[Dict]:
_count = self.df.step.count()
if _count == 0:
return None
return {
"count": sanitize_np_types(_count),
"min": sanitize_np_types(self.df.step.iloc[0]),
"max": sanitize_np_types(self.df.step.iloc[-1]),
}
def on_train_begin(self, logs=None): # pylint: disable=unused-argument
if not self.run:
return
params = {}
try:
params["num_layers"] = len(self.model.layers)
except Exception: # noqa
pass
try:
params["optimizer_name"] = type(self.model.optimizer).__name__
except Exception: # noqa
pass
try:
if hasattr(self.model.optimizer, "lr"):
params["optimizer_lr"] = sanitize_np_types(
self.model.optimizer.lr
if type(self.model.optimizer.lr) is float
else keras.backend.eval(self.model.optimizer.lr)
)
except Exception: # noqa
pass
try:
if hasattr(self.model.optimizer, "epsilon"):
params["optimizer_epsilon"] = sanitize_np_types(
self.model.optimizer.epsilon
if type(self.model.optimizer.epsilon) is float
else keras.backend.eval(self.model.optimizer.epsilon)
)
except Exception: # noqa
pass
if params:
self.run.log_inputs(**params)
try:
sum_list = []
self.model.summary(print_fn=sum_list.append)
summary = "\n".join(sum_list)
rel_path = self.run.get_outputs_path("model_summary.txt")
with open(rel_path, "w") as f:
f.write(summary)
self.run.log_file_ref(path=rel_path)
except Exception: # noqa
pass
def handler():
runs = (
client.list(
query=search.query,
sort=search.sort,
limit=search.limit,
offset=search.offset,
).results
or []
)
configs = []
metrics = []
run_uuids = []
for run in runs:
if optimization_metric in run.outputs:
run_uuids.append(run.uuid)
configs.append(run.inputs)
metrics.append(run.outputs[optimization_metric])
if configs or metrics or run_uuids:
artifact_run = V1RunArtifact(
name=name or "in-iteration-{}".format(iteration),
kind=V1ArtifactKind.ITERATION,
summary={
"iteration": iteration,
"configs": [sanitize_dict(s) for s in configs],
"metrics": [sanitize_np_types(s) for s in metrics],
"uuid": run_uuids,
},
is_input=True,
)
client.log_artifact_lineage(artifact_run)
return run_uuids, configs, metrics
def get_summary(self) -> Dict:
summary = {"is_event": True}
step_summary = self._get_step_summary()
if step_summary:
summary["step"] = step_summary
ts_summary = self._get_ts_summary()
if ts_summary:
summary["timestamp"] = ts_summary
if self.kind == V1ArtifactKind.METRIC:
summary[self.kind] = {
k: sanitize_np_types(v)
for k, v in self.df.metric.describe().to_dict().items()
}
summary[self.kind]["last"] = sanitize_np_types(
self.df.metric.iloc[-1])
return summary
def get_summary(self) -> Dict:
summary = {
"step": {
"count": sanitize_np_types(self.df.step.count()),
"min": sanitize_np_types(self.df.step.iloc[0]),
"max": sanitize_np_types(self.df.step.iloc[-1]),
},
"timestamp": {
"min": self.df.timestamp.iloc[0].isoformat(),
"max": self.df.timestamp.iloc[-1].isoformat(),
},
}
if self.kind == V1ArtifactKind.METRIC:
summary[self.kind] = {
k: sanitize_np_types(v)
for k, v in self.df.metric.describe().to_dict().items()
}
summary[self.kind]["last"] = sanitize_np_types(self.df.metric.iloc[-1])
return summary
def get_event_at(self, index):
event = self.df.iloc[index].to_dict()
event["timestamp"] = event["timestamp"].isoformat()
event["step"] = sanitize_np_types(event["step"])
return V1Event.from_dict(event)