def set_run_event_logger(self):
"""Sets an event logger.
Be careful, this method is called automatically
when a job is running in-cluster and follows some flags that Polyaxon sets. Polyaxon
has some processes to automatically sync your run's artifacts and outputs.
"""
self._event_logger = EventFileWriter(run_path=self._artifacts_path)
def setUp(self):
super().setUp()
self.run_path = tempfile.mkdtemp()
settings.CLIENT_CONFIG.is_managed = False
os.environ[POLYAXON_KEYS_COLLECT_ARTIFACTS] = "false"
os.environ[POLYAXON_KEYS_COLLECT_RESOURCES] = "false"
with patch("polyaxon.tracking.run.Run.refresh_data") as _:
self.run = Run(project="owner.project")
self.event_logger = EventFileWriter(run_path=self.run_path)
self.resource_logger = ResourceFileWriter(run_path=self.run_path)
self.run._artifacts_path = self.run_path
self.run._event_logger = self.event_logger
self.run._resource_logger = self.resource_logger
assert os.path.exists(get_event_path(self.run_path)) is True
assert os.path.exists(get_asset_path(self.run_path)) is True
class Run(RunClient):
@check_no_op
def __init__(
self,
owner=None,
project=None,
run_uuid=None,
client=None,
track_code=True,
track_env=False,
):
super().__init__(
owner=owner, project=project, run_uuid=run_uuid, client=client,
)
self.track_code = track_code
self.track_env = track_env
self._artifacts_path = None
self._outputs_path = None
self._event_logger = None
self._resource_logger = None
self._results = {}
if settings.CLIENT_CONFIG.is_managed and self.run_uuid:
self.set_run_event_path()
if (
self.artifacts_path
and settings.CLIENT_CONFIG.is_managed
and get_collect_artifact()
):
self.set_run_event_logger()
if get_collect_resources():
self.set_run_resource_logger()
self._register_wait()
self._run = polyaxon_sdk.V1Run()
if settings.CLIENT_CONFIG.is_offline:
return
if self._run_uuid:
self.refresh_data()
# Track run env
if settings.CLIENT_CONFIG.is_managed and self.track_env:
self.log_run_env()
@property
def artifacts_path(self):
return self._artifacts_path
@property
def outputs_path(self):
return self._outputs_path
@check_no_op
def set_run_event_path(self):
self._artifacts_path = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(self.run_uuid)
self._outputs_path = CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format(self.run_uuid)
@check_no_op
def set_run_event_logger(self):
self._event_logger = EventFileWriter(run_path=self.artifacts_path)
@check_no_op
def set_run_resource_logger(self):
self._resource_logger = ResourceFileWriter(run_path=self.artifacts_path)
@check_no_op
def create(self, name=None, tags=None, description=None, content=None):
run = polyaxon_sdk.V1Run()
if name:
run.name = name
if tags:
run.tags = tags
if description:
run.description = description
if content:
try:
specification = OperationSpecification.read(content)
except Exception as e:
raise PolyaxonClientException("Client error: %s" % e) from e
run.content = specification.to_dict(dump=True)
else:
run.is_managed = False
if self.client:
try:
run = self.client.runs_v1.create_run(
owner=self.owner, project=self.project, body=run
)
except (ApiException, HTTPError) as e:
raise PolyaxonClientException("Client error: %s" % e) from e
if not run:
raise PolyaxonClientException("Could not create a run.")
self._run = run
self._run_uuid = run.uuid
self.set_run_event_logger()
if self.track_code:
self.log_code_ref()
if self.track_env:
self.log_run_env()
if not settings.CLIENT_CONFIG.is_managed:
self._start()
else:
self._register_wait()
return self
@property
def is_service(self):
if settings.CLIENT_CONFIG.no_op:
return None
return settings.CLIENT_CONFIG.is_managed and settings.CLIENT_CONFIG.is_service
@check_no_op
@check_offline
@can_log_events
def log_metric(self, name, value, step=None, timestamp=None):
events = []
event_value = events_processors.metric(value)
if event_value == UNKNOWN:
return
events.append(
LoggedEventSpec(
name=name,
kind=V1ArtifactKind.METRIC,
event=V1Event.make(timestamp=timestamp, step=step, metric=event_value),
)
)
if events:
self._event_logger.add_events(events)
self._results[name] = event_value
@check_no_op
@check_offline
@can_log_events
def log_metrics(self, step=None, timestamp=None, **metrics):
events = []
for metric in metrics:
event_value = events_processors.metric(metrics[metric])
if event_value == UNKNOWN:
continue
events.append(
LoggedEventSpec(
name=metric,
kind=V1ArtifactKind.METRIC,
event=V1Event.make(
timestamp=timestamp, step=step, metric=event_value
),
)
)
if events:
self._event_logger.add_events(events)
@check_no_op
@check_offline
@can_log_events
def log_image(
self, data, name=None, step=None, timestamp=None, rescale=1, dataformats="CHW"
):
is_file = isinstance(data, str) and os.path.exists(data)
ext = "png"
if is_file:
name = name or os.path.basename(data)
ext = get_path_extension(filepath=data) or ext
else:
name = name or "image"
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.IMAGE,
name=name,
step=step,
ext=ext,
)
if is_file:
event_value = events_processors.image_path(
from_path=data, asset_path=asset_path
)
elif hasattr(data, "encoded_image_string"):
event_value = events_processors.encoded_image(
asset_path=asset_path, data=data
)
else:
event_value = events_processors.image(
asset_path=asset_path,
data=data,
rescale=rescale,
dataformats=dataformats,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.IMAGE,
event=V1Event(timestamp=timestamp, step=step, image=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_image_with_boxes(
self,
tensor_image,
tensor_boxes,
name=None,
step=None,
timestamp=None,
rescale=1,
dataformats="CHW",
):
name = name or "figure"
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.IMAGE,
name=name,
step=step,
)
event_value = events_processors.image_boxes(
asset_path=asset_path,
tensor_image=tensor_image,
tensor_boxes=tensor_boxes,
rescale=rescale,
dataformats=dataformats,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.IMAGE,
event=V1Event(timestamp=timestamp, step=step, image=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_mpl_image(self, data, name=None, close=True, step=None, timestamp=None):
name = name or "figure"
if isinstance(data, list):
event_value = events_processors.figures_to_images(figures=data, close=close)
if event_value == UNKNOWN:
return
self.log_image(
name=name,
data=event_value,
step=step,
timestamp=timestamp,
dataformats="NCHW",
)
else:
event_value = events_processors.figure_to_image(figure=data, close=close)
self.log_image(
name=name,
data=event_value,
step=step,
timestamp=timestamp,
dataformats="CHW",
)
@check_no_op
@check_offline
@can_log_events
def log_video(
self, data, name=None, fps=4, step=None, timestamp=None, content_type=None
):
is_file = isinstance(data, str) and os.path.exists(data)
content_type = content_type or "gif"
if is_file:
name = name or os.path.basename(data)
content_type = get_path_extension(filepath=data) or content_type
else:
name = name or "video"
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.VIDEO,
name=name,
step=step,
ext=content_type,
)
if is_file:
event_value = events_processors.video_path(
from_path=data, asset_path=asset_path, content_type=content_type
)
else:
event_value = events_processors.video(
asset_path=asset_path, tensor=data, fps=fps, content_type=content_type
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.VIDEO,
event=V1Event(timestamp=timestamp, step=step, video=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_audio(
self,
data,
name=None,
sample_rate=44100,
step=None,
timestamp=None,
content_type=None,
):
is_file = isinstance(data, str) and os.path.exists(data)
ext = content_type or "wav"
if is_file:
name = name or os.path.basename(data)
ext = get_path_extension(filepath=data) or ext
else:
name = name or "audio"
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.AUDIO,
name=name,
step=step,
ext=ext,
)
if is_file:
event_value = events_processors.audio_path(
from_path=data, asset_path=asset_path, content_type=content_type
)
else:
event_value = events_processors.audio(
asset_path=asset_path, tensor=data, sample_rate=sample_rate
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.AUDIO,
event=V1Event(timestamp=timestamp, step=step, audio=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_text(self, name, text, step=None, timestamp=None):
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.TEXT,
event=V1Event(timestamp=timestamp, step=step, text=text),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_html(self, name, html, step=None, timestamp=None):
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.HTML,
event=V1Event(timestamp=timestamp, step=step, html=html),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_np_histogram(self, name, values, counts, step=None, timestamp=None):
event_value = events_processors.np_histogram(values=values, counts=counts)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.HISTOGRAM,
event=V1Event(timestamp=timestamp, step=step, histogram=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_histogram(
self, name, values, bins, max_bins=None, step=None, timestamp=None
):
event_value = events_processors.histogram(
values=values, bins=bins, max_bins=max_bins
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.HISTOGRAM,
event=V1Event(timestamp=timestamp, step=step, histogram=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_model(
self, path, name=None, framework=None, spec=None, step=None, timestamp=None
):
name = name or os.path.basename(path)
ext = None
if os.path.isfile(path):
ext = get_path_extension(filepath=path)
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.MODEL,
name=name,
step=step,
ext=ext,
)
model = events_processors.model_path(
from_path=path, asset_path=asset_path, framework=framework, spec=spec
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.MODEL,
event=V1Event(timestamp=timestamp, step=step, model=model),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_dataframe(
self, path, name=None, content_type=None, step=None, timestamp=None
):
name = name or os.path.basename(path)
ext = get_path_extension(filepath=path)
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.DATAFRAME,
name=name,
step=step,
ext=ext,
)
df = events_processors.dataframe_path(
from_path=path, asset_path=asset_path, content_type=content_type
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.DATAFRAME,
event=V1Event(timestamp=timestamp, step=step, dataframe=df),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_artifact(
self, path, name=None, artifact_kind=None, step=None, timestamp=None
):
name = name or os.path.basename(name)
ext = get_path_extension(filepath=path)
artifact_kind = artifact_kind or V1ArtifactKind.FILE
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=artifact_kind,
name=name,
step=step,
ext=ext,
)
artifact = events_processors.artifact_path(
from_path=path, asset_path=asset_path, kind=artifact_kind
)
logged_event = LoggedEventSpec(
name=name,
kind=artifact_kind,
event=V1Event(timestamp=timestamp, step=step, artifact=artifact),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_plotly_chart(self, name, figure, step=None, timestamp=None):
chart = events_processors.plotly_chart(figure=figure)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CHART,
event=V1Event(timestamp=timestamp, step=step, chart=chart),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_bokeh_chart(self, name, figure, step=None, timestamp=None):
chart = events_processors.bokeh_chart(figure=figure)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CHART,
event=V1Event(timestamp=timestamp, step=step, chart=chart),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_mpl_plotly_chart(self, name, figure, step=None, timestamp=None):
chart = events_processors.mpl_plotly_chart(figure=figure)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CHART,
event=V1Event(timestamp=timestamp, step=step, chart=chart),
)
self._event_logger.add_event(logged_event)
@check_no_op
def get_log_level(self):
return get_log_level()
@check_no_op
@check_offline
def _register_wait(self):
atexit.register(self._wait)
@check_no_op
@check_offline
def _start(self):
atexit.register(self._end)
self.start()
def excepthook(exception, value, tb):
self.log_failed(message="Type: {}, Value: {}".format(exception, value))
# Resume normal work
sys.__excepthook__(exception, value, tb)
sys.excepthook = excepthook
@check_no_op
@check_offline
def _end(self):
self.log_succeeded()
self._wait()
@check_no_op
@check_offline
def _wait(self):
if self._event_logger:
self._event_logger.flush()
if self._resource_logger:
self._resource_logger.flush()
if self._results:
self.log_outputs(**self._results)
time.sleep(1)
@check_no_op
@check_offline
@can_log_events
def log_run_env(self):
# TODO: log to file
return get_run_env()
def set_run_event_logger(self):
self._event_logger = EventFileWriter(run_path=self.artifacts_path)
class Run(RunClient):
"""Run tracking is client to instrument your machine learning model and track experiments.
If no values are passed to this class,
Polyaxon will try to resolve the owner, project, and run uuid from the environment:
* If you have a configured CLI, Polyaxon will use the configuration of the cli.
* If you have a cached run using the CLI,
the client will default to that cached run unless you override the values.
* If you use this client in the context of a job or a service managed by Polyaxon,
a configuration will be available to resolve the values based on that run.
You can always access the `self.client` to execute more APIs.
Properties:
project: str.
owner: str.
run_uuid: str.
run_data: V1Run.
status: str.
namespace: str.
client: [PolyaxonClient](/docs/core/python-library/polyaxon-client/)
Args:
owner: str, optional, the owner is the username or
the organization name owning this project.
project: str, optional, project name owning the run(s).
run_uuid: str, optional, run uuid.
client: [PolyaxonClient](/docs/core/python-library/polyaxon-client/), optional,
an instance of a configured client, if not passed,
a new instance will be created based on the available environment.
track_code: bool, optional, default True, to track code version.
Polyaxon will try to track information about any repo
configured in the context where this client is instantiated.
track_env: bool, optional, default True, to track information about the environment.
refresh_data: bool, optional, default False, to refresh the run data at instantiation.
artifacts_path: str, optional, for in-cluster runs it will be set automatically.
Raises:
PolyaxonClientException: If no owner and/or project are passed and Polyaxon cannot
resolve the values from the environment.
"""
@check_no_op
def __init__(
self,
owner: str = None,
project: str = None,
run_uuid: str = None,
client: RunClient = None,
track_code: bool = True,
track_env: bool = True,
refresh_data: bool = False,
artifacts_path: str = None,
):
super().__init__(
owner=owner,
project=project,
run_uuid=run_uuid,
client=client,
)
self.track_code = track_code
self.track_env = track_env
self._has_model = False
self._has_events = False
self._has_tensorboard = False
self._artifacts_path = None
self._outputs_path = None
self._event_logger = None
self._resource_logger = None
self._results = {}
if (settings.CLIENT_CONFIG.is_managed
and self.run_uuid) or artifacts_path:
self.set_artifacts_path(artifacts_path)
if (self._artifacts_path and settings.CLIENT_CONFIG.is_managed
and get_collect_artifact()):
self.set_run_event_logger()
if get_collect_resources():
self.set_run_resource_logger()
# no artifacts path is set, we use the temp path
if not self._artifacts_path:
self._artifacts_path = TEMP_RUN_ARTIFACTS
self._outputs_path = TEMP_RUN_ARTIFACTS
self._run = polyaxon_sdk.V1Run()
if settings.CLIENT_CONFIG.is_offline:
return
if self._run_uuid and (refresh_data
or settings.CLIENT_CONFIG.is_managed):
self.refresh_data()
# Track run env
if settings.CLIENT_CONFIG.is_managed and self.track_env:
self.log_env()
if settings.CLIENT_CONFIG.is_managed:
self._register_wait()
@property
def is_service(self):
if settings.CLIENT_CONFIG.no_op:
return None
return settings.CLIENT_CONFIG.is_managed and settings.CLIENT_CONFIG.is_service
@check_no_op
def get_artifacts_path(self):
"""Returns the current artifacts path configured for this instance.
Returns:
str, artifacts_path
"""
return self._artifacts_path
@check_no_op
def get_outputs_path(self):
"""Returns the current outputs path configured for this instance.
Returns:
str, outputs_path
"""
return self._outputs_path
@check_no_op
def get_model_path(self, rel_path: str = "model"):
"""Returns a model path for this run relative to the outputs path.
Args:
rel_path: str, optional, default "model",
the relative path to the `outputs` context.
Returns:
str, outputs_path / rel_path
"""
path = self._outputs_path
if rel_path:
path = os.path.join(path, rel_path)
return path
@check_no_op
@check_offline
@can_log_outputs
def get_tensorboard_path(self, rel_path: str = "tensorboard"):
"""Returns a tensorboard path for this run relative to the outputs path.
Args:
rel_path: str, optional, default "tensorboard",
the relative path to the `outputs` context.
Returns:
str, outputs_path / rel_path
"""
path = self._outputs_path
if rel_path:
path = os.path.join(path, rel_path)
if not self._has_tensorboard:
self.log_tensorboard_ref(path)
self._has_tensorboard = True
return path
@check_no_op
def set_artifacts_path(self, artifacts_path: str = None):
"""Sets an artifacts_path.
Be careful, this method is called automatically
when a job is running in-cluster and follows some flags that Polyaxon sets. Polyaxon
has some processes to automatically sync your run's artifacts and outputs.
Args:
artifacts_path: str, optional
"""
_artifacts_path = artifacts_path or CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(
self.run_uuid)
_outputs_path = "{}/outputs".format(_artifacts_path)
self._artifacts_path = _artifacts_path
self._outputs_path = _outputs_path
@check_no_op
def set_run_event_logger(self):
"""Sets an event logger.
Be careful, this method is called automatically
when a job is running in-cluster and follows some flags that Polyaxon sets. Polyaxon
has some processes to automatically sync your run's artifacts and outputs.
"""
self._event_logger = EventFileWriter(run_path=self._artifacts_path)
@check_no_op
def set_run_resource_logger(self):
"""Sets an resources logger.
Be careful, this method is called automatically
when a job is running in-cluster and follows some flags that Polyaxon sets. Polyaxon
has some processes to automatically sync your run's artifacts and outputs.
"""
self._resource_logger = ResourceFileWriter(
run_path=self._artifacts_path)
def _post_create(self):
if self._artifacts_path:
self.set_run_event_logger()
if self.track_code:
self.log_code_ref()
if self.track_env:
self.log_env()
if not settings.CLIENT_CONFIG.is_managed:
self._start()
else:
self._register_wait()
def _log_has_events(self):
if not self._has_events:
self._has_events = True
self._log_meta(has_events=True)
def _log_has_model(self):
if not self._has_model:
self._has_model = True
self._log_meta(has_model=True)
@check_no_op
@check_offline
@can_log_events
def log_metric(self,
name: str,
value: float,
step: int = None,
timestamp=None):
"""Logs a metric datapoint.
```python
>>> log_metric(name="loss", value=0.01, step=10)
```
> It's very important to log `step` as one of your metrics
if you want to compare experiments on the dashboard
and use the steps in x-axis instead of timestamps.
Args:
name: str, metric name
value: float, metric value
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
events = []
event_value = events_processors.metric(value)
if event_value == UNKNOWN:
return
events.append(
LoggedEventSpec(
name=name,
kind=V1ArtifactKind.METRIC,
event=V1Event.make(timestamp=timestamp,
step=step,
metric=event_value),
))
if events:
self._event_logger.add_events(events)
self._results[name] = event_value
@check_no_op
@check_offline
@can_log_events
def log_metrics(self, step=None, timestamp=None, **metrics):
"""Logs multiple metrics.
```python
>>> log_metrics(step=123, loss=0.023, accuracy=0.91)
```
> It's very important to log `step` as one of your metrics
if you want to compare experiments on the dashboard
and use the steps in x-axis instead of timestamps.
Args:
step: int, optional
timestamp: datetime, optional
**metrics: **kwargs, key: value
"""
self._log_has_events()
events = []
for metric in metrics:
event_value = events_processors.metric(metrics[metric])
if event_value == UNKNOWN:
continue
events.append(
LoggedEventSpec(
name=metric,
kind=V1ArtifactKind.METRIC,
event=V1Event.make(timestamp=timestamp,
step=step,
metric=event_value),
))
if events:
self._event_logger.add_events(events)
@check_no_op
@check_offline
@can_log_events
def log_roc_auc_curve(self,
name,
fpr,
tpr,
auc=None,
step=None,
timestamp=None):
"""Logs ROC/AUC curve. This method expects an already processed values.
```python
>>> log_roc_auc_curve("roc_value", fpr, tpr, auc=0.6, step=1)
```
Args:
name: str, name of the curve
fpr: List[float] or numpy.array, false positive rate
tpr: List[float] or numpy.array, true positive rate
auc: float, optional, calculated area under curve
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.roc_auc_curve(
fpr=fpr,
tpr=tpr,
auc=auc,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp,
step=step,
curve=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_sklearn_roc_auc_curve(self,
name,
y_preds,
y_targets,
step=None,
timestamp=None):
"""Calculates and logs ROC/AUC curve using sklearn.
```python
>>> log_sklearn_roc_auc_curve("roc_value", y_preds, y_targets, step=10)
```
Args:
name: str, name of the curve
y_preds: List[float] or numpy.array
y_targets: List[float] or numpy.array
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.sklearn_roc_auc_curve(
y_preds=y_preds,
y_targets=y_targets,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp,
step=step,
curve=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_pr_curve(self,
name,
precision,
recall,
average_precision=None,
step=None,
timestamp=None):
"""Logs PR curve. This method expects an already processed values.
```python
>>> log_pr_curve("pr_value", precision, recall, step=10)
```
Args:
name: str, name of the curve
y_preds: List[float] or numpy.array
y_targets: List[float] or numpy.array
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.pr_curve(
precision=precision,
recall=recall,
average_precision=average_precision,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp,
step=step,
curve=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_sklearn_pr_curve(self,
name,
y_preds,
y_targets,
step=None,
timestamp=None):
"""Calculates and logs PR curve using sklearn.
```python
>>> log_sklearn_pr_curve("pr_value", y_preds, y_targets, step=10)
```
Args:
name: str, name of the event
y_preds: List[float] or numpy.array
y_targets: List[float] or numpy.array
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.sklearn_pr_curve(
y_preds=y_preds,
y_targets=y_targets,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp,
step=step,
curve=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_curve(self,
name,
x,
y,
annotation=None,
step=None,
timestamp=None):
"""Logs a custom curve.
```python
>>> log_curve("pr_value", x, y, annotation="more=info", step=10)
```
Args:
name: str, name of the curve
x: List[float] or numpy.array
y: List[float] or numpy.array
annotation: str, optional
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.curve(
x=x,
y=y,
annotation=annotation,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp,
step=step,
curve=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_image(self,
data,
name=None,
step=None,
timestamp=None,
rescale=1,
dataformats="CHW"):
"""Logs an image.
```python
>>> log_image(data="path/to/image.png", step=10)
>>> log_image(data=np_array, name="generated_image", step=10)
```
Args:
data: str or numpy.array, a file path or numpy array
name: str, name of the image,
if a path is passed this can be optional and the name of the file will be used
step: int, optional
timestamp: datetime, optional
rescale: int, optional
dataformats: str, optional
"""
self._log_has_events()
is_file = isinstance(data, str) and os.path.exists(data)
ext = "png"
if is_file:
name = name or os.path.basename(data)
ext = get_path_extension(filepath=data) or ext
else:
name = name or "image"
asset_path = get_asset_path(
run_path=self._artifacts_path,
kind=V1ArtifactKind.IMAGE,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self._artifacts_path)
if is_file:
event_value = events_processors.image_path(from_path=data,
asset_path=asset_path)
elif hasattr(data, "encoded_image_string"):
event_value = events_processors.encoded_image(
asset_path=asset_path,
data=data,
asset_rel_path=asset_rel_path,
)
else:
event_value = events_processors.image(
asset_path=asset_path,
data=data,
rescale=rescale,
dataformats=dataformats,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.IMAGE,
event=V1Event.make(timestamp=timestamp,
step=step,
image=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_image_with_boxes(
self,
tensor_image,
tensor_boxes,
name=None,
step=None,
timestamp=None,
rescale=1,
dataformats="CHW",
):
"""Logs an image with bounding boxes.
```python
>>> log_image_with_boxes(
>>> name="my_image",
>>> tensor_image=np.arange(np.prod((3, 32, 32)), dtype=float).reshape((3, 32, 32)),
>>> tensor_boxes=np.array([[10, 10, 40, 40]]),
>>> )
```
Args:
tensor_image: numpy.array or str: Image data or file name
tensor_boxes: numpy.array or str: Box data (for detected objects)
box should be represented as [x1, y1, x2, y2]
name: str, name of the image
step: int, optional
timestamp: datetime, optional
rescale: int, optional
dataformats: str, optional
"""
self._log_has_events()
name = name or "figure"
asset_path = get_asset_path(
run_path=self._artifacts_path,
kind=V1ArtifactKind.IMAGE,
name=name,
step=step,
)
asset_rel_path = os.path.relpath(asset_path, self._artifacts_path)
event_value = events_processors.image_boxes(
asset_path=asset_path,
tensor_image=tensor_image,
tensor_boxes=tensor_boxes,
rescale=rescale,
dataformats=dataformats,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.IMAGE,
event=V1Event.make(timestamp=timestamp,
step=step,
image=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_mpl_image(self,
data,
name=None,
close=True,
step=None,
timestamp=None):
"""Logs a matplotlib image.
```python
>>> log_mpl_image(name="figure", data=figure, step=1, close=False)
```
Args:
data: matplotlib.pyplot.figure or List[matplotlib.pyplot.figure]
name: sre, optional, name
close: bool, optional, default True
step: int, optional
timestamp: datetime, optional
"""
name = name or "figure"
if isinstance(data, list):
event_value = events_processors.figures_to_images(figures=data,
close=close)
if event_value == UNKNOWN:
return
self.log_image(
name=name,
data=event_value,
step=step,
timestamp=timestamp,
dataformats="NCHW",
)
else:
event_value = events_processors.figure_to_image(figure=data,
close=close)
self.log_image(
name=name,
data=event_value,
step=step,
timestamp=timestamp,
dataformats="CHW",
)
@check_no_op
@check_offline
@can_log_events
def log_video(self,
data,
name=None,
fps=4,
step=None,
timestamp=None,
content_type=None):
"""Logs a video.
```python
>>> log_video("path/to/my_video1"),
>>> log_video(name="my_vide2", data=np.arange(np.prod((4, 3, 1, 8, 8)), dtype=float).reshape((4, 3, 1, 8, 8))) # noqa
```
Args:
data: video data or str.
name: str, optional, if data is a filepath the name will be the name of the file
fps: int, optional, frames per second
step: int, optional
timestamp: datetime, optional
content_type: str, optional, default "gif"
"""
self._log_has_events()
is_file = isinstance(data, str) and os.path.exists(data)
content_type = content_type or "gif"
if is_file:
name = name or os.path.basename(data)
content_type = get_path_extension(filepath=data) or content_type
else:
name = name or "video"
asset_path = get_asset_path(
run_path=self._artifacts_path,
kind=V1ArtifactKind.VIDEO,
name=name,
step=step,
ext=content_type,
)
asset_rel_path = os.path.relpath(asset_path, self._artifacts_path)
if is_file:
event_value = events_processors.video_path(
from_path=data,
asset_path=asset_path,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
else:
event_value = events_processors.video(
asset_path=asset_path,
tensor=data,
fps=fps,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.VIDEO,
event=V1Event.make(timestamp=timestamp,
step=step,
video=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_audio(
self,
data,
name=None,
sample_rate=44100,
step=None,
timestamp=None,
content_type=None,
):
"""Logs a audio.
```python
>>> log_audio("path/to/my_audio1"),
>>> log_audio(name="my_audio2", data=np.arange(np.prod((42,)), dtype=float).reshape((42,)))
```
Args:
data: str or audio data
name: str, optional, if data is a filepath the name will be the name of the file
sample_rate: int, optional, sample rate in Hz
step: int, optional
timestamp: datetime, optional
content_type: str, optional, default "wav"
"""
self._log_has_events()
is_file = isinstance(data, str) and os.path.exists(data)
ext = content_type or "wav"
if is_file:
name = name or os.path.basename(data)
ext = get_path_extension(filepath=data) or ext
else:
name = name or "audio"
asset_path = get_asset_path(
run_path=self._artifacts_path,
kind=V1ArtifactKind.AUDIO,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self._artifacts_path)
if is_file:
event_value = events_processors.audio_path(
from_path=data,
asset_path=asset_path,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
else:
event_value = events_processors.audio(
asset_path=asset_path,
tensor=data,
sample_rate=sample_rate,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.AUDIO,
event=V1Event.make(timestamp=timestamp,
step=step,
audio=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_text(self, name, text, step=None, timestamp=None):
"""Logs a text.
```python
>>> log_text(name="text", text="value")
```
Args:
name: str, name
text: str, text value
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.TEXT,
event=V1Event.make(timestamp=timestamp, step=step, text=text),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_html(self, name, html, step=None, timestamp=None):
"""Logs an html.
```python
>>> log_html(name="text", html="<p>value</p>")
```
Args:
name: str, name
html: str, text value
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.HTML,
event=V1Event.make(timestamp=timestamp, step=step, html=html),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_np_histogram(self,
name,
values,
counts,
step=None,
timestamp=None):
"""Logs a numpy histogram.
```python
>>> values, counts = np.histogram(np.random.randint(255, size=(1000,)))
>>> log_np_histogram(name="histo1", values=values, counts=counts, step=1)
```
Args:
name: str, name
values: np.array
counts: np.array
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.np_histogram(values=values,
counts=counts)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.HISTOGRAM,
event=V1Event.make(timestamp=timestamp,
step=step,
histogram=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_histogram(self,
name,
values,
bins,
max_bins=None,
step=None,
timestamp=None):
"""Logs a histogram.
```python
>>> log_histogram(name="histo", values=np.arange(np.prod((1024,)), dtype=float).reshape((1024,)), bins="auto", step=1) # noqa
```
Args:
name: str, name
values: np.array
bins: int or str
max_bins: int, optional
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
event_value = events_processors.histogram(values=values,
bins=bins,
max_bins=max_bins)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.HISTOGRAM,
event=V1Event.make(timestamp=timestamp,
step=step,
histogram=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_model(self,
path,
name=None,
framework=None,
spec=None,
step=None,
timestamp=None):
"""Logs a model.
Args:
path: str, path to the model to log
name: str, name
framework: str, optional ,name of the framework
spec: Dict, optional, key, value information about the model
step: int, optional
timestamp: datetime, optional
"""
self._log_has_model()
name = name or os.path.basename(path)
ext = None
if os.path.isfile(path):
ext = get_path_extension(filepath=path)
asset_path = get_asset_path(
run_path=self._artifacts_path,
kind=V1ArtifactKind.MODEL,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self._artifacts_path)
model = events_processors.model_path(
from_path=path,
asset_path=asset_path,
framework=framework,
spec=spec,
asset_rel_path=asset_rel_path,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.MODEL,
event=V1Event.make(timestamp=timestamp, step=step, model=model),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_dataframe(self,
path,
name=None,
content_type=None,
step=None,
timestamp=None):
"""Logs a dataframe.
Args:
path: path to the dataframe saved as file
name: str, optional, if not provided the name of the file will be used
content_type: str, optional
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
name = name or os.path.basename(path)
ext = get_path_extension(filepath=path)
asset_path = get_asset_path(
run_path=self._artifacts_path,
kind=V1ArtifactKind.DATAFRAME,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self._artifacts_path)
df = events_processors.dataframe_path(
from_path=path,
asset_path=asset_path,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.DATAFRAME,
event=V1Event.make(timestamp=timestamp, step=step, dataframe=df),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_artifact(self,
path,
name=None,
artifact_kind=None,
step=None,
timestamp=None):
"""Logs a generic artifact.
Args:
path: str, path to the artifact
name: str, optional, if not provided the name of the file will be used
artifact_kind: optional, str
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
name = name or os.path.basename(name)
ext = get_path_extension(filepath=path)
artifact_kind = artifact_kind or V1ArtifactKind.FILE
asset_path = get_asset_path(
run_path=self._artifacts_path,
kind=artifact_kind,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self._artifacts_path)
artifact = events_processors.artifact_path(
from_path=path,
asset_path=asset_path,
kind=artifact_kind,
asset_rel_path=asset_rel_path,
)
logged_event = LoggedEventSpec(
name=name,
kind=artifact_kind,
event=V1Event.make(timestamp=timestamp,
step=step,
artifact=artifact),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_plotly_chart(self, name, figure, step=None, timestamp=None):
"""Logs a plotly chart/figure.
Args:
name: str, name of the figure
figure: plotly.figure
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
chart = events_processors.plotly_chart(figure=figure)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CHART,
event=V1Event.make(timestamp=timestamp, step=step, chart=chart),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_bokeh_chart(self, name, figure, step=None, timestamp=None):
"""Logs a bokeh chart/figure.
Args:
name: str, name of the figure
figure: bokeh.figure
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
chart = events_processors.bokeh_chart(figure=figure)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CHART,
event=V1Event.make(timestamp=timestamp, step=step, chart=chart),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_altair_chart(self, name, figure, step=None, timestamp=None):
"""Logs a vega/altair chart/figure.
Args:
name: str, name of the figure
figure: figure
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
chart = events_processors.altair_chart(figure=figure)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CHART,
event=V1Event.make(timestamp=timestamp, step=step, chart=chart),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_mpl_plotly_chart(self, name, figure, step=None, timestamp=None):
"""Logs a matplotlib figure to plotly figure.
Args:
name: str, name of the figure
figure: figure
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
chart = events_processors.mpl_plotly_chart(figure=figure)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CHART,
event=V1Event.make(timestamp=timestamp, step=step, chart=chart),
)
self._event_logger.add_event(logged_event)
@check_no_op
def get_log_level(self):
return get_log_level()
@check_no_op
@check_offline
def _register_wait(self):
atexit.register(self._wait)
@check_no_op
@check_offline
def _start(self):
self.start()
atexit.register(self._end)
def excepthook(exception, value, tb):
self.log_failed(
message="Type: {}, Value: {}".format(exception, value))
# Resume normal work
sys.__excepthook__(exception, value, tb)
sys.excepthook = excepthook
@check_no_op
@check_offline
def _end(self):
self.log_succeeded()
self._wait()
@check_no_op
@check_offline
def _wait(self):
if self._event_logger:
self._event_logger.close()
if self._resource_logger:
self._resource_logger.close()
if self._results:
self.log_outputs(**self._results)
time.sleep(settings.CLIENT_CONFIG.tracking_timeout)
@check_no_op
@check_offline
@can_log_outputs
def log_env(self):
"""Logs information about the environment.
Called automatically if track_env is set to True.
"""
env_data = get_run_env()
with open(os.path.join(self._outputs_path, "env.json"),
"w") as env_file:
env_file.write(ujson.dumps(env_data))
class TestRunLogging(TestEnvVarsCase):
def setUp(self):
super().setUp()
self.run_path = tempfile.mkdtemp()
settings.CLIENT_CONFIG.is_managed = False
os.environ[POLYAXON_KEYS_COLLECT_ARTIFACTS] = "false"
os.environ[POLYAXON_KEYS_COLLECT_RESOURCES] = "false"
with patch("polyaxon.tracking.run.Run.refresh_data") as _:
self.run = Run(project="owner.project")
self.event_logger = EventFileWriter(run_path=self.run_path)
self.resource_logger = ResourceFileWriter(run_path=self.run_path)
self.run._artifacts_path = self.run_path
self.run._event_logger = self.event_logger
self.run._resource_logger = self.resource_logger
assert os.path.exists(get_event_path(self.run_path)) is True
assert os.path.exists(get_asset_path(self.run_path)) is True
@staticmethod
def touch(path):
with open(path, "w") as f:
f.write("test")
def test_log_empty_metric(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.METRIC))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.METRIC))
is False
)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_metrics()
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.METRIC))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.METRIC))
is False
)
def test_log_single_metric(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.METRIC))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.METRIC))
is False
)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_metrics(step=1, metric1=1.1)
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.METRIC))
is False
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.METRIC))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.METRIC, name="metric1"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="metric", name="metric1", data=events_file)
assert len(results.df.values) == 1
def test_log_multiple_metric(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.METRIC))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.METRIC))
is False
)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_metrics(step=1, metric1=1.1, metric2=21.1)
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.METRIC))
is False
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.METRIC))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.METRIC, name="metric1"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="metric", name="metric1", data=events_file)
assert len(results.df.values) == 1
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.METRIC, name="metric2"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="metric", name="metric2", data=events_file)
assert len(results.df.values) == 1
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_metrics(step=2, metric1=1.1, metric2=21.1, metric3=12.1)
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.METRIC))
is False
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.METRIC))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.METRIC, name="metric1"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="metric", name="metric1", data=events_file)
assert len(results.df.values) == 2
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.METRIC, name="metric2"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="metric", name="metric2", data=events_file)
assert len(results.df.values) == 2
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.METRIC, name="metric3"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="metric", name="metric3", data=events_file)
assert len(results.df.values) == 1
def test_log_image_from_path(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is False
)
image_file = tempfile.mkdtemp() + "/file.png"
self.touch(image_file)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_image(name="my_image", data=image_file)
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is True
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.IMAGE, name="my_image"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="image", name="my_image", data=events_file)
assert len(results.df.values) == 1
asset_file = get_asset_path(
self.run_path, kind=V1ArtifactKind.IMAGE, name="my_image", ext="png"
)
assert os.path.exists(asset_file) is True
def test_log_image_from_path_with_step(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is False
)
image_file = tempfile.mkdtemp() + "/file.png"
self.touch(image_file)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_image(name="my_image", data=image_file, step=1)
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is True
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.IMAGE, name="my_image"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="image", name="my_image", data=events_file)
assert len(results.df.values) == 1
asset_file = get_asset_path(
self.run_path, kind=V1ArtifactKind.IMAGE, name="my_image", step=1, ext="png"
)
assert os.path.exists(asset_file) is True
def test_log_data_image(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is False
)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_image(
name="my_image", data=tensor_np(shape=(1, 8, 8)), dataformats="CHW"
)
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is True
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.IMAGE, name="my_image"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="image", name="my_image", data=events_file)
assert len(results.df.values) == 1
asset_file = get_asset_path(
self.run_path, kind=V1ArtifactKind.IMAGE, name="my_image", ext="png"
)
assert os.path.exists(asset_file) is True
def test_log_image_with_boxes(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is False
)
image_file = tempfile.mkdtemp() + "/file.png"
self.touch(image_file)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_image_with_boxes(
name="my_image",
tensor_image=tensor_np(shape=(3, 32, 32)),
tensor_boxes=np.array([[10, 10, 40, 40]]),
)
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is True
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.IMAGE, name="my_image"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="image", name="my_image", data=events_file)
assert len(results.df.values) == 1
asset_file = get_asset_path(
self.run_path, kind=V1ArtifactKind.IMAGE, name="my_image"
)
assert os.path.exists(asset_file) is True
def test_log_mpl_image(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is False
)
figure, axes = plt.figure(), plt.gca()
circle1 = plt.Circle((0.2, 0.5), 0.2, color="r")
circle2 = plt.Circle((0.8, 0.5), 0.2, color="g")
axes.add_patch(circle1)
axes.add_patch(circle2)
plt.axis("scaled")
plt.tight_layout()
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_mpl_image(name="figure", data=figure, step=1, close=False)
assert log_dashboard.call_count == 1
assert plt.fignum_exists(figure.number) is True
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is True
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.IMAGE, name="figure"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="image", name="figure", data=events_file)
assert len(results.df.values) == 1
asset_file = get_asset_path(
self.run_path, kind=V1ArtifactKind.IMAGE, name="figure", step=1, ext="png"
)
assert os.path.exists(asset_file) is True
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_mpl_image(name="figure", data=figure, step=2)
assert log_dashboard.call_count == 1
assert plt.fignum_exists(figure.number) is False
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is True
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.IMAGE, name="figure"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="image", name="figure", data=events_file)
assert len(results.df.values) == 2
asset_file = get_asset_path(
self.run_path, kind=V1ArtifactKind.IMAGE, name="figure", step=1, ext="png"
)
assert os.path.exists(asset_file) is True
@pytest.mark.filterwarnings("ignore::FutureWarning")
def test_log_mpl_images(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is False
)
figures = []
for i in range(5):
figure = plt.figure()
plt.plot([i * 1, i * 2, i * 3], label="Plot " + str(i))
plt.xlabel("X")
plt.xlabel("Y")
plt.legend()
plt.tight_layout()
figures.append(figure)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_mpl_image(name="figure", data=figures, step=1, close=False)
assert log_dashboard.call_count == 1
assert all([plt.fignum_exists(figure.number) is True for figure in figures])
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is True
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.IMAGE, name="figure"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="image", name="figure", data=events_file)
assert len(results.df.values) == 1
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_mpl_image(name="figure", data=figures, step=2)
assert log_dashboard.call_count == 1
assert all([plt.fignum_exists(figure.number) is False for figure in figures])
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is True
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.IMAGE))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.IMAGE, name="figure"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="image", name="figure", data=events_file)
assert len(results.df.values) == 2
asset_file = get_asset_path(
self.run_path, kind=V1ArtifactKind.IMAGE, name="figure", step=1, ext="png"
)
assert os.path.exists(asset_file) is True
@pytest.mark.filterwarnings("ignore::RuntimeWarning")
def test_log_mpl_plotly(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.CHART))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.CHART))
is False
)
figure, axes = plt.figure(), plt.gca()
circle1 = plt.Circle((0.2, 0.5), 0.2, color="r")
circle2 = plt.Circle((0.8, 0.5), 0.2, color="g")
axes.add_patch(circle1)
axes.add_patch(circle2)
plt.axis("scaled")
plt.tight_layout()
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_mpl_plotly_chart(name="figure", figure=figure, step=1)
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.CHART))
is False
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.CHART))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.CHART, name="figure"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="image", name="figure", data=events_file)
assert len(results.df.values) == 1
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_mpl_plotly_chart(name="figure", figure=figure, step=2)
assert log_dashboard.call_count == 1
assert plt.fignum_exists(figure.number) is False
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.CHART))
is False
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.CHART))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.CHART, name="figure"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="image", name="figure", data=events_file)
assert len(results.df.values) == 2
def test_log_video_from_path(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.VIDEO))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.VIDEO))
is False
)
video_file = tempfile.mkdtemp() + "/video.gif"
self.touch(video_file)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_video(name="my_video", data=video_file)
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.VIDEO))
is True
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.VIDEO))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.VIDEO, name="my_video"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="video", name="my_video", data=events_file)
assert len(results.df.values) == 1
asset_file = get_asset_path(
self.run_path, kind=V1ArtifactKind.VIDEO, name="my_video", ext="gif"
)
assert os.path.exists(asset_file) is True
def test_log_data_video(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.VIDEO))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.VIDEO))
is False
)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_video(name="my_video", data=tensor_np(shape=(4, 3, 1, 8, 8)))
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.VIDEO))
is True
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.VIDEO))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.VIDEO, name="my_video"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="video", name="my_video", data=events_file)
assert len(results.df.values) == 1
asset_file = get_asset_path(
self.run_path, kind=V1ArtifactKind.VIDEO, name="my_video", ext="gif"
)
assert os.path.exists(asset_file) is True
def test_log_audio_from_path(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.AUDIO))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.AUDIO))
is False
)
audio_file = tempfile.mkdtemp() + "/audio.wav"
self.touch(audio_file)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_audio(name="my_audio", data=audio_file)
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.AUDIO))
is True
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.AUDIO))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.AUDIO, name="my_audio"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="audio", name="my_audio", data=events_file)
assert len(results.df.values) == 1
asset_file = get_asset_path(
self.run_path, kind=V1ArtifactKind.AUDIO, name="my_audio", ext="wav"
)
assert os.path.exists(asset_file) is True
def test_log_data_audio(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.AUDIO))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.AUDIO))
is False
)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_audio(name="my_audio", data=tensor_np(shape=(42,)))
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.AUDIO))
is True
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.AUDIO))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.AUDIO, name="my_audio"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="audio", name="my_audio", data=events_file)
assert len(results.df.values) == 1
asset_file = get_asset_path(
self.run_path, kind=V1ArtifactKind.AUDIO, name="my_audio", ext="wav"
)
assert os.path.exists(asset_file) is True
def test_log_text(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.TEXT))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.TEXT))
is False
)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_text(name="my_text", text="some text", step=1)
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.TEXT))
is False
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.TEXT))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.TEXT, name="my_text"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="text", name="my_text", data=events_file)
assert len(results.df.values) == 1
def test_log_html(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.HTML))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.HTML))
is False
)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_html(name="my_div", html="<div>test<div/>", step=1)
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.HTML))
is False
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.HTML))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.HTML, name="my_div"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="html", name="my_div", data=events_file)
assert len(results.df.values) == 1
def test_log_histogram(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.HISTOGRAM))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.HISTOGRAM))
is False
)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_histogram(
name="histo", values=tensor_np(shape=(1024,)), bins="auto", step=1
)
self.run.log_histogram(
name="histo", values=tensor_np(shape=(1024,)), bins="fd", step=1
)
self.run.log_histogram(
name="histo", values=tensor_np(shape=(1024,)), bins="doane", step=1
)
assert log_dashboard.call_count == 3
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.HISTOGRAM))
is False
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.HISTOGRAM))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.HISTOGRAM, name="histo"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="histogram", name="histo", data=events_file)
assert len(results.df.values) == 3
def test_log_np_histogram(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.HISTOGRAM))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.HISTOGRAM))
is False
)
values, counts = np.histogram(np.random.randint(255, size=(1000,)))
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_np_histogram(
name="histo", values=values, counts=counts, step=1
)
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.HISTOGRAM))
is False
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.HISTOGRAM))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.HISTOGRAM, name="histo"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="histogram", name="histo", data=events_file)
assert len(results.df.values) == 1
def test_log_model_file(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.MODEL))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.MODEL))
is False
)
model_file = tempfile.mkdtemp() + "model.pkl"
self.touch(model_file)
with patch("polyaxon.tracking.run.Run._log_has_model") as log_model:
self.run.log_model(name="my_model", path=model_file, framework="scikit")
assert log_model.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.MODEL))
is True
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.MODEL))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.MODEL, name="my_model"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="model", name="my_model", data=events_file)
assert len(results.df.values) == 1
asset_file = get_asset_path(
self.run_path, kind=V1ArtifactKind.MODEL, name="my_model", ext="pkl"
)
assert os.path.exists(asset_file) is True
def test_log_model_dir(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.MODEL))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.MODEL))
is False
)
model_dir = tempfile.mkdtemp() + "/model"
create_path(model_dir)
model_file = model_dir + "/model.pkl"
self.touch(model_file)
weights_file = model_dir + "/weights"
self.touch(weights_file)
configs_file = model_dir + "/configs"
self.touch(configs_file)
with patch("polyaxon.tracking.run.Run._log_has_model") as log_model:
self.run.log_model(name="my_model", path=model_dir, framework="tensorflow")
assert log_model.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.MODEL))
is True
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.MODEL))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.MODEL, name="my_model"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="model", name="my_model", data=events_file)
assert len(results.df.values) == 1
asset_file = get_asset_path(
self.run_path, kind=V1ArtifactKind.MODEL, name="my_model"
)
assert os.path.exists(asset_file) is True
def test_log_dataframe(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.DATAFRAME))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.DATAFRAME))
is False
)
model_file = tempfile.mkdtemp() + "/df.pkl"
self.touch(model_file)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_dataframe(
name="dataframe", path=model_file, content_type="pickel"
)
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.DATAFRAME))
is True
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.DATAFRAME))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.DATAFRAME, name="dataframe"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind="dataframe", name="dataframe", data=events_file)
assert len(results.df.values) == 1
asset_file = get_asset_path(
self.run_path, kind=V1ArtifactKind.DATAFRAME, name="dataframe", ext="pkl"
)
assert os.path.exists(asset_file) is True
def test_log_artifact(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.TSV))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.TSV))
is False
)
tsv_file = tempfile.mkdtemp() + "/file.tsv"
self.touch(tsv_file)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_artifact(
name="file", path=tsv_file, artifact_kind=V1ArtifactKind.TSV
)
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.TSV))
is True
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.TSV))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.TSV, name="file"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind=V1ArtifactKind.TSV, name="file", data=events_file)
assert len(results.df.values) == 1
asset_file = get_asset_path(
self.run_path, kind=V1ArtifactKind.TSV, name="file", ext="tsv"
)
assert os.path.exists(asset_file) is True
def test_log_artifacts(self):
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.TSV))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.TSV))
is False
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.DATAFRAME))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.DATAFRAME))
is False
)
tsv_file = tempfile.mkdtemp() + "/file.tsv"
self.touch(tsv_file)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_artifact(
name="file", path=tsv_file, artifact_kind=V1ArtifactKind.TSV
)
assert log_dashboard.call_count == 1
pd_file = tempfile.mkdtemp() + "/dataframe"
self.touch(pd_file)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_artifact(
name="file2", path=pd_file, artifact_kind=V1ArtifactKind.DATAFRAME
)
assert log_dashboard.call_count == 1
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.TSV))
is True
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.TSV))
is True
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.DATAFRAME))
is True
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.DATAFRAME))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.TSV, name="file"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind=V1ArtifactKind.TSV, name="file", data=events_file)
assert len(results.df.values) == 1
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.DATAFRAME, name="file2"
)
assert os.path.exists(events_file) is True
results = V1Events.read(kind=V1ArtifactKind.TSV, name="file", data=events_file)
assert len(results.df.values) == 1
asset_file = get_asset_path(
self.run_path, kind=V1ArtifactKind.TSV, name="file", ext="tsv"
)
assert os.path.exists(asset_file) is True
asset_file = get_asset_path(
self.run_path, kind=V1ArtifactKind.DATAFRAME, name="file2"
)
assert os.path.exists(asset_file) is True
def test_log_charts(self):
x = [1, 2, 3, 4, 5]
y = [6, 7, 2, 4, 5]
bokeh_test = figure(
title="simple line example", x_axis_label="x", y_axis_label="y"
)
bokeh_test.line(x, y, line_width=2)
x1 = np.random.randn(200) - 2
x2 = np.random.randn(200)
x3 = np.random.randn(200) + 2
hist_data = [x1, x2, x3]
group_labels = ["Group 1", "Group 2", "Group 3"]
plotly_test = figure_factory.create_distplot(
hist_data, group_labels, bin_size=[0.1, 0.25, 0.5]
)
df1 = pd.DataFrame([["A", "B", "C", "D"], [28, 55, 43, 91]], index=["a", "b"]).T
alt_test = alt.Chart(df1).mark_bar().encode(x="a", y="b")
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.CHART))
is False
)
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.CHART))
is False
)
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_bokeh_chart(name="bokeh_test", figure=bokeh_test, step=1)
self.run.log_plotly_chart(name="plotly_test", figure=plotly_test, step=1)
self.run.log_altair_chart(name="alt_test", figure=alt_test, step=1)
assert log_dashboard.call_count == 3
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.CHART))
is False
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.CHART))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.CHART, name="bokeh_test"
)
assert os.path.exists(events_file) is True
results = V1Events.read(
kind=V1ArtifactKind.CHART, name="bokeh_test", data=events_file
)
assert len(results.df.values) == 1
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.CHART, name="plotly_test"
)
assert os.path.exists(events_file) is True
results = V1Events.read(
kind=V1ArtifactKind.CHART, name="plotly_test", data=events_file
)
assert len(results.df.values) == 1
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.CHART, name="alt_test"
)
assert os.path.exists(events_file) is True
results = V1Events.read(
kind=V1ArtifactKind.CHART, name="alt_test", data=events_file
)
assert len(results.df.values) == 1
def test_log_curves(self):
x = [1, 2, 3, 4, 5]
y = [6, 7, 2, 4, 5]
with patch("polyaxon.tracking.run.Run._log_has_events") as log_dashboard:
self.run.log_roc_auc_curve(name="roc_test", fpr=x, tpr=y, auc=0.6, step=1)
self.run.log_pr_curve(
name="pr_test", precision=x, recall=y, average_precision=0.6, step=1
)
self.run.log_curve(name="curve_test", x=x, y=y, annotation=0.6, step=1)
assert log_dashboard.call_count == 3
self.event_logger.flush()
assert (
os.path.exists(get_asset_path(self.run_path, kind=V1ArtifactKind.CURVE))
is False
)
assert (
os.path.exists(get_event_path(self.run_path, kind=V1ArtifactKind.CURVE))
is True
)
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.CURVE, name="roc_test"
)
assert os.path.exists(events_file) is True
results = V1Events.read(
kind=V1ArtifactKind.CURVE, name="roc_test", data=events_file
)
assert len(results.df.values) == 1
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.CURVE, name="pr_test"
)
assert os.path.exists(events_file) is True
results = V1Events.read(
kind=V1ArtifactKind.CHART, name="pr_test", data=events_file
)
assert len(results.df.values) == 1
events_file = get_event_path(
self.run_path, kind=V1ArtifactKind.CURVE, name="curve_test"
)
assert os.path.exists(events_file) is True
results = V1Events.read(
kind=V1ArtifactKind.CHART, name="curve_test", data=events_file
)
assert len(results.df.values) == 1
class Run(RunClient):
@check_no_op
def __init__(
self,
owner: str = None,
project: str = None,
run_uuid: str = None,
client: RunClient = None,
track_code: bool = True,
track_env: bool = False,
refresh_data: bool = False,
artifacts_path: str = None,
):
super().__init__(
owner=owner, project=project, run_uuid=run_uuid, client=client,
)
self.track_code = track_code
self.track_env = track_env
self._has_model = False
self._has_events = False
self._has_tensorboard = False
self._artifacts_path = None
self._outputs_path = None
self._event_logger = None
self._resource_logger = None
self._results = {}
if (settings.CLIENT_CONFIG.is_managed and self.run_uuid) or artifacts_path:
self.set_artifacts_path(artifacts_path)
if (
self.artifacts_path
and settings.CLIENT_CONFIG.is_managed
and get_collect_artifact()
):
self.set_run_event_logger()
if get_collect_resources():
self.set_run_resource_logger()
self._run = polyaxon_sdk.V1Run()
if settings.CLIENT_CONFIG.is_offline:
return
if self._run_uuid and (refresh_data or settings.CLIENT_CONFIG.is_managed):
self.refresh_data()
# Track run env
if settings.CLIENT_CONFIG.is_managed and self.track_env:
self.log_env()
if settings.CLIENT_CONFIG.is_managed:
self._register_wait()
@property
def is_service(self):
if settings.CLIENT_CONFIG.no_op:
return None
return settings.CLIENT_CONFIG.is_managed and settings.CLIENT_CONFIG.is_service
@property
def artifacts_path(self):
return self._artifacts_path
@property
def outputs_path(self):
return self._outputs_path
@check_no_op
@check_offline
@can_log_outputs
def get_tensorboard_path(self):
path = os.path.join(self._outputs_path, "tensorboard")
if not self._has_tensorboard:
self.log_tensorboard_ref(path)
self._has_tensorboard = True
return path
@check_no_op
def set_artifacts_path(self, artifacts_path: str = None):
_artifacts_path = artifacts_path or CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(
self.run_uuid
)
if artifacts_path:
_outputs_path = "{}/outputs".format(artifacts_path)
else:
_outputs_path = CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format(self.run_uuid)
self._artifacts_path = _artifacts_path
self._outputs_path = _outputs_path
@check_no_op
def set_run_event_logger(self):
self._event_logger = EventFileWriter(run_path=self.artifacts_path)
@check_no_op
def set_run_resource_logger(self):
self._resource_logger = ResourceFileWriter(run_path=self.artifacts_path)
def _post_create(self):
if self.artifacts_path:
self.set_run_event_logger()
if self.track_code:
self.log_code_ref()
if self.track_env:
self.log_env()
if not settings.CLIENT_CONFIG.is_managed:
self._start()
else:
self._register_wait()
def _log_has_events(self):
if not self._has_events:
self._has_events = True
self._log_meta(has_events=True)
def _log_has_model(self):
if not self._has_model:
self._has_model = True
self._log_meta(has_model=True)
@check_no_op
@check_offline
@can_log_events
def log_metric(self, name, value, step=None, timestamp=None):
self._log_has_events()
events = []
event_value = events_processors.metric(value)
if event_value == UNKNOWN:
return
events.append(
LoggedEventSpec(
name=name,
kind=V1ArtifactKind.METRIC,
event=V1Event.make(timestamp=timestamp, step=step, metric=event_value),
)
)
if events:
self._event_logger.add_events(events)
self._results[name] = event_value
@check_no_op
@check_offline
@can_log_events
def log_metrics(self, step=None, timestamp=None, **metrics):
self._log_has_events()
events = []
for metric in metrics:
event_value = events_processors.metric(metrics[metric])
if event_value == UNKNOWN:
continue
events.append(
LoggedEventSpec(
name=metric,
kind=V1ArtifactKind.METRIC,
event=V1Event.make(
timestamp=timestamp, step=step, metric=event_value
),
)
)
if events:
self._event_logger.add_events(events)
@check_no_op
@check_offline
@can_log_events
def log_roc_auc_curve(self, name, fpr, tpr, auc=None, step=None, timestamp=None):
self._log_has_events()
event_value = events_processors.roc_auc_curve(fpr=fpr, tpr=tpr, auc=auc,)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp, step=step, curve=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_sklearn_roc_auc_curve(
self, name, y_preds, y_targets, step=None, timestamp=None
):
self._log_has_events()
event_value = events_processors.sklearn_roc_auc_curve(
y_preds=y_preds, y_targets=y_targets,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp, step=step, curve=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_pr_curve(
self, name, precision, recall, average_precision=None, step=None, timestamp=None
):
self._log_has_events()
event_value = events_processors.pr_curve(
precision=precision, recall=recall, average_precision=average_precision,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp, step=step, curve=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_sklearn_pr_curve(self, name, y_preds, y_targets, step=None, timestamp=None):
self._log_has_events()
event_value = events_processors.sklearn_pr_curve(
y_preds=y_preds, y_targets=y_targets,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp, step=step, curve=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_curve(self, name, x, y, annotation=None, step=None, timestamp=None):
self._log_has_events()
event_value = events_processors.curve(x=x, y=y, annotation=annotation,)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CURVE,
event=V1Event.make(timestamp=timestamp, step=step, curve=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_image(
self, data, name=None, step=None, timestamp=None, rescale=1, dataformats="CHW"
):
self._log_has_events()
is_file = isinstance(data, str) and os.path.exists(data)
ext = "png"
if is_file:
name = name or os.path.basename(data)
ext = get_path_extension(filepath=data) or ext
else:
name = name or "image"
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.IMAGE,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self.artifacts_path)
if is_file:
event_value = events_processors.image_path(
from_path=data, asset_path=asset_path
)
elif hasattr(data, "encoded_image_string"):
event_value = events_processors.encoded_image(
asset_path=asset_path, data=data, asset_rel_path=asset_rel_path,
)
else:
event_value = events_processors.image(
asset_path=asset_path,
data=data,
rescale=rescale,
dataformats=dataformats,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.IMAGE,
event=V1Event.make(timestamp=timestamp, step=step, image=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_image_with_boxes(
self,
tensor_image,
tensor_boxes,
name=None,
step=None,
timestamp=None,
rescale=1,
dataformats="CHW",
):
self._log_has_events()
name = name or "figure"
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.IMAGE,
name=name,
step=step,
)
asset_rel_path = os.path.relpath(asset_path, self.artifacts_path)
event_value = events_processors.image_boxes(
asset_path=asset_path,
tensor_image=tensor_image,
tensor_boxes=tensor_boxes,
rescale=rescale,
dataformats=dataformats,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.IMAGE,
event=V1Event.make(timestamp=timestamp, step=step, image=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_mpl_image(self, data, name=None, close=True, step=None, timestamp=None):
name = name or "figure"
if isinstance(data, list):
event_value = events_processors.figures_to_images(figures=data, close=close)
if event_value == UNKNOWN:
return
self.log_image(
name=name,
data=event_value,
step=step,
timestamp=timestamp,
dataformats="NCHW",
)
else:
event_value = events_processors.figure_to_image(figure=data, close=close)
self.log_image(
name=name,
data=event_value,
step=step,
timestamp=timestamp,
dataformats="CHW",
)
@check_no_op
@check_offline
@can_log_events
def log_video(
self, data, name=None, fps=4, step=None, timestamp=None, content_type=None
):
self._log_has_events()
is_file = isinstance(data, str) and os.path.exists(data)
content_type = content_type or "gif"
if is_file:
name = name or os.path.basename(data)
content_type = get_path_extension(filepath=data) or content_type
else:
name = name or "video"
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.VIDEO,
name=name,
step=step,
ext=content_type,
)
asset_rel_path = os.path.relpath(asset_path, self.artifacts_path)
if is_file:
event_value = events_processors.video_path(
from_path=data,
asset_path=asset_path,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
else:
event_value = events_processors.video(
asset_path=asset_path,
tensor=data,
fps=fps,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.VIDEO,
event=V1Event.make(timestamp=timestamp, step=step, video=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_audio(
self,
data,
name=None,
sample_rate=44100,
step=None,
timestamp=None,
content_type=None,
):
self._log_has_events()
is_file = isinstance(data, str) and os.path.exists(data)
ext = content_type or "wav"
if is_file:
name = name or os.path.basename(data)
ext = get_path_extension(filepath=data) or ext
else:
name = name or "audio"
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.AUDIO,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self.artifacts_path)
if is_file:
event_value = events_processors.audio_path(
from_path=data,
asset_path=asset_path,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
else:
event_value = events_processors.audio(
asset_path=asset_path,
tensor=data,
sample_rate=sample_rate,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.AUDIO,
event=V1Event.make(timestamp=timestamp, step=step, audio=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_text(self, name, text, step=None, timestamp=None):
self._log_has_events()
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.TEXT,
event=V1Event.make(timestamp=timestamp, step=step, text=text),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_html(self, name, html, step=None, timestamp=None):
self._log_has_events()
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.HTML,
event=V1Event.make(timestamp=timestamp, step=step, html=html),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_np_histogram(self, name, values, counts, step=None, timestamp=None):
self._log_has_events()
event_value = events_processors.np_histogram(values=values, counts=counts)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.HISTOGRAM,
event=V1Event.make(timestamp=timestamp, step=step, histogram=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_histogram(
self, name, values, bins, max_bins=None, step=None, timestamp=None
):
self._log_has_events()
event_value = events_processors.histogram(
values=values, bins=bins, max_bins=max_bins
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.HISTOGRAM,
event=V1Event.make(timestamp=timestamp, step=step, histogram=event_value),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_model(
self, path, name=None, framework=None, spec=None, step=None, timestamp=None
):
self._log_has_model()
name = name or os.path.basename(path)
ext = None
if os.path.isfile(path):
ext = get_path_extension(filepath=path)
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.MODEL,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self.artifacts_path)
model = events_processors.model_path(
from_path=path,
asset_path=asset_path,
framework=framework,
spec=spec,
asset_rel_path=asset_rel_path,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.MODEL,
event=V1Event.make(timestamp=timestamp, step=step, model=model),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_dataframe(
self, path, name=None, content_type=None, step=None, timestamp=None
):
self._log_has_events()
name = name or os.path.basename(path)
ext = get_path_extension(filepath=path)
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.DATAFRAME,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self.artifacts_path)
df = events_processors.dataframe_path(
from_path=path,
asset_path=asset_path,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.DATAFRAME,
event=V1Event.make(timestamp=timestamp, step=step, dataframe=df),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_artifact(
self, path, name=None, artifact_kind=None, step=None, timestamp=None
):
self._log_has_events()
name = name or os.path.basename(name)
ext = get_path_extension(filepath=path)
artifact_kind = artifact_kind or V1ArtifactKind.FILE
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=artifact_kind,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self.artifacts_path)
artifact = events_processors.artifact_path(
from_path=path,
asset_path=asset_path,
kind=artifact_kind,
asset_rel_path=asset_rel_path,
)
logged_event = LoggedEventSpec(
name=name,
kind=artifact_kind,
event=V1Event.make(timestamp=timestamp, step=step, artifact=artifact),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_plotly_chart(self, name, figure, step=None, timestamp=None):
self._log_has_events()
chart = events_processors.plotly_chart(figure=figure)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CHART,
event=V1Event.make(timestamp=timestamp, step=step, chart=chart),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_bokeh_chart(self, name, figure, step=None, timestamp=None):
self._log_has_events()
chart = events_processors.bokeh_chart(figure=figure)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CHART,
event=V1Event.make(timestamp=timestamp, step=step, chart=chart),
)
self._event_logger.add_event(logged_event)
@check_no_op
@check_offline
@can_log_events
def log_mpl_plotly_chart(self, name, figure, step=None, timestamp=None):
self._log_has_events()
chart = events_processors.mpl_plotly_chart(figure=figure)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.CHART,
event=V1Event.make(timestamp=timestamp, step=step, chart=chart),
)
self._event_logger.add_event(logged_event)
@check_no_op
def get_log_level(self):
return get_log_level()
@check_no_op
@check_offline
def _register_wait(self):
atexit.register(self._wait)
@check_no_op
@check_offline
def _start(self):
self.start()
atexit.register(self._end)
def excepthook(exception, value, tb):
self.log_failed(message="Type: {}, Value: {}".format(exception, value))
# Resume normal work
sys.__excepthook__(exception, value, tb)
sys.excepthook = excepthook
@check_no_op
@check_offline
def _end(self):
self.log_succeeded()
self._wait()
@check_no_op
@check_offline
def _wait(self):
if self._event_logger:
self._event_logger.close()
if self._resource_logger:
self._resource_logger.close()
if self._results:
self.log_outputs(**self._results)
time.sleep(settings.CLIENT_CONFIG.tracking_timeout)
@check_no_op
@check_offline
@can_log_outputs
def log_env(self):
env_data = get_run_env()
with open(os.path.join(self.outputs_path, ".polyaxon"), "w") as env_file:
env_file.write(env_data)