def log_artifact(self,
path,
name=None,
artifact_kind=None,
step=None,
timestamp=None):
name = name or os.path.basename(name)
ext = get_path_extension(filepath=path)
artifact_kind = artifact_kind or V1ArtifactKind.FILE
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=artifact_kind,
name=name,
step=step,
ext=ext,
)
artifact = events_processors.artifact_path(from_path=path,
asset_path=asset_path,
kind=artifact_kind)
logged_event = LoggedEventSpec(
name=name,
kind=artifact_kind,
event=V1Event(timestamp=timestamp, step=step, artifact=artifact),
)
self._event_logger.add_event(logged_event)
def log_dataframe(self,
path,
name=None,
content_type=None,
step=None,
timestamp=None):
name = name or os.path.basename(path)
ext = get_path_extension(filepath=path)
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.DATAFRAME,
name=name,
step=step,
ext=ext,
)
df = events_processors.dataframe_path(from_path=path,
asset_path=asset_path,
content_type=content_type)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.DATAFRAME,
event=V1Event(timestamp=timestamp, step=step, dataframe=df),
)
self._event_logger.add_event(logged_event)
def log_model(self,
path,
name=None,
framework=None,
spec=None,
step=None,
timestamp=None):
name = name or os.path.basename(path)
ext = None
if os.path.isfile(path):
ext = get_path_extension(filepath=path)
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.MODEL,
name=name,
step=step,
ext=ext,
)
model = events_processors.model_path(from_path=path,
asset_path=asset_path,
framework=framework,
spec=spec)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.MODEL,
event=V1Event(timestamp=timestamp, step=step, model=model),
)
self._event_logger.add_event(logged_event)
def log_audio(
self,
data,
name=None,
sample_rate=44100,
step=None,
timestamp=None,
content_type=None,
):
self._log_dashboard()
is_file = isinstance(data, str) and os.path.exists(data)
ext = content_type or "wav"
if is_file:
name = name or os.path.basename(data)
ext = get_path_extension(filepath=data) or ext
else:
name = name or "audio"
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.AUDIO,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self.artifacts_path)
if is_file:
event_value = events_processors.audio_path(
from_path=data,
asset_path=asset_path,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
else:
event_value = events_processors.audio(
asset_path=asset_path,
tensor=data,
sample_rate=sample_rate,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.AUDIO,
event=V1Event.make(timestamp=timestamp,
step=step,
audio=event_value),
)
self._event_logger.add_event(logged_event)
def log_image(self,
data,
name=None,
step=None,
timestamp=None,
rescale=1,
dataformats="CHW"):
self._log_dashboard()
is_file = isinstance(data, str) and os.path.exists(data)
ext = "png"
if is_file:
name = name or os.path.basename(data)
ext = get_path_extension(filepath=data) or ext
else:
name = name or "image"
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.IMAGE,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self.artifacts_path)
if is_file:
event_value = events_processors.image_path(from_path=data,
asset_path=asset_path)
elif hasattr(data, "encoded_image_string"):
event_value = events_processors.encoded_image(
asset_path=asset_path,
data=data,
asset_rel_path=asset_rel_path,
)
else:
event_value = events_processors.image(
asset_path=asset_path,
data=data,
rescale=rescale,
dataformats=dataformats,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.IMAGE,
event=V1Event.make(timestamp=timestamp,
step=step,
image=event_value),
)
self._event_logger.add_event(logged_event)
def log_video(self,
data,
name=None,
fps=4,
step=None,
timestamp=None,
content_type=None):
self._log_dashboard()
is_file = isinstance(data, str) and os.path.exists(data)
content_type = content_type or "gif"
if is_file:
name = name or os.path.basename(data)
content_type = get_path_extension(filepath=data) or content_type
else:
name = name or "video"
asset_path = get_asset_path(
run_path=self.artifacts_path,
kind=V1ArtifactKind.VIDEO,
name=name,
step=step,
ext=content_type,
)
asset_rel_path = os.path.relpath(asset_path, self.artifacts_path)
if is_file:
event_value = events_processors.video_path(
from_path=data,
asset_path=asset_path,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
else:
event_value = events_processors.video(
asset_path=asset_path,
tensor=data,
fps=fps,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.VIDEO,
event=V1Event.make(timestamp=timestamp,
step=step,
video=event_value),
)
self._event_logger.add_event(logged_event)
def log_model(self,
path,
name=None,
framework=None,
spec=None,
step=None,
timestamp=None):
"""Logs a model.
Args:
path: str, path to the model to log
name: str, name
framework: str, optional ,name of the framework
spec: Dict, optional, key, value information about the model
step: int, optional
timestamp: datetime, optional
"""
self._log_has_model()
name = name or os.path.basename(path)
ext = None
if os.path.isfile(path):
ext = get_path_extension(filepath=path)
asset_path = get_asset_path(
run_path=self._artifacts_path,
kind=V1ArtifactKind.MODEL,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self._artifacts_path)
model = events_processors.model_path(
from_path=path,
asset_path=asset_path,
framework=framework,
spec=spec,
asset_rel_path=asset_rel_path,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.MODEL,
event=V1Event.make(timestamp=timestamp, step=step, model=model),
)
self._event_logger.add_event(logged_event)
def log_artifact(self,
path,
name=None,
artifact_kind=None,
step=None,
timestamp=None):
"""Logs a generic artifact.
Args:
path: str, path to the artifact
name: str, optional, if not provided the name of the file will be used
artifact_kind: optional, str
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
name = name or os.path.basename(name)
ext = get_path_extension(filepath=path)
artifact_kind = artifact_kind or V1ArtifactKind.FILE
asset_path = get_asset_path(
run_path=self._artifacts_path,
kind=artifact_kind,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self._artifacts_path)
artifact = events_processors.artifact_path(
from_path=path,
asset_path=asset_path,
kind=artifact_kind,
asset_rel_path=asset_rel_path,
)
logged_event = LoggedEventSpec(
name=name,
kind=artifact_kind,
event=V1Event.make(timestamp=timestamp,
step=step,
artifact=artifact),
)
self._event_logger.add_event(logged_event)
def log_dataframe(self,
path,
name=None,
content_type=None,
step=None,
timestamp=None):
"""Logs a dataframe.
Args:
path: path to the dataframe saved as file
name: str, optional, if not provided the name of the file will be used
content_type: str, optional
step: int, optional
timestamp: datetime, optional
"""
self._log_has_events()
name = name or os.path.basename(path)
ext = get_path_extension(filepath=path)
asset_path = get_asset_path(
run_path=self._artifacts_path,
kind=V1ArtifactKind.DATAFRAME,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self._artifacts_path)
df = events_processors.dataframe_path(
from_path=path,
asset_path=asset_path,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.DATAFRAME,
event=V1Event.make(timestamp=timestamp, step=step, dataframe=df),
)
self._event_logger.add_event(logged_event)
def log_audio(
self,
data,
name=None,
sample_rate=44100,
step=None,
timestamp=None,
content_type=None,
):
"""Logs a audio.
```python
>>> log_audio("path/to/my_audio1"),
>>> log_audio(name="my_audio2", data=np.arange(np.prod((42,)), dtype=float).reshape((42,)))
```
Args:
data: str or audio data
name: str, optional, if data is a filepath the name will be the name of the file
sample_rate: int, optional, sample rate in Hz
step: int, optional
timestamp: datetime, optional
content_type: str, optional, default "wav"
"""
self._log_has_events()
is_file = isinstance(data, str) and os.path.exists(data)
ext = content_type or "wav"
if is_file:
name = name or os.path.basename(data)
ext = get_path_extension(filepath=data) or ext
else:
name = name or "audio"
asset_path = get_asset_path(
run_path=self._artifacts_path,
kind=V1ArtifactKind.AUDIO,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self._artifacts_path)
if is_file:
event_value = events_processors.audio_path(
from_path=data,
asset_path=asset_path,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
else:
event_value = events_processors.audio(
asset_path=asset_path,
tensor=data,
sample_rate=sample_rate,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.AUDIO,
event=V1Event.make(timestamp=timestamp,
step=step,
audio=event_value),
)
self._event_logger.add_event(logged_event)
def log_video(self,
data,
name=None,
fps=4,
step=None,
timestamp=None,
content_type=None):
"""Logs a video.
```python
>>> log_video("path/to/my_video1"),
>>> log_video(name="my_vide2", data=np.arange(np.prod((4, 3, 1, 8, 8)), dtype=float).reshape((4, 3, 1, 8, 8))) # noqa
```
Args:
data: video data or str.
name: str, optional, if data is a filepath the name will be the name of the file
fps: int, optional, frames per second
step: int, optional
timestamp: datetime, optional
content_type: str, optional, default "gif"
"""
self._log_has_events()
is_file = isinstance(data, str) and os.path.exists(data)
content_type = content_type or "gif"
if is_file:
name = name or os.path.basename(data)
content_type = get_path_extension(filepath=data) or content_type
else:
name = name or "video"
asset_path = get_asset_path(
run_path=self._artifacts_path,
kind=V1ArtifactKind.VIDEO,
name=name,
step=step,
ext=content_type,
)
asset_rel_path = os.path.relpath(asset_path, self._artifacts_path)
if is_file:
event_value = events_processors.video_path(
from_path=data,
asset_path=asset_path,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
else:
event_value = events_processors.video(
asset_path=asset_path,
tensor=data,
fps=fps,
content_type=content_type,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.VIDEO,
event=V1Event.make(timestamp=timestamp,
step=step,
video=event_value),
)
self._event_logger.add_event(logged_event)
def log_image(self,
data,
name=None,
step=None,
timestamp=None,
rescale=1,
dataformats="CHW"):
"""Logs an image.
```python
>>> log_image(data="path/to/image.png", step=10)
>>> log_image(data=np_array, name="generated_image", step=10)
```
Args:
data: str or numpy.array, a file path or numpy array
name: str, name of the image,
if a path is passed this can be optional and the name of the file will be used
step: int, optional
timestamp: datetime, optional
rescale: int, optional
dataformats: str, optional
"""
self._log_has_events()
is_file = isinstance(data, str) and os.path.exists(data)
ext = "png"
if is_file:
name = name or os.path.basename(data)
ext = get_path_extension(filepath=data) or ext
else:
name = name or "image"
asset_path = get_asset_path(
run_path=self._artifacts_path,
kind=V1ArtifactKind.IMAGE,
name=name,
step=step,
ext=ext,
)
asset_rel_path = os.path.relpath(asset_path, self._artifacts_path)
if is_file:
event_value = events_processors.image_path(from_path=data,
asset_path=asset_path)
elif hasattr(data, "encoded_image_string"):
event_value = events_processors.encoded_image(
asset_path=asset_path,
data=data,
asset_rel_path=asset_rel_path,
)
else:
event_value = events_processors.image(
asset_path=asset_path,
data=data,
rescale=rescale,
dataformats=dataformats,
asset_rel_path=asset_rel_path,
)
if event_value == UNKNOWN:
return
logged_event = LoggedEventSpec(
name=name,
kind=V1ArtifactKind.IMAGE,
event=V1Event.make(timestamp=timestamp,
step=step,
image=event_value),
)
self._event_logger.add_event(logged_event)
