def test_file_metrics_writer_context_manager(timestamp, filepath):
with metrics.SageMakerFileMetricsWriter(filepath) as writer:
writer.log_metric('foo', value=1.0, timestamp=timestamp)
entry = json.loads(open(filepath, 'r').read().strip())
assert {
'MetricName': 'foo',
'Value': 1.0,
'Timestamp': timestamp.timestamp()
}.items() <= entry.items()
def test_file_metrics_writer_context_manager(timestamp, filepath):
with metrics.SageMakerFileMetricsWriter(filepath) as writer:
writer.log_metric("foo", value=1.0, timestamp=timestamp)
entry = json.loads(open(filepath, "r").read().strip())
assert {
"MetricName": "foo",
"Value": 1.0,
"Timestamp": timestamp.timestamp()
}.items() <= entry.items()
def create(
cls,
display_name=None,
artifact_bucket=None,
artifact_prefix=None,
boto3_session=None,
sagemaker_boto_client=None,
):
"""Create a new ``Tracker`` by creating a new trial component.
Examples
.. code-block:: python
from smexperiments import tracker
my_tracker = tracker.Tracker.create()
Args:
display_name: (str, optional). The display name of the trial component to track.
artifact_bucket: (str, optional) The name of the S3 bucket to store artifacts to.
artifact_prefix: (str, optional) The prefix to write artifacts to within ``artifact_bucket``
boto3_session: (boto3.Session, optional) The boto3.Session to use to interact with AWS services.
If not specified a new default boto3 session will be created.
sagemaker_boto_client: (boto3.Client, optional) The SageMaker AWS service client to use. If not
specified a new client will be created from the specified ``boto3_session`` or default
boto3.Session.
Returns:
Tracker: The tracker for the new trial component.
"""
boto3_session = boto3_session or _utils.boto_session()
sagemaker_boto_client = sagemaker_boto_client or _utils.sagemaker_client(
)
tc = trial_component.TrialComponent.create(
trial_component_name=_utils.name("TrialComponent"),
display_name=display_name,
sagemaker_boto_client=sagemaker_boto_client,
)
metrics_writer = metrics.SageMakerFileMetricsWriter()
return cls(
tc,
metrics_writer,
_ArtifactUploader(tc.trial_component_name, artifact_bucket,
artifact_prefix, boto3_session),
)
def test_file_metrics_writer_log_metric(timestamp, filepath):
now = datetime.datetime.now(datetime.timezone.utc)
writer = metrics.SageMakerFileMetricsWriter(filepath)
writer.log_metric(metric_name="foo", value=1.0)
writer.log_metric(metric_name="foo", value=2.0, iteration_number=1)
writer.log_metric(metric_name="foo", value=3.0, timestamp=timestamp)
writer.log_metric(metric_name="foo", value=4.0, timestamp=timestamp, iteration_number=2)
writer.close()
lines = [x for x in open(filepath).read().split("\n") if x]
[entry_one, entry_two, entry_three, entry_four] = [json.loads(line) for line in lines]
assert "foo" == entry_one["MetricName"]
assert 1.0 == entry_one["Value"]
assert (now.timestamp() - entry_one["Timestamp"]) < 1
assert "IterationNumber" not in entry_one
assert 1 == entry_two["IterationNumber"]
assert timestamp.timestamp() == entry_three["Timestamp"]
assert 2 == entry_four["IterationNumber"]
def test_file_metrics_writer_flushes_buffer_every_line_log_metric(filepath):
writer = metrics.SageMakerFileMetricsWriter(filepath)
writer.log_metric(metric_name="foo", value=1.0)
lines = [x for x in open(filepath).read().split("\n") if x]
[entry_one] = [json.loads(line) for line in lines]
assert "foo" == entry_one["MetricName"]
assert 1.0 == entry_one["Value"]
writer.log_metric(metric_name="bar", value=2.0)
lines = [x for x in open(filepath).read().split("\n") if x]
[entry_one, entry_two] = [json.loads(line) for line in lines]
assert "bar" == entry_two["MetricName"]
assert 2.0 == entry_two["Value"]
writer.log_metric(metric_name="biz", value=3.0)
lines = [x for x in open(filepath).read().split("\n") if x]
[entry_one, entry_two, entry_three] = [json.loads(line) for line in lines]
assert "biz" == entry_three["MetricName"]
assert 3.0 == entry_three["Value"]
writer.close()
def load(
cls,
trial_component_name=None,
artifact_bucket=None,
artifact_prefix=None,
boto3_session=None,
sagemaker_boto_client=None,
):
"""Create a new ``Tracker`` by loading an existing trial component.
Examples:
.. code-block:: python
from smexperiments import tracker
my_tracker = tracker.Tracker.load(trial_component_name='xgboost')
Args:
trial_component_name: (str, optional). The name of the trial component to track. If specified, this
trial component must exist in SageMaker. If you invoke this method in a running SageMaker training
or processing job, then trial_component_name can be left empty. In this case, the Tracker will
resolve the trial component automatically created for your SageMaker Job.
artifact_bucket: (str, optional) The name of the S3 bucket to store artifacts to.
artifact_prefix: (str, optional) The prefix to write artifacts to within ``artifact_bucket``
boto3_session: (boto3.Session, optional) The boto3.Session to use to interact with AWS services.
If not specified a new default boto3 session will be created.
sagemaker_boto_client: (boto3.Client, optional) The SageMaker AWS service client to use. If not
specified a new client will be created from the specified ``boto3_session`` or default
boto3.Session.
Returns:
Tracker: The tracker for the given trial component.
Raises:
ValueError: If the trial component failed to load.
"""
boto3_session = boto3_session or _utils.boto_session()
sagemaker_boto_client = sagemaker_boto_client or _utils.sagemaker_client(
)
tce = _environment.TrialComponentEnvironment.load()
# Resolve the trial component for this tracker to track: If a trial component name was passed in, then load
# and track that trial component. Otherwise, try to find a trial component given the current environment,
# failing if we're unable to load one.
if trial_component_name:
tc = trial_component.TrialComponent.load(
trial_component_name=trial_component_name,
sagemaker_boto_client=sagemaker_boto_client)
elif tce:
tc = tce.get_trial_component(sagemaker_boto_client)
else:
raise ValueError(
'Could not load TrialComponent. Specify a trial_component_name or invoke "create"'
)
# if running in a SageMaker context write metrics to file
if not trial_component_name and tce.environment_type == _environment.EnvironmentType.SageMakerTrainingJob:
metrics_writer = metrics.SageMakerFileMetricsWriter()
else:
metrics_writer = None
tracker = cls(
tc,
metrics_writer,
_ArtifactUploader(tc.trial_component_name, artifact_bucket,
artifact_prefix, boto3_session),
)
tracker._in_sagemaker_job = True if tce else False
return tracker
def test_file_metrics_writer_no_write(filepath):
writer = metrics.SageMakerFileMetricsWriter(filepath)
writer.close()
assert not os.path.exists(filepath)
def test_file_metrics_writer_fail_write_on_close(filepath):
writer = metrics.SageMakerFileMetricsWriter(filepath)
writer.log_metric(metric_name='foo', value=1.0)
writer.close()
with pytest.raises(metrics.SageMakerMetricsWriterException):
writer.log_metric(metric_name='foo', value=1.0)