def export(self, read_time=None):
"""Executes the export flow.
Args:
read_time: A fixed timestamp from which to export data, as float seconds
since epoch (like `time.time()`). Optional; defaults to the current
time.
Yields:
After each experiment is successfully downloaded, the ID of that
experiment, as a string.
"""
if read_time is None:
read_time = time.time()
experiment_metadata_mask = experiment_pb2.ExperimentMask(
create_time=True,
update_time=True,
name=True,
description=True,
)
experiments = list_experiments(self._api,
fieldmask=experiment_metadata_mask,
read_time=read_time)
for experiment in experiments:
experiment_id = experiment.experiment_id
experiment_metadata = {
"name": experiment.name,
"description": experiment.description,
"create_time":
util.format_time_absolute(experiment.create_time),
"update_time":
util.format_time_absolute(experiment.update_time),
}
experiment_dir = _experiment_directory(self._outdir, experiment_id)
os.mkdir(experiment_dir)
metadata_filepath = os.path.join(experiment_dir,
_FILENAME_METADATA)
with _open_excl(metadata_filepath) as outfile:
json.dump(experiment_metadata, outfile, sort_keys=True)
outfile.write("\n")
scalars_filepath = os.path.join(experiment_dir, _FILENAME_SCALARS)
try:
with _open_excl(scalars_filepath) as outfile:
data = self._request_scalar_data(experiment_id, read_time)
for block in data:
json.dump(block, outfile, sort_keys=True)
outfile.write("\n")
outfile.flush()
yield experiment_id
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.CANCELLED:
raise GrpcTimeoutException(experiment_id)
else:
raise
def format_experiment(self, experiment, experiment_url):
data = [
("url", experiment_url),
("name", experiment.name),
("description", experiment.description),
("id", experiment.experiment_id),
("created", util.format_time_absolute(experiment.create_time)),
("updated", util.format_time_absolute(experiment.update_time)),
("runs", experiment.num_runs),
("tags", experiment.num_tags),
("scalars", experiment.num_scalars),
("binary_object_bytes", experiment.total_blob_bytes),
]
return json.dumps(
collections.OrderedDict(data),
indent=self._JSON_INDENT,
)
def _run(self, t=None, tz=None):
timestamp_pb = timestamp_pb2.Timestamp()
util.set_timestamp(timestamp_pb, t)
try:
with mock.patch.dict(os.environ, {"TZ": tz}):
time.tzset()
return util.format_time_absolute(timestamp_pb)
finally:
time.tzset()
