def _create():
run_id = str(uuid.uuid4())
experiment_id = str(random_int(10, 2000))
user_id = random_str(random_int(10, 25))
status = RunStatus.to_string(random.choice(RunStatus.all_status()))
start_time = random_int(1, 10)
end_time = start_time + random_int(1, 10)
lifecycle_stage = LifecycleStage.ACTIVE
artifact_uri = random_str(random_int(10, 40))
ri = RunInfo(
run_uuid=run_id,
run_id=run_id,
experiment_id=experiment_id,
user_id=user_id,
status=status,
start_time=start_time,
end_time=end_time,
lifecycle_stage=lifecycle_stage,
artifact_uri=artifact_uri,
)
return (
ri,
run_id,
experiment_id,
user_id,
status,
start_time,
end_time,
lifecycle_stage,
artifact_uri,
)
def test_with_latest_model_versions(self):
name = random_str()
mvd_1 = ModelVersion(name, "1", 1000, 2000,
"version 1", "user 1", "Production", "source 1", "run ID 1",
"PENDING_REGISTRATION", "Model version is in production!")
mvd_2 = ModelVersion(name, "4", 1300, 2002,
"version 4", "user 2", "Staging", "source 4", "run ID 12",
"READY", "Model copied over!")
as_dict = {"name": name,
"creation_timestamp": 1,
"last_updated_timestamp": 4000,
"description": random_str(),
"latest_versions": [mvd_1, mvd_2],
"tags": []}
rmd_1 = RegisteredModel.from_dictionary(as_dict)
as_dict["tags"] = {}
self.assertEqual(dict(rmd_1), as_dict)
proto = rmd_1.to_proto()
self.assertEqual(proto.creation_timestamp, 1)
self.assertEqual(proto.last_updated_timestamp, 4000)
self.assertEqual(set([mvd.version for mvd in proto.latest_versions]),
set(["1", "4"]))
self.assertEqual(set([mvd.name for mvd in proto.latest_versions]),
set([name, ]))
self.assertEqual(set([mvd.current_stage for mvd in proto.latest_versions]),
set(["Production", "Staging"]))
self.assertEqual(set([mvd.last_updated_timestamp for mvd in proto.latest_versions]),
set([2000, 2002]))
self.assertEqual(set([mvd.creation_timestamp for mvd in proto.latest_versions]),
set([1300, 1000]))
def test_creation_and_hydration(self):
name = random_str()
description = random_str()
rmd_1 = RegisteredModel(name, 1, 2, description, [])
self._check(rmd_1, name, 1, 2, description, [])
as_dict = {
"name": name,
"creation_timestamp": 1,
"last_updated_timestamp": 2,
"description": description,
"latest_versions": []
}
self.assertEqual(dict(rmd_1), as_dict)
proto = rmd_1.to_proto()
self.assertEqual(proto.name, name)
self.assertEqual(proto.creation_timestamp, 1)
self.assertEqual(proto.last_updated_timestamp, 2)
self.assertEqual(proto.description, description)
rmd_2 = RegisteredModel.from_proto(proto)
self._check(rmd_2, name, 1, 2, description, [])
rmd_3 = RegisteredModel.from_dictionary(as_dict)
self._check(rmd_3, name, 1, 2, description, [])
def _create():
metrics = [Metric(random_str(10),
random_int(0, 1000), int(time.time()) + random_int(-1e4, 1e4))]
params = [Param(random_str(10), random_str(random_int(10, 35))) for _ in range(10)] # noqa
tags = [RunTag(random_str(10), random_str(random_int(10, 35))) for _ in range(10)] # noqa
rd = RunData(metrics=metrics, params=params, tags=tags)
return rd, metrics, params, tags
def _create():
metrics = [
Metric(
key=random_str(10),
value=random_int(0, 1000),
timestamp=int(time.time()) + random_int(-1e4, 1e4),
step=random_int(),
)
]
params = [Param(random_str(10), random_str(random_int(10, 35))) for _ in range(10)]
tags = [RunTag(random_str(10), random_str(random_int(10, 35))) for _ in range(10)]
rd = RunData(metrics=metrics, params=params, tags=tags)
return rd, metrics, params, tags
def _create():
run_uuid = str(uuid.uuid4())
experiment_id = random_int(10, 2000)
name = random_str(random_int(10, 40))
source_type = random_int(1, 4)
source_name = random_str(random_int(100, 300))
entry_point_name = random_str(random_int(100, 300))
user_id = random_str(random_int(10, 25))
status = random_int(1, 5)
start_time = random_int(1, 10)
end_time = start_time + random_int(1, 10)
source_version = random_str(random_int(10, 40))
tags = [
RunTag(key=random_str(random_int(1, 5)),
value=random_str(random_int(1, 5))) for _ in range(2)
]
artifact_uri = random_str(random_int(10, 40))
ri = RunInfo(run_uuid=run_uuid,
experiment_id=experiment_id,
name=name,
source_type=source_type,
source_name=source_name,
entry_point_name=entry_point_name,
user_id=user_id,
status=status,
start_time=start_time,
end_time=end_time,
source_version=source_version,
tags=tags,
artifact_uri=artifact_uri)
return (ri, run_uuid, experiment_id, name, source_type, source_name,
entry_point_name, user_id, status, start_time, end_time,
source_version, tags, artifact_uri)
def test_infer_and_parse_json_input():
size = 20
# input is correctly recognized as list, and parsed as pd df with orient 'records'
data = {
"col_m": [random_int(0, 1000) for _ in range(size)],
"col_z": [random_str(4) for _ in range(size)],
"col_a": [random_int() for _ in range(size)],
}
p1 = pd.DataFrame.from_dict(data)
p2 = pyfunc_scoring_server.infer_and_parse_json_input(
p1.to_json(orient="records"))
assert all(p1 == p2)
# input is correctly recognized as a dict, and parsed as pd df with orient 'split'
data = {
"col_m": [random_int(0, 1000) for _ in range(size)],
"col_z": [random_str(4) for _ in range(size)],
"col_a": [random_int() for _ in range(size)],
}
p1 = pd.DataFrame.from_dict(data)
p2 = pyfunc_scoring_server.infer_and_parse_json_input(
p1.to_json(orient="split"))
assert all(p1 == p2)
# input is correctly recognized as tf serving input
arr = [
[[1, 2, 3], [4, 5, 6], [7, 8, 9]],
[[3, 2, 1], [6, 5, 4], [9, 8, 7]],
]
tfserving_input = {"instances": arr}
result = pyfunc_scoring_server.infer_and_parse_json_input(
json.dumps(tfserving_input))
assert result.shape == (2, 3, 3)
assert (result == np.array(arr)).all()
# input is unrecognized JSON input
with pytest.raises(MlflowException) as ex:
pyfunc_scoring_server.infer_and_parse_json_input(
json.dumps('"just a string"'))
assert (
"Failed to parse input from JSON. Ensure that input is a valid JSON"
" list or dictionary." in str(ex))
# input is not json str
with pytest.raises(MlflowException) as ex:
pyfunc_scoring_server.infer_and_parse_json_input("(not a json string)")
assert (
"Failed to parse input from JSON. Ensure that input is a valid JSON"
" formatted string." in str(ex))
def test_create_experiment(self):
fs = FileStore(self.test_root)
# Error cases
with self.assertRaises(Exception):
fs.create_experiment(None)
with self.assertRaises(Exception):
fs.create_experiment("")
exp_id_ints = (int(exp_id) for exp_id in self.experiments)
next_id = str(max(exp_id_ints) + 1)
name = random_str(25) # since existing experiments are 10 chars long
created_id = fs.create_experiment(name)
# test that newly created experiment matches expected id
self.assertEqual(created_id, next_id)
# get the new experiment (by id) and verify (by name)
exp1 = fs.get_experiment(created_id)
self.assertEqual(exp1.name, name)
self.assertEqual(
exp1.artifact_location,
path_to_local_file_uri(posixpath.join(self.test_root, created_id)))
# get the new experiment (by name) and verify (by id)
exp2 = fs.get_experiment_by_name(name)
self.assertEqual(exp2.experiment_id, created_id)
def _create():
metrics = [
Metric(random_str(10), random_int(),
int(time.time() + random_int(-1e4, 1e4)))
for x in range(100)
] # noqa
params = [
Param(random_str(10), random_str(random_int(10, 35)))
for x in range(10)
] # noqa
rd = RunData()
for p in params:
rd.add_param(p)
for m in metrics:
rd.add_metric(m)
return rd, metrics, params
def test_create_first_experiment(self):
fs = FileStore(self.test_root)
fs.list_experiments = mock.Mock(return_value=[])
fs._create_experiment_with_id = mock.Mock()
fs.create_experiment(random_str(1))
fs._create_experiment_with_id.assert_called_once()
experiment_id = fs._create_experiment_with_id.call_args[0][1]
self.assertEqual(experiment_id, 0)
def test_parse_json_input_split_oriented():
size = 200
data = {"col_m": [random_int(0, 1000) for _ in range(size)],
"col_z": [random_str(4) for _ in range(size)],
"col_a": [random_int() for _ in range(size)]}
p1 = pd.DataFrame.from_dict(data)
p2 = pyfunc_scoring_server.parse_json_input(p1.to_json(orient="split"), orient="split")
assert all(p1 == p2)
def test_creation_and_hydration(self):
key = random_str(random_int(
10, 25)) # random string on size in range [10, 25]
value = random_str(random_int(
55, 75)) # random string on size in range [55, 75]
param = Param(key, value)
self._check(param, key, value)
as_dict = {"key": key, "value": value}
self.assertEqual(dict(param), as_dict)
proto = param.to_proto()
param2 = Param.from_proto(proto)
self._check(param2, key, value)
param3 = Param.from_dictionary(as_dict)
self._check(param3, key, value)
def test_parse_json_input_records_oriented():
size = 20
data = {"col_m": [random_int(0, 1000) for _ in range(size)],
"col_z": [random_str(4) for _ in range(size)],
"col_a": [random_int() for _ in range(size)]}
p1 = pd.DataFrame.from_dict(data)
p2 = pyfunc_scoring_server.parse_json_input(p1.to_json(orient="records"), orient="records")
# "records" orient may shuffle column ordering. Hence comparing each column Series
for col in data.keys():
assert all(p1[col] == p2[col])
def test_with_latest_model_versions(self):
name = random_str()
mvd_1 = ModelVersionDetailed(RegisteredModel(name), 1, 1000, 2000,
"version 1", "user 1", "Production",
"source 1", "run ID 1", "FAILED_DELETION",
"Model version is in production!")
mvd_2 = ModelVersionDetailed(RegisteredModel(name), 4, 1300, 2002,
"version 4", "user 2", "Staging",
"source 4", "run ID 12", "READY",
"Model copied over!")
as_dict = {
"name": name,
"creation_timestamp": 1,
"last_updated_timestamp": 4000,
"description": random_str(),
"latest_versions": [mvd_1, mvd_2]
}
rmd_1 = RegisteredModelDetailed.from_dictionary(as_dict)
self.assertEqual(dict(rmd_1), as_dict)
proto = rmd_1.to_proto()
self.assertEqual(proto.creation_timestamp, 1)
self.assertEqual(proto.last_updated_timestamp, 4000)
self.assertEqual(
set([mvd.model_version.version for mvd in proto.latest_versions]),
set([1, 4]))
self.assertEqual(
set([
mvd.model_version.registered_model.name
for mvd in proto.latest_versions
]), set([
name,
]))
self.assertEqual(
set([mvd.current_stage for mvd in proto.latest_versions]),
set(["Production", "Staging"]))
self.assertEqual(
set([mvd.last_updated_timestamp for mvd in proto.latest_versions]),
set([2000, 2002]))
self.assertEqual(
set([mvd.creation_timestamp for mvd in proto.latest_versions]),
set([1300, 1000]))
def test_creation_and_hydration(self):
name = random_str()
t1, t2 = 100, 150
source = "path/to/source"
run_id = uuid.uuid4().hex
run_link = "http://localhost:5000/path/to/run"
tags = [ModelVersionTag("key", "value"), ModelVersionTag("randomKey", "not a random value")]
mvd = ModelVersion(name, "5", t1, t2, "version five", "user 1", "Production",
source, run_id, "READY", "Model version #5 is ready to use.", tags,
run_link)
self._check(mvd, name, "5", t1, t2, "version five", "user 1",
"Production", source, run_id, "READY",
"Model version #5 is ready to use.",
{tag.key: tag.value for tag in (tags or [])})
expected_dict = {
"name": name,
"version": "5",
"creation_timestamp": t1,
"last_updated_timestamp": t2,
"description": "version five",
"user_id": "user 1",
"current_stage": "Production",
"source": source,
"run_id": run_id,
"run_link": run_link,
"status": "READY",
"status_message": "Model version #5 is ready to use.",
"tags": {tag.key: tag.value for tag in (tags or [])}}
model_version_as_dict = dict(mvd)
self.assertEqual(model_version_as_dict, expected_dict)
proto = mvd.to_proto()
self.assertEqual(proto.name, name)
self.assertEqual(proto.version, "5")
self.assertEqual(proto.status, ModelVersionStatus.from_string("READY"))
self.assertEqual(proto.status_message, "Model version #5 is ready to use.")
self.assertEqual(set([tag.key for tag in proto.tags]),
set(["key", "randomKey"]))
self.assertEqual(set([tag.value for tag in proto.tags]),
set(["value", "not a random value"]))
mvd_2 = ModelVersion.from_proto(proto)
self._check(mvd_2, name, "5", t1, t2, "version five", "user 1",
"Production", source, run_id, "READY",
"Model version #5 is ready to use.",
{tag.key: tag.value for tag in (tags or [])})
expected_dict.update({"registered_model": RegisteredModel(name)})
expected_dict["tags"] = tags
mvd_3 = ModelVersion.from_dictionary(expected_dict)
self._check(mvd_3, name, "5", t1, t2, "version five", "user 1",
"Production", source, run_id, "READY",
"Model version #5 is ready to use.",
{tag.key: tag.value for tag in (tags or [])})
def test_with_tags(self):
name = random_str()
tag1 = RegisteredModelTag("key", "value")
tag2 = RegisteredModelTag("randomKey", "not a random value")
tags = [tag1, tag2]
as_dict = {"name": name,
"creation_timestamp": 1,
"last_updated_timestamp": 4000,
"description": random_str(),
"latest_versions": [],
"tags": tags}
rmd_1 = RegisteredModel.from_dictionary(as_dict)
as_dict["tags"] = {tag.key: tag.value for tag in (tags or [])}
self.assertEqual(dict(rmd_1), as_dict)
proto = rmd_1.to_proto()
self.assertEqual(proto.creation_timestamp, 1)
self.assertEqual(proto.last_updated_timestamp, 4000)
self.assertEqual(set([tag.key for tag in proto.tags]),
set(["key", "randomKey"]))
self.assertEqual(set([tag.value for tag in proto.tags]),
set(["value", "not a random value"]))
def _create():
run_id = str(uuid.uuid4())
experiment_id = str(random_int(10, 2000))
name = random_str(random_int(10, 40))
source_type = random_int(1, 4)
source_name = random_str(random_int(100, 300))
entry_point_name = random_str(random_int(100, 300))
user_id = random_str(random_int(10, 25))
status = random_int(1, 5)
start_time = random_int(1, 10)
end_time = start_time + random_int(1, 10)
source_version = random_str(random_int(10, 40))
lifecycle_stage = LifecycleStage.ACTIVE
artifact_uri = random_str(random_int(10, 40))
ri = RunInfo(run_uuid=run_id,
run_id=run_id,
experiment_id=experiment_id,
name=name,
source_type=source_type,
source_name=source_name,
entry_point_name=entry_point_name,
user_id=user_id,
status=status,
start_time=start_time,
end_time=end_time,
source_version=source_version,
lifecycle_stage=lifecycle_stage,
artifact_uri=artifact_uri)
return (ri, run_id, experiment_id, name, source_type, source_name,
entry_point_name, user_id, status, start_time, end_time,
source_version, lifecycle_stage, artifact_uri)
def test_parse_json_input_split_oriented_to_numpy_array():
size = 200
data = OrderedDict([("col_m", [random_int(0, 1000) for _ in range(size)]),
("col_z", [random_str(4) for _ in range(size)]),
("col_a", [random_int() for _ in range(size)])])
p0 = pd.DataFrame.from_dict(data)
np_array = np.array([[a, b, c] for a, b, c in
zip(data['col_m'], data['col_z'], data['col_a'])],
dtype=object)
p1 = pd.DataFrame(np_array).infer_objects()
p2 = pyfunc_scoring_server.parse_split_oriented_json_input_to_numpy(
p0.to_json(orient="split"))
np.testing.assert_array_equal(p1, p2)
def _create():
metrics = [
Metric(random_str(10), random_int(0, 1000),
int(time.time() + random_int(-1e4, 1e4)))
for _ in range(100)
]
params = [
Param(random_str(10), random_str(random_int(10, 35)))
for _ in range(10)
] # noqa
tags = [
RunTag(random_str(10), random_str(random_int(10, 35)))
for _ in range(10)
] # noqa
rd = RunData()
for p in params:
rd._add_param(p)
for m in metrics:
rd._add_metric(m)
for t in tags:
rd._add_tag(t)
return rd, metrics, params, tags
def test_get_experiment_by_name(self):
fs = FileStore(self.test_root)
for exp_id in self.experiments:
name = self.exp_data[exp_id]["name"]
exp = fs.get_experiment_by_name(name)
self.assertEqual(exp.experiment_id, exp_id)
self.assertEqual(exp.name, self.exp_data[exp_id]["name"])
self.assertEqual(exp.artifact_location, self.exp_data[exp_id]["artifact_location"])
# test that fake experiments dont exist.
# look up experiments with names of length 15 since created ones are of length 10
for exp_names in set(random_str(15) for x in range(20)):
exp = fs.get_experiment_by_name(exp_names)
self.assertIsNone(exp)
def test_create_registered_model(self):
name = random_str() + "abCD"
rm1 = self._rm_maker(name)
self.assertEqual(rm1.name, name)
# error on duplicate
with self.assertRaises(MlflowException) as exception_context:
self._rm_maker(name)
assert exception_context.exception.error_code == ErrorCode.Name(RESOURCE_ALREADY_EXISTS)
# slightly different name is ok
for name2 in [name + "extra", name.lower(), name.upper(), name + name]:
rm2 = self._rm_maker(name2)
self.assertEqual(rm2.name, name2)
def test_list_experiments_paginated(self):
fs = FileStore(self.test_root)
for _ in range(10):
fs.create_experiment(random_str(12))
exps1 = fs.list_experiments(max_results=4, page_token=None)
self.assertEqual(len(exps1), 4)
self.assertIsNotNone(exps1.token)
exps2 = fs.list_experiments(max_results=4, page_token=None)
self.assertEqual(len(exps2), 4)
self.assertIsNotNone(exps2.token)
self.assertNotEqual(exps1, exps2)
exps3 = fs.list_experiments(max_results=500, page_token=exps2.token)
self.assertLessEqual(len(exps3), 500)
if len(exps3) < 500:
self.assertIsNone(exps3.token)
def test_create_experiment(self):
fs = FileStore(self.test_root)
next_id = max(self.experiments) + 1
name = random_str(25) # since existing experiments are 10 chars long
created_id = fs.create_experiment(name)
# test that newly created experiment matches expected id
self.assertEqual(created_id, next_id)
# get the new experiment (by id) and verify (by name)
exp1 = fs.get_experiment(created_id)
self.assertEqual(exp1.name, name)
# get the new experiment (by name) and verify (by id)
exp2 = fs.get_experiment_by_name(name)
self.assertEqual(exp2.experiment_id, created_id)
def test_creation_and_hydration(self):
name = random_str()
registered_model = RegisteredModel(name)
self._check(registered_model, name)
as_dict = {"name": name}
self.assertEqual(dict(registered_model), as_dict)
proto = registered_model.to_proto()
self.assertEqual(proto.name, name)
registered_model2 = RegisteredModel.from_proto(proto)
self._check(registered_model2, name)
registered_model3 = RegisteredModel.from_dictionary(as_dict)
self._check(registered_model3, name)
def test_creation_and_hydration(self):
name = random_str()
rm = RegisteredModel(name)
t1, t2 = 100, 150
source = "path/to/source"
run_id = uuid.uuid4().hex
mvd = ModelVersionDetailed(rm, 5, t1, t2, "version five", "user 1",
"Production", source, run_id, "READY",
"Model version #5 is ready to use.")
self._check(mvd, name, 5, t1, t2, "version five", "user 1",
"Production", source, run_id, "READY",
"Model version #5 is ready to use.")
expected_dict = {
"version": 5,
"creation_timestamp": t1,
"last_updated_timestamp": t2,
"description": "version five",
"user_id": "user 1",
"current_stage": "Production",
"source": source,
"run_id": run_id,
"status": "READY",
"status_message": "Model version #5 is ready to use."
}
model_version_as_dict = dict(mvd)
self.assertIsInstance(model_version_as_dict["registered_model"],
RegisteredModel)
self.assertEqual(model_version_as_dict["registered_model"].name, name)
model_version_as_dict.pop("registered_model")
self.assertEqual(model_version_as_dict, expected_dict)
proto = mvd.to_proto()
self.assertEqual(proto.model_version.registered_model.name, name)
self.assertEqual(proto.model_version.version, 5)
self.assertEqual(proto.status, ModelVersionStatus.from_string("READY"))
self.assertEqual(proto.status_message,
"Model version #5 is ready to use.")
mvd_2 = ModelVersionDetailed.from_proto(proto)
self._check(mvd_2, name, 5, t1, t2, "version five", "user 1",
"Production", source, run_id, "READY",
"Model version #5 is ready to use.")
expected_dict.update({"registered_model": RegisteredModel(name)})
mvd_3 = ModelVersionDetailed.from_dictionary(expected_dict)
self._check(mvd_3, name, 5, t1, t2, "version five", "user 1",
"Production", source, run_id, "READY",
"Model version #5 is ready to use.")
def test_creation_and_hydration(self):
path = random_str(random_int(10, 50))
is_dir = random_int(10, 2500) % 2 == 0
size_in_bytes = random_int(1, 10000)
fi1 = FileInfo(path, is_dir, size_in_bytes)
self._check(fi1, path, is_dir, size_in_bytes)
as_dict = {"path": path, "is_dir": is_dir, "file_size": size_in_bytes}
self.assertEqual(dict(fi1), as_dict)
proto = fi1.to_proto()
fi2 = FileInfo.from_proto(proto)
self._check(fi2, path, is_dir, size_in_bytes)
fi3 = FileInfo.from_dictionary(as_dict)
self._check(fi3, path, is_dir, size_in_bytes)
def test_creation_and_hydration(self):
key = random_str()
value = 10000
ts = int(time.time())
metric = Metric(key, value, ts)
self._check(metric, key, value, ts)
as_dict = {"key": key, "value": value, "timestamp": ts}
self.assertEqual(dict(metric), as_dict)
proto = metric.to_proto()
metric2 = metric.from_proto(proto)
self._check(metric2, key, value, ts)
metric3 = Metric.from_dictionary(as_dict)
self._check(metric3, key, value, ts)
def test_creation_and_hydration():
key = random_str()
value = 10000
ts = int(1000 * time.time())
step = random_int()
metric = Metric(key, value, ts, step)
_check(metric, key, value, ts, step)
as_dict = {"key": key, "value": value, "timestamp": ts, "step": step}
assert dict(metric) == as_dict
proto = metric.to_proto()
metric2 = metric.from_proto(proto)
_check(metric2, key, value, ts, step)
metric3 = Metric.from_dictionary(as_dict)
_check(metric3, key, value, ts, step)
def test_creation_and_hydration(self):
name = random_str()
rm = RegisteredModel(name)
model_version = ModelVersion(rm, 100)
self._check(model_version, name, 100)
expected_dict = {"version": 100}
model_version_as_dict = dict(model_version)
self.assertEqual(model_version_as_dict["version"], 100)
self.assertIsInstance(model_version_as_dict["registered_model"],
RegisteredModel)
self.assertEqual(model_version_as_dict["registered_model"].name, name)
model_version_as_dict.pop("registered_model")
self.assertEqual(model_version_as_dict, expected_dict)
proto = model_version.to_proto()
self.assertEqual(proto.registered_model.name, name)
self.assertEqual(proto.version, 100)
model_version_2 = ModelVersion.from_proto(proto)
self._check(model_version_2, name, 100)
expected_dict.update({"registered_model": RegisteredModel(name)})
model_version_3 = ModelVersion.from_dictionary(expected_dict)
self._check(model_version_3, name, 100)
def _create_root(self, root):
self.test_root = os.path.join(root, "test_file_store_%d" % random_int())
os.mkdir(self.test_root)
self.experiments = [random_int(100, int(1e9)) for _ in range(3)]
self.exp_data = {}
self.run_data = {}
# Include default experiment
self.experiments.append(Experiment.DEFAULT_EXPERIMENT_ID)
for exp in self.experiments:
# create experiment
exp_folder = os.path.join(self.test_root, str(exp))
os.makedirs(exp_folder)
d = {"experiment_id": exp, "name": random_str(), "artifact_location": exp_folder}
self.exp_data[exp] = d
write_yaml(exp_folder, FileStore.META_DATA_FILE_NAME, d)
# add runs
self.exp_data[exp]["runs"] = []
for _ in range(2):
run_uuid = uuid.uuid4().hex
self.exp_data[exp]["runs"].append(run_uuid)
run_folder = os.path.join(exp_folder, run_uuid)
os.makedirs(run_folder)
run_info = {"run_uuid": run_uuid,
"experiment_id": exp,
"name": random_str(random_int(10, 40)),
"source_type": random_int(1, 4),
"source_name": random_str(random_int(100, 300)),
"entry_point_name": random_str(random_int(100, 300)),
"user_id": random_str(random_int(10, 25)),
"status": random_int(1, 5),
"start_time": random_int(1, 10),
"end_time": random_int(20, 30),
"source_version": random_str(random_int(10, 30)),
"tags": [],
"artifact_uri": "%s/%s" % (run_folder, FileStore.ARTIFACTS_FOLDER_NAME),
}
write_yaml(run_folder, FileStore.META_DATA_FILE_NAME, run_info)
self.run_data[run_uuid] = run_info
# params
params_folder = os.path.join(run_folder, FileStore.PARAMS_FOLDER_NAME)
os.makedirs(params_folder)
params = {}
for _ in range(5):
param_name = random_str(random_int(4, 12))
param_value = random_str(random_int(10, 15))
param_file = os.path.join(params_folder, param_name)
with open(param_file, 'w') as f:
f.write(param_value)
params[param_name] = param_value
self.run_data[run_uuid]["params"] = params
# metrics
metrics_folder = os.path.join(run_folder, FileStore.METRICS_FOLDER_NAME)
os.makedirs(metrics_folder)
metrics = {}
for _ in range(3):
metric_name = random_str(random_int(6, 10))
timestamp = int(time.time())
metric_file = os.path.join(metrics_folder, metric_name)
values = []
for _ in range(10):
metric_value = random_int(100, 2000)
timestamp += random_int(10000, 2000000)
values.append((timestamp, metric_value))
with open(metric_file, 'a') as f:
f.write("%d %d\n" % (timestamp, metric_value))
metrics[metric_name] = values
self.run_data[run_uuid]["metrics"] = metrics
# artifacts
os.makedirs(os.path.join(run_folder, FileStore.ARTIFACTS_FOLDER_NAME))