def _create():
run_id = str(uuid.uuid4())
experiment_id = str(random_int(10, 2000))
user_id = random_str(random_int(10, 25))
status = RunStatus.to_string(random.choice(RunStatus.all_status()))
start_time = random_int(1, 10)
end_time = start_time + random_int(1, 10)
lifecycle_stage = LifecycleStage.ACTIVE
artifact_uri = random_str(random_int(10, 40))
ri = RunInfo(
run_uuid=run_id,
run_id=run_id,
experiment_id=experiment_id,
user_id=user_id,
status=status,
start_time=start_time,
end_time=end_time,
lifecycle_stage=lifecycle_stage,
artifact_uri=artifact_uri,
)
return (
ri,
run_id,
experiment_id,
user_id,
status,
start_time,
end_time,
lifecycle_stage,
artifact_uri,
)
def test_all_status_covered(self):
# ensure that all known status are returned. Test will fail if new status are added to PB
all_statuses = set([RunStatus.RUNNING,
RunStatus.SCHEDULED,
RunStatus.FINISHED,
RunStatus.FAILED,
RunStatus.KILLED,
])
self.assertSequenceEqual(all_statuses, set(RunStatus.all_status()))
def _populate_tables(self,
exp_count=3,
run_count=2,
param_count=5,
metric_count=3,
values_count=10):
print("populate tables")
self.experiments = [
str(random_int(100, int(1e9))) for _ in range(exp_count)
]
self.exp_data = {}
self.run_data = {}
self.experiments.append(TestDynamodbStore.DEFAULT_EXPERIMENT_ID)
for exp in self.experiments:
# create experiment
exp_folder = os.path.join(self.table_prefix, exp)
d = {
"experiment_id": exp,
"name": random_str(),
"artifact_location": exp_folder,
"lifecycle_stage": LifecycleStage.ACTIVE, # Required for tests
}
self.exp_data[exp] = d
self._write_table(DynamodbStore.EXPERIMENT_TABLE, d)
# add runs
self.exp_data[exp]["runs"] = []
for _ in range(run_count):
run_id = uuid.uuid4().hex
self.exp_data[exp]["runs"].append(run_id)
run_folder = os.path.join(exp_folder, run_id)
run_info = {
"run_uuid": run_id,
"run_id": run_id,
"experiment_id": exp,
"user_id": random_str(random_int(10, 25)),
"status": random.choice(RunStatus.all_status()),
"start_time": random_int(1, 10),
"end_time": random_int(20, 30),
"tags": [],
"artifact_uri": "{}/artifacts".format(run_folder),
"lifecycle_stage":
LifecycleStage.ACTIVE, # Required for tests
}
self._write_table("run", run_info)
self.run_data[run_id] = run_info
# params
params = {}
for _ in range(param_count):
param_name = random_str(random_int(4, 12))
param_value = random_str(random_int(10, 15))
self._write_table(
"run_param",
{
"run_id": run_id,
"key": param_name,
"value": param_value
},
)
params[param_name] = param_value
self.run_data[run_id]["params"] = params
# metrics
metrics = {}
for _ in range(metric_count):
metric_name = random_str(random_int(6, 10))
timestamp = int(time.time())
values, values_map = [], []
for i in range(values_count):
metric_value = random_int(i * 100, (i * 1) * 100)
timestamp += random_int(i * 1000, (i + 1) * 1000)
values.append((timestamp, metric_value))
values_map.insert(0, {
"timestamp": timestamp,
"value": metric_value
})
self._write_table(
"run_metric",
{
"run_id": run_id,
"key": metric_name,
"metrics": values_map
},
)
metrics[metric_name] = values
self.run_data[run_id]["metrics"] = metrics
def _create_root(self, root):
self.test_root = os.path.join(root,
"test_file_store_%d" % random_int())
os.mkdir(self.test_root)
self.experiments = [str(random_int(100, int(1e9))) for _ in range(3)]
self.exp_data = {}
self.run_data = {}
# Include default experiment
self.experiments.append(FileStore.DEFAULT_EXPERIMENT_ID)
for exp in self.experiments:
# create experiment
exp_folder = os.path.join(self.test_root, str(exp))
os.makedirs(exp_folder)
d = {
"experiment_id": exp,
"name": random_str(),
"artifact_location": exp_folder
}
self.exp_data[exp] = d
write_yaml(exp_folder, FileStore.META_DATA_FILE_NAME, d)
# add runs
self.exp_data[exp]["runs"] = []
for _ in range(2):
run_id = uuid.uuid4().hex
self.exp_data[exp]["runs"].append(run_id)
run_folder = os.path.join(exp_folder, run_id)
os.makedirs(run_folder)
run_info = {
"run_uuid":
run_id,
"run_id":
run_id,
"experiment_id":
exp,
"user_id":
random_str(random_int(10, 25)),
"status":
random.choice(RunStatus.all_status()),
"start_time":
random_int(1, 10),
"end_time":
random_int(20, 30),
"tags": [],
"artifact_uri":
"%s/%s" % (run_folder, FileStore.ARTIFACTS_FOLDER_NAME),
}
write_yaml(run_folder, FileStore.META_DATA_FILE_NAME, run_info)
self.run_data[run_id] = run_info
# params
params_folder = os.path.join(run_folder,
FileStore.PARAMS_FOLDER_NAME)
os.makedirs(params_folder)
params = {}
for _ in range(5):
param_name = random_str(random_int(4, 12))
param_value = random_str(random_int(10, 15))
param_file = os.path.join(params_folder, param_name)
with open(param_file, 'w') as f:
f.write(param_value)
params[param_name] = param_value
self.run_data[run_id]["params"] = params
# metrics
metrics_folder = os.path.join(run_folder,
FileStore.METRICS_FOLDER_NAME)
os.makedirs(metrics_folder)
metrics = {}
for _ in range(3):
metric_name = random_str(random_int(6, 10))
timestamp = int(time.time())
metric_file = os.path.join(metrics_folder, metric_name)
values = []
for _ in range(10):
metric_value = random_int(100, 2000)
timestamp += random_int(10000, 2000000)
values.append((timestamp, metric_value))
with open(metric_file, 'a') as f:
f.write("%d %d\n" % (timestamp, metric_value))
metrics[metric_name] = values
self.run_data[run_id]["metrics"] = metrics
# artifacts
os.makedirs(
os.path.join(run_folder, FileStore.ARTIFACTS_FOLDER_NAME))
