def _create(): run_id = str(uuid.uuid4()) experiment_id = str(random_int(10, 2000)) user_id = random_str(random_int(10, 25)) status = RunStatus.to_string(random.choice(RunStatus.all_status())) start_time = random_int(1, 10) end_time = start_time + random_int(1, 10) lifecycle_stage = LifecycleStage.ACTIVE artifact_uri = random_str(random_int(10, 40)) ri = RunInfo( run_uuid=run_id, run_id=run_id, experiment_id=experiment_id, user_id=user_id, status=status, start_time=start_time, end_time=end_time, lifecycle_stage=lifecycle_stage, artifact_uri=artifact_uri, ) return ( ri, run_id, experiment_id, user_id, status, start_time, end_time, lifecycle_stage, artifact_uri, )
def test_all_status_covered(self): # ensure that all known status are returned. Test will fail if new status are added to PB all_statuses = set([RunStatus.RUNNING, RunStatus.SCHEDULED, RunStatus.FINISHED, RunStatus.FAILED, RunStatus.KILLED, ]) self.assertSequenceEqual(all_statuses, set(RunStatus.all_status()))
def _populate_tables(self, exp_count=3, run_count=2, param_count=5, metric_count=3, values_count=10): print("populate tables") self.experiments = [ str(random_int(100, int(1e9))) for _ in range(exp_count) ] self.exp_data = {} self.run_data = {} self.experiments.append(TestDynamodbStore.DEFAULT_EXPERIMENT_ID) for exp in self.experiments: # create experiment exp_folder = os.path.join(self.table_prefix, exp) d = { "experiment_id": exp, "name": random_str(), "artifact_location": exp_folder, "lifecycle_stage": LifecycleStage.ACTIVE, # Required for tests } self.exp_data[exp] = d self._write_table(DynamodbStore.EXPERIMENT_TABLE, d) # add runs self.exp_data[exp]["runs"] = [] for _ in range(run_count): run_id = uuid.uuid4().hex self.exp_data[exp]["runs"].append(run_id) run_folder = os.path.join(exp_folder, run_id) run_info = { "run_uuid": run_id, "run_id": run_id, "experiment_id": exp, "user_id": random_str(random_int(10, 25)), "status": random.choice(RunStatus.all_status()), "start_time": random_int(1, 10), "end_time": random_int(20, 30), "tags": [], "artifact_uri": "{}/artifacts".format(run_folder), "lifecycle_stage": LifecycleStage.ACTIVE, # Required for tests } self._write_table("run", run_info) self.run_data[run_id] = run_info # params params = {} for _ in range(param_count): param_name = random_str(random_int(4, 12)) param_value = random_str(random_int(10, 15)) self._write_table( "run_param", { "run_id": run_id, "key": param_name, "value": param_value }, ) params[param_name] = param_value self.run_data[run_id]["params"] = params # metrics metrics = {} for _ in range(metric_count): metric_name = random_str(random_int(6, 10)) timestamp = int(time.time()) values, values_map = [], [] for i in range(values_count): metric_value = random_int(i * 100, (i * 1) * 100) timestamp += random_int(i * 1000, (i + 1) * 1000) values.append((timestamp, metric_value)) values_map.insert(0, { "timestamp": timestamp, "value": metric_value }) self._write_table( "run_metric", { "run_id": run_id, "key": metric_name, "metrics": values_map }, ) metrics[metric_name] = values self.run_data[run_id]["metrics"] = metrics
def _create_root(self, root): self.test_root = os.path.join(root, "test_file_store_%d" % random_int()) os.mkdir(self.test_root) self.experiments = [str(random_int(100, int(1e9))) for _ in range(3)] self.exp_data = {} self.run_data = {} # Include default experiment self.experiments.append(FileStore.DEFAULT_EXPERIMENT_ID) for exp in self.experiments: # create experiment exp_folder = os.path.join(self.test_root, str(exp)) os.makedirs(exp_folder) d = { "experiment_id": exp, "name": random_str(), "artifact_location": exp_folder } self.exp_data[exp] = d write_yaml(exp_folder, FileStore.META_DATA_FILE_NAME, d) # add runs self.exp_data[exp]["runs"] = [] for _ in range(2): run_id = uuid.uuid4().hex self.exp_data[exp]["runs"].append(run_id) run_folder = os.path.join(exp_folder, run_id) os.makedirs(run_folder) run_info = { "run_uuid": run_id, "run_id": run_id, "experiment_id": exp, "user_id": random_str(random_int(10, 25)), "status": random.choice(RunStatus.all_status()), "start_time": random_int(1, 10), "end_time": random_int(20, 30), "tags": [], "artifact_uri": "%s/%s" % (run_folder, FileStore.ARTIFACTS_FOLDER_NAME), } write_yaml(run_folder, FileStore.META_DATA_FILE_NAME, run_info) self.run_data[run_id] = run_info # params params_folder = os.path.join(run_folder, FileStore.PARAMS_FOLDER_NAME) os.makedirs(params_folder) params = {} for _ in range(5): param_name = random_str(random_int(4, 12)) param_value = random_str(random_int(10, 15)) param_file = os.path.join(params_folder, param_name) with open(param_file, 'w') as f: f.write(param_value) params[param_name] = param_value self.run_data[run_id]["params"] = params # metrics metrics_folder = os.path.join(run_folder, FileStore.METRICS_FOLDER_NAME) os.makedirs(metrics_folder) metrics = {} for _ in range(3): metric_name = random_str(random_int(6, 10)) timestamp = int(time.time()) metric_file = os.path.join(metrics_folder, metric_name) values = [] for _ in range(10): metric_value = random_int(100, 2000) timestamp += random_int(10000, 2000000) values.append((timestamp, metric_value)) with open(metric_file, 'a') as f: f.write("%d %d\n" % (timestamp, metric_value)) metrics[metric_name] = values self.run_data[run_id]["metrics"] = metrics # artifacts os.makedirs( os.path.join(run_folder, FileStore.ARTIFACTS_FOLDER_NAME))