def test_models(self):
username = self._random_str()
password = self._random_str()
project_name = self._random_str()
model_name = self._random_str()
api = TrackerMLAPI(username, password, base_url=self.BASE_URL)
api.create_user()
project_id = api.post_project(project_name)["id"]
self.assertEqual(len(api.get_models(project_id)), 0)
model_id = api.post_model(model_name, project_id)
models = api.get_models(project_id)
self.assertEqual(len(models), 1)
self.assertEqual(model_name, models[0]["type"])
self.assertEqual(model_id, models[0]["model_id"])
self.assertEqual(project_id, models[0]["project_id"])
with self.assertRaises(HTTPError):
api.post_model(model_name, project_id)
def test_runs(self):
username = self._random_str()
password = self._random_str()
project_name = self._random_str()
model_name = self._random_str()
api = TrackerMLAPI(username, password, base_url=self.BASE_URL)
api.create_user()
project_id = api.post_project(project_name)["id"]
model_id = api.post_model(model_name, project_id)
self.assertEqual(len(api.get_runs(project_id, model_id)), 0)
params = {"p1": 1}
api.post_run(project_id, model_id, params)
runs = api.get_runs(project_id, model_id)
self.assertEqual(len(runs), 1)
self.assertEqual(runs[0]["parameters"], params)
class __TMLRun:
"""
A Run is a single execution of a ml model train
"""
def __init__(self):
self.__trials_dir = fo.get_trials_dir()
self.__id = max([int(f) for f in os.listdir(self.__trials_dir)], default=0) + 1
self.__curr_dir = os.path.join(self.__trials_dir, str(self.__id))
self.__meta = collections.OrderedDict()
self.__meta["id"] = self.__id
self.__api = None
self.__model_name = ""
atexit.register(self.__save)
def login(self, username: str, password: str):
self.__api = TrackerMLAPI(username, password)
self.__api.ensure_token()
def model(self, model_name: str):
self.__model_name = model_name
def record(self, key: str, value):
if key in self.__meta:
raise ValueError("{} already recorded".format(key))
if isinstance(value, str) or isinstance(value, int) or isinstance(value, float):
self.__meta[key] = value
else:
raise TypeError("Value must be of type str, int, or float")
def mrecord(self, key: str, value):
if key not in self.__meta:
self.__meta[key] = []
elif not isinstance(self.__meta[key], list):
raise TypeError("Key was already used to record a single value")
if isinstance(value, str) or isinstance(value, int) or isinstance(value, float):
self.__meta[key].append(value)
else:
raise TypeError("Value must be of type str, int, or float")
def __save(self):
os.makedirs(self.__curr_dir)
for file, md5 in fo.get_meta()["files"].items():
copyfile(file, os.path.join(self.__curr_dir, md5))
with open(os.path.join(self.__curr_dir, "meta.json"), "w+") as fp:
json.dump(self.__meta, fp, indent=2)
config = fo.get_config()
meta = fo.get_meta()
meta["current_trial"] = self.__id
try:
if self.__model_name and self.__api:
meta["model_name"] = self.__model_name
project_id = config["project_id"]
if self.__model_name in meta["models"]:
model_id = meta["models"][self.__model_name]
else:
model_id = self.__api.post_model(self.__model_name, project_id)
meta["models"][self.__model_name] = model_id
self.__api.post_run(project_id, model_id, meta)
except Exception as e:
logging.exception("Problem using tracker.ml API")
finally:
fo.set_meta(meta)