def init_dir(username: str,
password: str,
project_name: str,
project_id: int,
api_key: str,
rolling: bool,
max_roll: int,
ctx=None):
if os.path.exists(".tracker/"):
click.secho("Error: tracker was already initialized", fg="red")
return
os.makedirs(".tracker/")
os.makedirs(".tracker/logs/")
os.makedirs(".tracker/trials/")
current_time = str(datetime.datetime.now())
meta = {
"created": current_time,
"updated": current_time,
"files": {},
"current_trial": 0,
"models": {}
}
config = {
"project_name": project_name,
"project_id": project_id,
"api_key": api_key,
"rolling": rolling,
"max_roll": max_roll
}
try:
if username and password:
api = TrackerMLAPI(username, password)
if project_name and not project_id:
config["project_id"] = api.post_project(project_name)["id"]
elif project_id and not project_name:
for p in api.get_projects():
if p["id"] == project_id:
config["project_name"] = p["name"]
if not config["project_name"]:
click.secho("Warning: no project with id {} found".format(
project_id),
fg="yellow")
except:
click.secho("Warning: problem connecting to tracker.ml API",
fg="yellow")
finally:
fo.set_meta(meta, ctx)
fo.set_config(config, ctx)
def test_runs(self):
username = self._random_str()
password = self._random_str()
project_name = self._random_str()
model_name = self._random_str()
api = TrackerMLAPI(username, password, base_url=self.BASE_URL)
api.create_user()
project_id = api.post_project(project_name)["id"]
model_id = api.post_model(model_name, project_id)
self.assertEqual(len(api.get_runs(project_id, model_id)), 0)
params = {"p1": 1}
api.post_run(project_id, model_id, params)
runs = api.get_runs(project_id, model_id)
self.assertEqual(len(runs), 1)
self.assertEqual(runs[0]["parameters"], params)
def test_signup_and_login(self):
username = self._random_str()
password = self._random_str()
api = TrackerMLAPI(username, password, base_url=self.BASE_URL)
api.create_user()
api.ensure_token()
self.assertTrue(bool(api._token))
self.assertGreater(api._expiration, 0)
with self.assertRaises(HTTPError):
api.create_user()
def test_models(self):
username = self._random_str()
password = self._random_str()
project_name = self._random_str()
model_name = self._random_str()
api = TrackerMLAPI(username, password, base_url=self.BASE_URL)
api.create_user()
project_id = api.post_project(project_name)["id"]
self.assertEqual(len(api.get_models(project_id)), 0)
model_id = api.post_model(model_name, project_id)
models = api.get_models(project_id)
self.assertEqual(len(models), 1)
self.assertEqual(model_name, models[0]["type"])
self.assertEqual(model_id, models[0]["model_id"])
self.assertEqual(project_id, models[0]["project_id"])
with self.assertRaises(HTTPError):
api.post_model(model_name, project_id)
def test_projects(self):
username = self._random_str()
password = self._random_str()
project_name = self._random_str()
api = TrackerMLAPI(username, password, base_url=self.BASE_URL)
api.create_user()
self.assertEqual(len(api.get_projects()), 0)
project = api.post_project(project_name)
self.assertEqual(project["name"], project_name)
projects = api.get_projects()
self.assertEqual(len(projects), 1)
self.assertEqual(project["name"], projects[0]["name"])
self.assertEqual(project["id"], projects[0]["id"])
self.assertEqual(project["creator_id"], projects[0]["creator_id"])
with self.assertRaises(HTTPError):
api.post_project(project_name)
def init_dir(username: str,
password: str,
project_name: str,
project_id: int,
api_key: str,
rolling: bool,
max_roll: int,
ctx=None):
if os.path.exists(".tracker/"):
click.secho("Error: tracker was already initialized", fg="red")
return
os.makedirs(".tracker/")
os.makedirs(".tracker/logs/")
os.makedirs(".tracker/trials/")
current_time = str(datetime.datetime.now())
meta = {
"created": current_time,
"updated": current_time,
"files": {},
"current_trial": 0,
"models": {}
}
config = {
"project_name": project_name,
"project_id": project_id,
"api_key": api_key,
"rolling": rolling,
"max_roll": max_roll
}
# # Kaggle setup
# kaggle_dir = ""
# if "KAGGLE_CONFIG_DIR" in os.environ:
# kaggle_dir = os.environ["KAGGLE_CONFIG_DIR"]
# elif "win" in sys.platform():
# kaggle_dir = "C:\\Users\\{}\\.kaggle".format(os.getlogin())
# else:
# kaggle_dir = "~/.kaggle"
try:
if username and password:
api = TrackerMLAPI(username, password, api_key=api_key)
if project_name and not project_id:
config["project_id"] = api.post_project(project_name)["id"]
elif project_id and not project_name:
for p in api.get_projects():
if p["id"] == project_id:
config["project_name"] = p["name"]
if not config["project_name"]:
click.secho("Warning: no project with id {} found".format(
project_id),
fg="yellow")
except Exception as e:
print(e)
click.secho("Warning: problem connecting to tracker.ml API",
fg="yellow")
finally:
fo.set_meta(meta, ctx)
fo.set_config(config, ctx)
def login(self, username: str, password: str):
self.__api = TrackerMLAPI(username, password)
self.__api.ensure_token()
class __TMLRun:
"""
A Run is a single execution of a ml model train
"""
def __init__(self):
self.__trials_dir = fo.get_trials_dir()
self.__id = max([int(f) for f in os.listdir(self.__trials_dir)], default=0) + 1
self.__curr_dir = os.path.join(self.__trials_dir, str(self.__id))
self.__meta = collections.OrderedDict()
self.__meta["id"] = self.__id
self.__api = None
self.__model_name = ""
atexit.register(self.__save)
def login(self, username: str, password: str):
self.__api = TrackerMLAPI(username, password)
self.__api.ensure_token()
def model(self, model_name: str):
self.__model_name = model_name
def record(self, key: str, value):
if key in self.__meta:
raise ValueError("{} already recorded".format(key))
if isinstance(value, str) or isinstance(value, int) or isinstance(value, float):
self.__meta[key] = value
else:
raise TypeError("Value must be of type str, int, or float")
def mrecord(self, key: str, value):
if key not in self.__meta:
self.__meta[key] = []
elif not isinstance(self.__meta[key], list):
raise TypeError("Key was already used to record a single value")
if isinstance(value, str) or isinstance(value, int) or isinstance(value, float):
self.__meta[key].append(value)
else:
raise TypeError("Value must be of type str, int, or float")
def __save(self):
os.makedirs(self.__curr_dir)
for file, md5 in fo.get_meta()["files"].items():
copyfile(file, os.path.join(self.__curr_dir, md5))
with open(os.path.join(self.__curr_dir, "meta.json"), "w+") as fp:
json.dump(self.__meta, fp, indent=2)
config = fo.get_config()
meta = fo.get_meta()
meta["current_trial"] = self.__id
try:
if self.__model_name and self.__api:
meta["model_name"] = self.__model_name
project_id = config["project_id"]
if self.__model_name in meta["models"]:
model_id = meta["models"][self.__model_name]
else:
model_id = self.__api.post_model(self.__model_name, project_id)
meta["models"][self.__model_name] = model_id
self.__api.post_run(project_id, model_id, meta)
except Exception as e:
logging.exception("Problem using tracker.ml API")
finally:
fo.set_meta(meta)