def init_dir(username: str, password: str, project_name: str, project_id: int, api_key: str, rolling: bool, max_roll: int, ctx=None): if os.path.exists(".tracker/"): click.secho("Error: tracker was already initialized", fg="red") return os.makedirs(".tracker/") os.makedirs(".tracker/logs/") os.makedirs(".tracker/trials/") current_time = str(datetime.datetime.now()) meta = { "created": current_time, "updated": current_time, "files": {}, "current_trial": 0, "models": {} } config = { "project_name": project_name, "project_id": project_id, "api_key": api_key, "rolling": rolling, "max_roll": max_roll } try: if username and password: api = TrackerMLAPI(username, password) if project_name and not project_id: config["project_id"] = api.post_project(project_name)["id"] elif project_id and not project_name: for p in api.get_projects(): if p["id"] == project_id: config["project_name"] = p["name"] if not config["project_name"]: click.secho("Warning: no project with id {} found".format( project_id), fg="yellow") except: click.secho("Warning: problem connecting to tracker.ml API", fg="yellow") finally: fo.set_meta(meta, ctx) fo.set_config(config, ctx)
def test_runs(self): username = self._random_str() password = self._random_str() project_name = self._random_str() model_name = self._random_str() api = TrackerMLAPI(username, password, base_url=self.BASE_URL) api.create_user() project_id = api.post_project(project_name)["id"] model_id = api.post_model(model_name, project_id) self.assertEqual(len(api.get_runs(project_id, model_id)), 0) params = {"p1": 1} api.post_run(project_id, model_id, params) runs = api.get_runs(project_id, model_id) self.assertEqual(len(runs), 1) self.assertEqual(runs[0]["parameters"], params)
def test_signup_and_login(self): username = self._random_str() password = self._random_str() api = TrackerMLAPI(username, password, base_url=self.BASE_URL) api.create_user() api.ensure_token() self.assertTrue(bool(api._token)) self.assertGreater(api._expiration, 0) with self.assertRaises(HTTPError): api.create_user()
def test_models(self): username = self._random_str() password = self._random_str() project_name = self._random_str() model_name = self._random_str() api = TrackerMLAPI(username, password, base_url=self.BASE_URL) api.create_user() project_id = api.post_project(project_name)["id"] self.assertEqual(len(api.get_models(project_id)), 0) model_id = api.post_model(model_name, project_id) models = api.get_models(project_id) self.assertEqual(len(models), 1) self.assertEqual(model_name, models[0]["type"]) self.assertEqual(model_id, models[0]["model_id"]) self.assertEqual(project_id, models[0]["project_id"]) with self.assertRaises(HTTPError): api.post_model(model_name, project_id)
def test_projects(self): username = self._random_str() password = self._random_str() project_name = self._random_str() api = TrackerMLAPI(username, password, base_url=self.BASE_URL) api.create_user() self.assertEqual(len(api.get_projects()), 0) project = api.post_project(project_name) self.assertEqual(project["name"], project_name) projects = api.get_projects() self.assertEqual(len(projects), 1) self.assertEqual(project["name"], projects[0]["name"]) self.assertEqual(project["id"], projects[0]["id"]) self.assertEqual(project["creator_id"], projects[0]["creator_id"]) with self.assertRaises(HTTPError): api.post_project(project_name)
def init_dir(username: str, password: str, project_name: str, project_id: int, api_key: str, rolling: bool, max_roll: int, ctx=None): if os.path.exists(".tracker/"): click.secho("Error: tracker was already initialized", fg="red") return os.makedirs(".tracker/") os.makedirs(".tracker/logs/") os.makedirs(".tracker/trials/") current_time = str(datetime.datetime.now()) meta = { "created": current_time, "updated": current_time, "files": {}, "current_trial": 0, "models": {} } config = { "project_name": project_name, "project_id": project_id, "api_key": api_key, "rolling": rolling, "max_roll": max_roll } # # Kaggle setup # kaggle_dir = "" # if "KAGGLE_CONFIG_DIR" in os.environ: # kaggle_dir = os.environ["KAGGLE_CONFIG_DIR"] # elif "win" in sys.platform(): # kaggle_dir = "C:\\Users\\{}\\.kaggle".format(os.getlogin()) # else: # kaggle_dir = "~/.kaggle" try: if username and password: api = TrackerMLAPI(username, password, api_key=api_key) if project_name and not project_id: config["project_id"] = api.post_project(project_name)["id"] elif project_id and not project_name: for p in api.get_projects(): if p["id"] == project_id: config["project_name"] = p["name"] if not config["project_name"]: click.secho("Warning: no project with id {} found".format( project_id), fg="yellow") except Exception as e: print(e) click.secho("Warning: problem connecting to tracker.ml API", fg="yellow") finally: fo.set_meta(meta, ctx) fo.set_config(config, ctx)
def login(self, username: str, password: str): self.__api = TrackerMLAPI(username, password) self.__api.ensure_token()
class __TMLRun: """ A Run is a single execution of a ml model train """ def __init__(self): self.__trials_dir = fo.get_trials_dir() self.__id = max([int(f) for f in os.listdir(self.__trials_dir)], default=0) + 1 self.__curr_dir = os.path.join(self.__trials_dir, str(self.__id)) self.__meta = collections.OrderedDict() self.__meta["id"] = self.__id self.__api = None self.__model_name = "" atexit.register(self.__save) def login(self, username: str, password: str): self.__api = TrackerMLAPI(username, password) self.__api.ensure_token() def model(self, model_name: str): self.__model_name = model_name def record(self, key: str, value): if key in self.__meta: raise ValueError("{} already recorded".format(key)) if isinstance(value, str) or isinstance(value, int) or isinstance(value, float): self.__meta[key] = value else: raise TypeError("Value must be of type str, int, or float") def mrecord(self, key: str, value): if key not in self.__meta: self.__meta[key] = [] elif not isinstance(self.__meta[key], list): raise TypeError("Key was already used to record a single value") if isinstance(value, str) or isinstance(value, int) or isinstance(value, float): self.__meta[key].append(value) else: raise TypeError("Value must be of type str, int, or float") def __save(self): os.makedirs(self.__curr_dir) for file, md5 in fo.get_meta()["files"].items(): copyfile(file, os.path.join(self.__curr_dir, md5)) with open(os.path.join(self.__curr_dir, "meta.json"), "w+") as fp: json.dump(self.__meta, fp, indent=2) config = fo.get_config() meta = fo.get_meta() meta["current_trial"] = self.__id try: if self.__model_name and self.__api: meta["model_name"] = self.__model_name project_id = config["project_id"] if self.__model_name in meta["models"]: model_id = meta["models"][self.__model_name] else: model_id = self.__api.post_model(self.__model_name, project_id) meta["models"][self.__model_name] = model_id self.__api.post_run(project_id, model_id, meta) except Exception as e: logging.exception("Problem using tracker.ml API") finally: fo.set_meta(meta)