def __save(self): os.makedirs(self.__curr_dir) for file, md5 in fo.get_meta()["files"].items(): copyfile(file, os.path.join(self.__curr_dir, md5)) with open(os.path.join(self.__curr_dir, "meta.json"), "w+") as fp: json.dump(self.__meta, fp, indent=2) config = fo.get_config() meta = fo.get_meta() meta["current_trial"] = self.__id try: if self.__model_name and self.__api: meta["model_name"] = self.__model_name project_id = config["project_id"] if self.__model_name in meta["models"]: model_id = meta["models"][self.__model_name] else: model_id = self.__api.post_model(self.__model_name, project_id) meta["models"][self.__model_name] = model_id self.__api.post_run(project_id, model_id, meta) except Exception as e: logging.exception("Problem using tracker.ml API") finally: fo.set_meta(meta)
def init_dir(username: str, password: str, project_name: str, project_id: int, api_key: str, rolling: bool, max_roll: int, ctx=None): if os.path.exists(".tracker/"): click.secho("Error: tracker was already initialized", fg="red") return os.makedirs(".tracker/") os.makedirs(".tracker/logs/") os.makedirs(".tracker/trials/") current_time = str(datetime.datetime.now()) meta = { "created": current_time, "updated": current_time, "files": {}, "current_trial": 0, "models": {} } config = { "project_name": project_name, "project_id": project_id, "api_key": api_key, "rolling": rolling, "max_roll": max_roll } try: if username and password: api = TrackerMLAPI(username, password) if project_name and not project_id: config["project_id"] = api.post_project(project_name)["id"] elif project_id and not project_name: for p in api.get_projects(): if p["id"] == project_id: config["project_name"] = p["name"] if not config["project_name"]: click.secho("Warning: no project with id {} found".format( project_id), fg="yellow") except: click.secho("Warning: problem connecting to tracker.ml API", fg="yellow") finally: fo.set_meta(meta, ctx) fo.set_config(config, ctx)
def remove_file(path: str, ctx=None): meta = fo.get_meta(ctx) if os.path.isfile(path): if path not in meta["files"]: click.secho("Error: {} is not tracked".format(path), fg="red") return meta["files"].pop(path) else: for (dir_path, _, files) in os.walk(path): for f in files: p = os.path.join(dir_path, f) if p in meta["files"]: meta["files"].pop(p) fo.set_meta(meta, ctx)
def add_file(path: str, ctx=None): meta = fo.get_meta(ctx=ctx) if not os.path.exists(path): click.secho("Error: {} could not be located".format(path), fg="red") return elif os.path.isfile(path): if path not in meta["files"]: meta["files"][path] = __hash(path) click.echo("Added {}".format(path)) else: for (dir_path, _, files) in os.walk(path): for f in files: p = os.path.join(dir_path, f) if p not in meta["files"]: meta["files"][p] = __hash(p) click.echo("Added {}".format(p)) fo.set_meta(meta, ctx)
def deploy_trial(run_id: int, ctx=None): trial_path = os.path.join(fo.get_trials_dir(ctx), str(run_id)) if not os.path.exists(trial_path): raise FileNotFoundError( "Run with ID {} couldn't be located".format(run_id)) meta = fo.get_meta(ctx=ctx) files = dict([(v, k) for k, v in meta["files"].items()]) for src_path in os.listdir(trial_path): if src_path not in files: continue dst_path = files[src_path] if not os.path.exists(dst_path): continue os.remove(dst_path) copyfile(os.path.join(trial_path, src_path), dst_path) meta["current_trial"] = run_id fo.set_meta(meta, ctx)
def init_dir(username: str, password: str, project_name: str, project_id: int, api_key: str, rolling: bool, max_roll: int, ctx=None): if os.path.exists(".tracker/"): click.secho("Error: tracker was already initialized", fg="red") return os.makedirs(".tracker/") os.makedirs(".tracker/logs/") os.makedirs(".tracker/trials/") current_time = str(datetime.datetime.now()) meta = { "created": current_time, "updated": current_time, "files": {}, "current_trial": 0, "models": {} } config = { "project_name": project_name, "project_id": project_id, "api_key": api_key, "rolling": rolling, "max_roll": max_roll } # # Kaggle setup # kaggle_dir = "" # if "KAGGLE_CONFIG_DIR" in os.environ: # kaggle_dir = os.environ["KAGGLE_CONFIG_DIR"] # elif "win" in sys.platform(): # kaggle_dir = "C:\\Users\\{}\\.kaggle".format(os.getlogin()) # else: # kaggle_dir = "~/.kaggle" try: if username and password: api = TrackerMLAPI(username, password, api_key=api_key) if project_name and not project_id: config["project_id"] = api.post_project(project_name)["id"] elif project_id and not project_name: for p in api.get_projects(): if p["id"] == project_id: config["project_name"] = p["name"] if not config["project_name"]: click.secho("Warning: no project with id {} found".format( project_id), fg="yellow") except Exception as e: print(e) click.secho("Warning: problem connecting to tracker.ml API", fg="yellow") finally: fo.set_meta(meta, ctx) fo.set_config(config, ctx)