def __save(self):
os.makedirs(self.__curr_dir)
for file, md5 in fo.get_meta()["files"].items():
copyfile(file, os.path.join(self.__curr_dir, md5))
with open(os.path.join(self.__curr_dir, "meta.json"), "w+") as fp:
json.dump(self.__meta, fp, indent=2)
config = fo.get_config()
meta = fo.get_meta()
meta["current_trial"] = self.__id
try:
if self.__model_name and self.__api:
meta["model_name"] = self.__model_name
project_id = config["project_id"]
if self.__model_name in meta["models"]:
model_id = meta["models"][self.__model_name]
else:
model_id = self.__api.post_model(self.__model_name, project_id)
meta["models"][self.__model_name] = model_id
self.__api.post_run(project_id, model_id, meta)
except Exception as e:
logging.exception("Problem using tracker.ml API")
finally:
fo.set_meta(meta)
def init_dir(username: str,
password: str,
project_name: str,
project_id: int,
api_key: str,
rolling: bool,
max_roll: int,
ctx=None):
if os.path.exists(".tracker/"):
click.secho("Error: tracker was already initialized", fg="red")
return
os.makedirs(".tracker/")
os.makedirs(".tracker/logs/")
os.makedirs(".tracker/trials/")
current_time = str(datetime.datetime.now())
meta = {
"created": current_time,
"updated": current_time,
"files": {},
"current_trial": 0,
"models": {}
}
config = {
"project_name": project_name,
"project_id": project_id,
"api_key": api_key,
"rolling": rolling,
"max_roll": max_roll
}
try:
if username and password:
api = TrackerMLAPI(username, password)
if project_name and not project_id:
config["project_id"] = api.post_project(project_name)["id"]
elif project_id and not project_name:
for p in api.get_projects():
if p["id"] == project_id:
config["project_name"] = p["name"]
if not config["project_name"]:
click.secho("Warning: no project with id {} found".format(
project_id),
fg="yellow")
except:
click.secho("Warning: problem connecting to tracker.ml API",
fg="yellow")
finally:
fo.set_meta(meta, ctx)
fo.set_config(config, ctx)
def remove_file(path: str, ctx=None):
meta = fo.get_meta(ctx)
if os.path.isfile(path):
if path not in meta["files"]:
click.secho("Error: {} is not tracked".format(path), fg="red")
return
meta["files"].pop(path)
else:
for (dir_path, _, files) in os.walk(path):
for f in files:
p = os.path.join(dir_path, f)
if p in meta["files"]:
meta["files"].pop(p)
fo.set_meta(meta, ctx)
def add_file(path: str, ctx=None):
meta = fo.get_meta(ctx=ctx)
if not os.path.exists(path):
click.secho("Error: {} could not be located".format(path), fg="red")
return
elif os.path.isfile(path):
if path not in meta["files"]:
meta["files"][path] = __hash(path)
click.echo("Added {}".format(path))
else:
for (dir_path, _, files) in os.walk(path):
for f in files:
p = os.path.join(dir_path, f)
if p not in meta["files"]:
meta["files"][p] = __hash(p)
click.echo("Added {}".format(p))
fo.set_meta(meta, ctx)
def deploy_trial(run_id: int, ctx=None):
trial_path = os.path.join(fo.get_trials_dir(ctx), str(run_id))
if not os.path.exists(trial_path):
raise FileNotFoundError(
"Run with ID {} couldn't be located".format(run_id))
meta = fo.get_meta(ctx=ctx)
files = dict([(v, k) for k, v in meta["files"].items()])
for src_path in os.listdir(trial_path):
if src_path not in files:
continue
dst_path = files[src_path]
if not os.path.exists(dst_path):
continue
os.remove(dst_path)
copyfile(os.path.join(trial_path, src_path), dst_path)
meta["current_trial"] = run_id
fo.set_meta(meta, ctx)
def init_dir(username: str,
password: str,
project_name: str,
project_id: int,
api_key: str,
rolling: bool,
max_roll: int,
ctx=None):
if os.path.exists(".tracker/"):
click.secho("Error: tracker was already initialized", fg="red")
return
os.makedirs(".tracker/")
os.makedirs(".tracker/logs/")
os.makedirs(".tracker/trials/")
current_time = str(datetime.datetime.now())
meta = {
"created": current_time,
"updated": current_time,
"files": {},
"current_trial": 0,
"models": {}
}
config = {
"project_name": project_name,
"project_id": project_id,
"api_key": api_key,
"rolling": rolling,
"max_roll": max_roll
}
# # Kaggle setup
# kaggle_dir = ""
# if "KAGGLE_CONFIG_DIR" in os.environ:
# kaggle_dir = os.environ["KAGGLE_CONFIG_DIR"]
# elif "win" in sys.platform():
# kaggle_dir = "C:\\Users\\{}\\.kaggle".format(os.getlogin())
# else:
# kaggle_dir = "~/.kaggle"
try:
if username and password:
api = TrackerMLAPI(username, password, api_key=api_key)
if project_name and not project_id:
config["project_id"] = api.post_project(project_name)["id"]
elif project_id and not project_name:
for p in api.get_projects():
if p["id"] == project_id:
config["project_name"] = p["name"]
if not config["project_name"]:
click.secho("Warning: no project with id {} found".format(
project_id),
fg="yellow")
except Exception as e:
print(e)
click.secho("Warning: problem connecting to tracker.ml API",
fg="yellow")
finally:
fo.set_meta(meta, ctx)
fo.set_config(config, ctx)