def save_train_start(out, data: TrainingData, global_step: int,
evaluators: List[Evaluator], train_params: TrainParams,
notes: str):
""" Record the training parameters we are about to use into `out` """
if notes is not None:
with open(join(out, "train_from_%d_notes.txt" % global_step),
"w") as f:
f.write(notes)
import socket
hostname = socket.gethostname()
train = dict(train_params=train_params,
data=data,
start_at=global_step,
evaluators=evaluators,
date=datetime.now().strftime("%m%d-%H%M%S"),
host=hostname)
with open(join(out, "train_from_%d.json" % global_step), "w") as f:
f.write(configurable.config_to_json(train, indent=2))
with open(join(out, "train_from_%d.pkl" % global_step), "wb") as f:
pickle.dump(train, f)
def init(out: ModelDir, model: Model, override=False):
""" Save our intial setup into `out` """
for dir in [out.save_dir, out.log_dir]:
if os.path.exists(dir):
if len(os.listdir(dir)) > 0:
if override:
print("Clearing %d files/dirs that already existed in %s" % (len(os.listdir(dir)), dir))
shutil.rmtree(dir)
os.makedirs(dir)
else:
raise ValueError()
else:
os.makedirs(dir)
# JSON config just so we always have a human-readable dump of what we are working with
with open(join(out.dir, "model.json"), "w") as f:
f.write(configurable.config_to_json(model, indent=2))
# Actual model saved via pickle
with open(join(out.dir, "model.pkl"), "wb") as f:
pickle.dump(model, f)
