def train_evaluate(self, auto_params):
config = self._combine_params(self._split_config(auto_params), self.fixed_params)
# insert the stuff into their corresponding tables
dataset_hash = make_hash(config['dataset'])
entry_exists = {"dataset_loader": "{}".format(self.fns['dataset'])} in Dataset() and {"dataset_config_hash": "{}".format(dataset_hash)} in Dataset()
if not entry_exists:
Dataset().add_entry(self.fns['dataset'], config['dataset'],
dataset_architect=self.architect,
dataset_comment="Random Search")
model_hash = make_hash(config['model'])
entry_exists = {"configurator": "{}".format(self.fns['model'])} in Model() and {"config_hash": "{}".format(model_hash)} in Model()
if not entry_exists:
Model().add_entry(self.fns['model'], config['model'],
model_architect=self.architect,
model_comment="Random Search")
trainer_hash = make_hash(config['trainer'])
entry_exists = {"training_function": "{}".format(self.fns['trainer'])} in Trainer() and {"training_config_hash": "{}".format(trainer_hash)} in Trainer()
if not entry_exists:
Trainer().add_entry(self.fns['trainer'], config['trainer'],
trainer_architect=self.architect,
trainer_comment="Random Search")
# get the primary key values for all those entries
restriction = ('dataset_loader in ("{}")'.format(self.fns['dataset']), 'dataset_config_hash in ("{}")'.format(dataset_hash),
'configurator in ("{}")'.format(self.fns['model']), 'config_hash in ("{}")'.format(model_hash),
'training_function in ("{}")'.format(self.fns['trainer']), 'training_config_hash in ("{}")'.format(trainer_hash),)
# populate the table for those primary keys
TrainedModel().populate(*restriction)
def add_entry(cls, entry):
entry = {
p_key: entry[p_key]
for p_key in cls._master.heading.primary_key
}
entry["next_collapsed_history"] = make_hash(entry)
cls.insert1(entry, skip_duplicates=True)
def _add_etery(Table, fn, config):
entry_hash = make_hash(config)
entry_exists = {"configurator": "{}".format(fn)} in Table() and {"config_hash": "{}".format(entry_hash)} in Table()
if not entry_exists:
Table().add_entry(fn, config,
model_architect=self.architect,
model_comment="AutoMLing")
return fn, entry_hash
def make(self, key):
"""
Given key specifying configuration for dataloaders, model and trainer,
trains the model and saves the trained model.
"""
# lookup the fabrikant corresponding to the current DJ user
fabrikant_name = Fabrikant.get_current_user()
seed = (Seed & key).fetch1('seed')
# load everything
dataloaders, model, trainer = self.load_model(key,
include_trainer=True,
include_state_dict=False,
seed=seed)
# define callback with pinging
def call_back(**kwargs):
self.connection.ping()
self.call_back(**kwargs)
# model training
score, output, model_state = trainer(model=model,
dataloaders=dataloaders,
seed=seed,
uid=key,
cb=call_back)
with tempfile.TemporaryDirectory() as temp_dir:
filename = make_hash(key) + '.pth.tar'
filepath = os.path.join(temp_dir, filename)
torch.save(model_state, filepath)
key['score'] = score
key['output'] = output
key['fabrikant_name'] = fabrikant_name
comments = []
comments.append(
(self.trainer_table & key).fetch1("trainer_comment"))
comments.append((self.model_table & key).fetch1("model_comment"))
comments.append(
(self.dataset_table & key).fetch1("dataset_comment"))
key['comment'] = self.comment_delimitter.join(comments)
key['current_model_fn'], key['current_model_hash'] = (
Model & key).fetch1('model_fn', 'model_hash')
key['current_dataset_fn'], key['current_dataset_hash'] = (
Dataset & key).fetch1('dataset_fn', 'dataset_hash')
key['current_trainer_fn'], key['current_trainer_hash'] = (
Trainer & key).fetch1('trainer_fn', 'trainer_hash')
self.insert1(key)
key['model_state'] = filepath
self.ModelStorage.insert1(key, ignore_extra_fields=True)
def make(self, key):
"""
Given key specifying configuration for dataloaders, model and trainer,
trains the model and saves the trained model.
"""
fabrikant_name = self.user_table.get_current_user()
seed = (self.seed_table & key).fetch1("seed")
dataloaders, model, trainer = self.load_model(key,
include_trainer=True,
include_state_dict=True,
seed=seed)
def call_back(**kwargs):
self.connection.ping()
self.call_back(**kwargs)
score, output, model_state = trainer(model=model,
dataloaders=dataloaders,
seed=seed,
uid=key,
cb=call_back)
transfer_data = output.pop("transfer_data", None) if isinstance(
output, Mapping) else None
with tempfile.TemporaryDirectory() as temp_dir:
filename = make_hash(key)
key["score"] = score
key["output"] = output
key["fabrikant_name"] = fabrikant_name
comments = []
comments.append(
(self.trainer_table & key).fetch1("trainer_comment"))
comments.append((self.model_table & key).fetch1("model_comment"))
comments.append(
(self.dataset_table & key).fetch1("dataset_comment"))
key["comment"] = self.comment_delimitter.join(comments)
self.insert1(key)
self.CollapsedHistory().add_entry(key)
if key["data_transfer"] and transfer_data:
data_path = os.path.join(temp_dir,
filename + "_transfer_data.npz")
np.savez(data_path, **transfer_data)
key["transfer_data"] = data_path
self.DataStorage.insert1(key, ignore_extra_fields=True)
filename += ".pth.tar"
filepath = os.path.join(temp_dir, filename)
torch.save(model_state, filepath)
key["model_state"] = filepath
self.ModelStorage.insert1(key, ignore_extra_fields=True)
def add_to_table(self, epoch, model, score, state, uid):
with tempfile.TemporaryDirectory() as temp_dir:
key = copy.deepcopy(uid)
for k in self.keys:
if k not in key:
key[k] = ""
key["epoch"] = epoch
key["score"] = score
filename = make_hash(uid) + ".pth.tar"
filepath = os.path.join(temp_dir, filename)
state["net"] = model.state_dict()
torch.save(state, filepath)
key["state"] = filepath
self.checkpoint_table.insert1(key) # this is NOT in transaction and thus immediately completes!
def make(self, key):
"""
Given key specifying configuration for dataloaders, model and trainer,
trains the model and saves the trained model.
"""
# lookup the fabrikant corresponding to the current DJ user
fabrikant_name = self.user_table.get_current_user()
seed = (self.seed_table & key).fetch1("seed")
# load everything
dataloaders, model, trainer = self.load_model(key,
include_trainer=True,
include_state_dict=False,
seed=seed)
# define callback with pinging
def call_back(**kwargs):
self.connection.ping()
self.call_back(**kwargs)
# model training
score, output, model_state = trainer(model=model,
dataloaders=dataloaders,
seed=seed,
uid=key,
cb=call_back)
# save resulting model_state into a temporary file to be attached
with tempfile.TemporaryDirectory() as temp_dir:
filename = make_hash(key) + ".pth.tar"
filepath = os.path.join(temp_dir, filename)
torch.save(model_state, filepath)
key["score"] = score
key["output"] = output
key["fabrikant_name"] = fabrikant_name
comments = []
comments.append(
(self.trainer_table & key).fetch1("trainer_comment"))
comments.append((self.model_table & key).fetch1("model_comment"))
comments.append(
(self.dataset_table & key).fetch1("dataset_comment"))
key["comment"] = self.comment_delimitter.join(comments)
self.insert1(key)
key["model_state"] = filepath
self.ModelStorage.insert1(key, ignore_extra_fields=True)