def run_ht(
ht_config_file,
ht_config_file_additional,
ht_save_path,
ht_start_id=None,
ht_end_id=None,
find_best=False,
**kwargs
):
ht_config = json_reader(ht_config_file)
if ht_config_file_additional is not None:
ht_config_additional = json_reader(ht_config_file_additional)
ht_config = dict_update(
dict_base=ht_config,
dict_new=ht_config_additional
)
model_spec_config_base = json_reader(kwargs["model_kwargs"]["config_file"])
ht = HyperparameterTuner(
save_path=ht_save_path
)
ht_config["model_spec"] = dict_conservative_update(
dict_base=model_spec_config_base,
dict_new=ht_config["model_spec"]
) # update model spec for different models to avoid redundant hps
print(ht_config) # verbose
ht.initialize(hps=ht_config)
# remove "config_file" for safety, add "model_spec" to be updated by single hp_config #
del kwargs["model_kwargs"]["config_file"]
kwargs["model_kwargs"]["model_spec"] = model_spec_config_base
for test_id, hp_config in ht.generate(
start_id=ht_start_id,
end_id=ht_end_id
):
kwargs_single = copy.deepcopy(kwargs)
kwargs_single["model_kwargs"] = dict_conservative_update(
dict_base=kwargs["model_kwargs"],
dict_new=hp_config
)
print("run %d with config: \n%s" % (test_id, str(kwargs_single)))
result_single = run_single(
**kwargs_single
)
perf = result2perf(result=result_single)
ht.read_perf(id_=test_id, perf=perf)
if find_best:
print(ht.find_best())
def test(config_file,
meta_data_file,
id_map,
dataToken,
batch_data_dir,
max_doc_length=30,
size_context=2,
model_name=None,
restore_path=None):
np.random.seed(RANDOM_SEED_NP)
data = DataDUELoader(meta_data_file=meta_data_file,
batch_data_dir=batch_data_dir,
id_map=id_map,
dataToken=dataToken,
max_doc_length=max_doc_length,
size_context=size_context)
model_spec = json_reader(config_file)
model = Caden(data_spec=data.data_spec,
model_spec=model_spec,
model_name=model_name)
model.initialization()
if restore_path is not None:
model.restore(restore_path)
perf = performance(model, data)
print("ckpt_path: %s" % restore_path)
print("performance: %s" % str(perf))
def read_and_dump(filename, filename_target):
if not os.path.exists(filename):
warnings.warn("'%s' not exists" % filename)
return None
config = json_reader(filename)
model_spec = config["model_spec"]
with open(filename_target, "w") as f:
json_dumper(model_spec, f)
def test(config_file,
meta_data_file,
id_map,
dataToken,
batch_data_dir,
max_doc_length=30,
model_name=None,
restore_path=None,
no_doc_index=False):
np.random.seed(RANDOM_SEED_NP)
data = DataDUELoader(meta_data_file=meta_data_file,
batch_data_dir=batch_data_dir,
id_map=id_map,
dataToken=dataToken,
max_doc_length=max_doc_length,
no_doc_index=no_doc_index)
model_spec = json_reader(config_file)
model = FM(feature_shape=(0 if no_doc_index else data.D) + data.U +
data.V + 1,
feature_dim=(0 if no_doc_index else 1) + 1 + max_doc_length,
label_dim=data.E,
model_spec=model_spec,
model_name=model_name)
model.initialization()
def performance(model_local, data_local):
preds = model_local.predict(data_generator=data_local)
labels = []
for data_batched in data_local.generate(
batch_size=model_spec["batch_size"], random_shuffle=False):
labels.append(data_batched["label"])
labels = np.concatenate(labels, axis=0)
# one-hot to index #
trues = np.argmax(labels, axis=-1)
perf = evaluate(preds=preds, trues=trues)
return perf
if restore_path is not None:
if not isinstance(restore_path, list):
restore_paths = [restore_path]
else:
restore_paths = restore_path
for restore_path in restore_paths:
model.restore(restore_path)
perf = performance(model_local=model, data_local=data)
print("ckpt_path: %s" % restore_path)
print("performance: %s" % str(perf))
else:
perf = performance(model_local=model, data_local=data)
print("random initialization")
print("performance: %s" % str(perf))
def ht_find_best(
ht_config_file,
ht_config_file_additional,
ht_save_path,
ht_perf_files,
model_spec_config_file,
):
ht_config = json_reader(ht_config_file)
if ht_config_file_additional is not None:
ht_config_additional = json_reader(ht_config_file_additional)
ht_config = dict_update(dict_base=ht_config,
dict_new=ht_config_additional)
model_spec_config_base = json_reader(model_spec_config_file)
ht = HyperparameterTuner(save_path=ht_save_path)
ht_config["model_spec"] = dict_conservative_update(
dict_base=model_spec_config_base, dict_new=ht_config["model_spec"]
) # update model spec for different models to avoid redundant hps
print(ht_config) # verbose
ht.initialize(hps=ht_config)
for ht_perf_file in ht_perf_files:
ht.restore(perfs_file=ht_perf_file)
print(ht.find_best(max_best=True))
def train(config_file,
meta_data_file,
id_map,
dataToken,
batch_data_dir_train,
batch_data_dir_valid=None,
max_doc_length=30,
model_name=None,
restore_path=None,
no_doc_index=False):
np.random.seed(RANDOM_SEED_NP)
data_train = DataDUELoader(meta_data_file=meta_data_file,
batch_data_dir=batch_data_dir_train,
id_map=id_map,
dataToken=dataToken,
max_doc_length=max_doc_length,
no_doc_index=no_doc_index)
if batch_data_dir_valid is not None:
data_valid = DataDUELoader(meta_data_file=meta_data_file,
batch_data_dir=batch_data_dir_valid,
id_map=id_map,
dataToken=dataToken,
max_doc_length=max_doc_length,
no_doc_index=no_doc_index)
else:
data_valid = None
model_spec = json_reader(config_file)
model = FM(feature_shape=(0 if no_doc_index else data_train.D) +
data_train.U + data_train.V + 1,
feature_dim=(0 if no_doc_index else 1) + 1 + max_doc_length,
label_dim=data_train.E,
model_spec=model_spec,
model_name=model_name)
model.initialization()
if restore_path is not None:
model.restore(restore_path)
# train #
results = model.train(data_generator=data_train,
data_generator_valid=data_valid)
print("train_results: %s" % str(results))
best_epoch = read(directory="../summary/" + model.model_name,
main_indicator="epoch_losses_valid_00")[0]
print("best_epoch by validation loss: %d" % best_epoch)
def train(config_file,
meta_data_file,
id_map,
dataToken,
batch_data_dir_train,
batch_data_dir_valid=None,
max_doc_length=30,
size_context=2,
model_name=None,
restore_path=None,
w_emb_path="../ckpt/w_emb",
u_emb_path="../ckpt/u_emb"):
np.random.seed(RANDOM_SEED_NP)
data_train = DataDUELoader(meta_data_file=meta_data_file,
batch_data_dir=batch_data_dir_train,
id_map=id_map,
dataToken=dataToken,
max_doc_length=max_doc_length,
size_context=size_context)
if batch_data_dir_valid is not None:
data_valid = DataDUELoader(meta_data_file=meta_data_file,
batch_data_dir=batch_data_dir_valid,
id_map=id_map,
dataToken=dataToken,
max_doc_length=max_doc_length,
size_context=size_context)
else:
data_valid = None
model_spec = json_reader(config_file)
model = Caden(data_spec=data_train.data_spec,
model_spec=model_spec,
model_name=model_name)
model.initialization(w_emb_file=w_emb_path, u_emb_file=u_emb_path)
if restore_path is not None:
model.restore(restore_path)
# train #
results = model.train(data_generator=data_train,
data_generator_valid=data_valid)
print("train_results: %s" % str(results))
best_epoch = read(directory="../summary/" + model.model_name,
main_indicator="epoch_losses_valid_00")[0]
print("best_epoch by validation loss: %d" % best_epoch)
def run_single(
feature_file,
label_file,
weight_file=None,
task_file=None,
train_pattern="",
valid_pattern="",
**model_kwargs_wrap
):
"""
using previously split data for experiment
:param feature_file:
:param label_file:
:param weight_file:
:param task_file:
:param train_pattern: pattern to distinguish training files
:param valid_pattern: pattern to distinguish valid files
:param model_kwargs_wrap: {
"model_kwargs": {
Model:
config_file:
model_name:
partial_restore_paths:
restore_path:
full_split_saver:
}
"model_primary_kwargs": {
}
}
:return:
"""
np.random.seed(RANDOM_SEED_NP)
# initialize data #
datas = []
for data_file_pattern in [train_pattern, valid_pattern]:
datas.append(
DataGeneratorFull(
feature_file=feature_file + data_file_pattern,
label_file=label_file + data_file_pattern,
weight_file=weight_file if weight_file is None else weight_file + data_file_pattern,
task_file=task_file if task_file is None else task_file + data_file_pattern
)
)
data_train, data_valid = datas
# initialize model #
model_kwargs = model_kwargs_wrap["model_kwargs"]
model_primary_kwargs = model_kwargs_wrap["model_primary_kwargs"]
model_spec = model_kwargs["model_spec"]
if model_primary_kwargs is not None:
# train primary model to initialize model #
model_spec_primary_ = json_reader(model_primary_kwargs['config_file'])
model_spec_primary = dict_conservative_update(
dict_base=model_spec_primary_,
dict_new=model_spec
) # make sure the shared hps are consistent
model_spec_primary["optim_params"] = model_spec_primary_["optim_params"]
model_spec_primary["max_epoch"] = model_spec_primary_["max_epoch"]
[best_epoch_primary, results_], model_primary_save_path, _m = train_model(
data_train=data_train,
data_valid=data_valid,
model_spec=model_spec_primary,
**model_primary_kwargs
)
# for models use parts of primary model (e.g., cross_stitch)
for path_name in model_kwargs["partial_restore_paths"]:
name_pattern = model_kwargs["partial_restore_paths"][path_name]
if name_pattern is None:
partial_restore_path = None
else:
partial_restore_path = name_pattern.format(best_epoch_primary)
model_kwargs["partial_restore_paths"][path_name] = partial_restore_path
# for models use full of primary model for graph definition (e.g., dmtrl_Tucker)
if "primary_model_ckpt" in model_spec:
model_spec["primary_model_ckpt"] = model_primary_save_path.format(best_epoch_primary)
[best_epoch, train_final_results], model_best_save_path, model = train_model(
data_train=data_train,
data_valid=data_valid,
**model_kwargs
)
# test #
data_test = data_valid
model = init_model(
data=data_test,
Model=model_kwargs["Model"],
model_spec=model_spec,
model_name=model_kwargs["model_name"]
)
model.restore(
save_path=model_best_save_path.format(int(best_epoch))
)
results = simple_evaluate(
model=model,
data=data_test
)
return {
"training_final": train_final_results,
"best_epoch": best_epoch,
"valid_best": results
}
print("base '%s': %s" % (key_name, str(dict_base[key_name])))
print("new '%s': %s" % (key_name, str(dict_new[key_name])))
raise RuntimeError
if isinstance(dict_base[key_name], dict):
dict_updated[key_name] = dict_update(
dict_base=dict_base[key_name], dict_new=dict_new[key_name])
else:
dict_updated[key_name] = dict_new[key_name]
else:
dict_updated[key_name] = dict_new[key_name]
return dict_updated
if __name__ == "__main__":
ht = HyperparameterTuner(save_path="../ht_log/test")
config = json_reader("../ht_log/test_config.json")
print(config)
config_model = json_reader("../models/dmtrl_Tucker_config.json")
print(config_model)
print("conservative update")
config_model_updated = dict_conservative_update(dict_base=config_model,
dict_new=config)
print(config_model_updated)
ht.initialize(hps=config_model_updated)
print(ht.id2hps)
for id_ in ht.id2hps.keys():
print("id: %d" % id_)
print(ht.out_pattern(keys=ht.hp_names, values=ht.id2hps[id_]))
break
print("start testing")
name="task_hidden_a_dropout")
return hidden_a_dropout
if __name__ == "__main__":
from experiment import DataGeneratorTrainTest, json_reader
data_dir = "../data/MNIST_MTL/"
data = DataGeneratorTrainTest(feature_file=data_dir + "feature_train",
label_file=data_dir + "label_train",
weight_file=None,
task_file=data_dir + "id_train",
train_ratio=0.8,
valid_ratio=0.2,
stage_wise=True)
data_train = data.train
model_spec = json_reader(
"../models/topic_task_sparse_layer_wise_single_layer_config.json")
model = TopicTaskSparseLayerWiseSingleLayer(
feature_dim=data_train.feature_dim,
label_dim=data_train.label_dim,
task_dim=data_train.task_dim,
model_spec=model_spec,
model_name=
"topic_task_sparse_layer_wise_single_layer_MNIST_MTL_nan_test")
model.initialization()
model.restore(
save_path=
"../ckpt/topic_task_sparse_layer_wise_single_layer_MNIST_MTL_nan_test/epoch_087"
)
for weight in model.task_weight_list + model.task_bias_list:
print(weight.name)
print(model.sess.run(weight))