def run_ht( ht_config_file, ht_config_file_additional, ht_save_path, ht_start_id=None, ht_end_id=None, find_best=False, **kwargs ): ht_config = json_reader(ht_config_file) if ht_config_file_additional is not None: ht_config_additional = json_reader(ht_config_file_additional) ht_config = dict_update( dict_base=ht_config, dict_new=ht_config_additional ) model_spec_config_base = json_reader(kwargs["model_kwargs"]["config_file"]) ht = HyperparameterTuner( save_path=ht_save_path ) ht_config["model_spec"] = dict_conservative_update( dict_base=model_spec_config_base, dict_new=ht_config["model_spec"] ) # update model spec for different models to avoid redundant hps print(ht_config) # verbose ht.initialize(hps=ht_config) # remove "config_file" for safety, add "model_spec" to be updated by single hp_config # del kwargs["model_kwargs"]["config_file"] kwargs["model_kwargs"]["model_spec"] = model_spec_config_base for test_id, hp_config in ht.generate( start_id=ht_start_id, end_id=ht_end_id ): kwargs_single = copy.deepcopy(kwargs) kwargs_single["model_kwargs"] = dict_conservative_update( dict_base=kwargs["model_kwargs"], dict_new=hp_config ) print("run %d with config: \n%s" % (test_id, str(kwargs_single))) result_single = run_single( **kwargs_single ) perf = result2perf(result=result_single) ht.read_perf(id_=test_id, perf=perf) if find_best: print(ht.find_best())
def test(config_file, meta_data_file, id_map, dataToken, batch_data_dir, max_doc_length=30, size_context=2, model_name=None, restore_path=None): np.random.seed(RANDOM_SEED_NP) data = DataDUELoader(meta_data_file=meta_data_file, batch_data_dir=batch_data_dir, id_map=id_map, dataToken=dataToken, max_doc_length=max_doc_length, size_context=size_context) model_spec = json_reader(config_file) model = Caden(data_spec=data.data_spec, model_spec=model_spec, model_name=model_name) model.initialization() if restore_path is not None: model.restore(restore_path) perf = performance(model, data) print("ckpt_path: %s" % restore_path) print("performance: %s" % str(perf))
def read_and_dump(filename, filename_target): if not os.path.exists(filename): warnings.warn("'%s' not exists" % filename) return None config = json_reader(filename) model_spec = config["model_spec"] with open(filename_target, "w") as f: json_dumper(model_spec, f)
def test(config_file, meta_data_file, id_map, dataToken, batch_data_dir, max_doc_length=30, model_name=None, restore_path=None, no_doc_index=False): np.random.seed(RANDOM_SEED_NP) data = DataDUELoader(meta_data_file=meta_data_file, batch_data_dir=batch_data_dir, id_map=id_map, dataToken=dataToken, max_doc_length=max_doc_length, no_doc_index=no_doc_index) model_spec = json_reader(config_file) model = FM(feature_shape=(0 if no_doc_index else data.D) + data.U + data.V + 1, feature_dim=(0 if no_doc_index else 1) + 1 + max_doc_length, label_dim=data.E, model_spec=model_spec, model_name=model_name) model.initialization() def performance(model_local, data_local): preds = model_local.predict(data_generator=data_local) labels = [] for data_batched in data_local.generate( batch_size=model_spec["batch_size"], random_shuffle=False): labels.append(data_batched["label"]) labels = np.concatenate(labels, axis=0) # one-hot to index # trues = np.argmax(labels, axis=-1) perf = evaluate(preds=preds, trues=trues) return perf if restore_path is not None: if not isinstance(restore_path, list): restore_paths = [restore_path] else: restore_paths = restore_path for restore_path in restore_paths: model.restore(restore_path) perf = performance(model_local=model, data_local=data) print("ckpt_path: %s" % restore_path) print("performance: %s" % str(perf)) else: perf = performance(model_local=model, data_local=data) print("random initialization") print("performance: %s" % str(perf))
def ht_find_best( ht_config_file, ht_config_file_additional, ht_save_path, ht_perf_files, model_spec_config_file, ): ht_config = json_reader(ht_config_file) if ht_config_file_additional is not None: ht_config_additional = json_reader(ht_config_file_additional) ht_config = dict_update(dict_base=ht_config, dict_new=ht_config_additional) model_spec_config_base = json_reader(model_spec_config_file) ht = HyperparameterTuner(save_path=ht_save_path) ht_config["model_spec"] = dict_conservative_update( dict_base=model_spec_config_base, dict_new=ht_config["model_spec"] ) # update model spec for different models to avoid redundant hps print(ht_config) # verbose ht.initialize(hps=ht_config) for ht_perf_file in ht_perf_files: ht.restore(perfs_file=ht_perf_file) print(ht.find_best(max_best=True))
def train(config_file, meta_data_file, id_map, dataToken, batch_data_dir_train, batch_data_dir_valid=None, max_doc_length=30, model_name=None, restore_path=None, no_doc_index=False): np.random.seed(RANDOM_SEED_NP) data_train = DataDUELoader(meta_data_file=meta_data_file, batch_data_dir=batch_data_dir_train, id_map=id_map, dataToken=dataToken, max_doc_length=max_doc_length, no_doc_index=no_doc_index) if batch_data_dir_valid is not None: data_valid = DataDUELoader(meta_data_file=meta_data_file, batch_data_dir=batch_data_dir_valid, id_map=id_map, dataToken=dataToken, max_doc_length=max_doc_length, no_doc_index=no_doc_index) else: data_valid = None model_spec = json_reader(config_file) model = FM(feature_shape=(0 if no_doc_index else data_train.D) + data_train.U + data_train.V + 1, feature_dim=(0 if no_doc_index else 1) + 1 + max_doc_length, label_dim=data_train.E, model_spec=model_spec, model_name=model_name) model.initialization() if restore_path is not None: model.restore(restore_path) # train # results = model.train(data_generator=data_train, data_generator_valid=data_valid) print("train_results: %s" % str(results)) best_epoch = read(directory="../summary/" + model.model_name, main_indicator="epoch_losses_valid_00")[0] print("best_epoch by validation loss: %d" % best_epoch)
def train(config_file, meta_data_file, id_map, dataToken, batch_data_dir_train, batch_data_dir_valid=None, max_doc_length=30, size_context=2, model_name=None, restore_path=None, w_emb_path="../ckpt/w_emb", u_emb_path="../ckpt/u_emb"): np.random.seed(RANDOM_SEED_NP) data_train = DataDUELoader(meta_data_file=meta_data_file, batch_data_dir=batch_data_dir_train, id_map=id_map, dataToken=dataToken, max_doc_length=max_doc_length, size_context=size_context) if batch_data_dir_valid is not None: data_valid = DataDUELoader(meta_data_file=meta_data_file, batch_data_dir=batch_data_dir_valid, id_map=id_map, dataToken=dataToken, max_doc_length=max_doc_length, size_context=size_context) else: data_valid = None model_spec = json_reader(config_file) model = Caden(data_spec=data_train.data_spec, model_spec=model_spec, model_name=model_name) model.initialization(w_emb_file=w_emb_path, u_emb_file=u_emb_path) if restore_path is not None: model.restore(restore_path) # train # results = model.train(data_generator=data_train, data_generator_valid=data_valid) print("train_results: %s" % str(results)) best_epoch = read(directory="../summary/" + model.model_name, main_indicator="epoch_losses_valid_00")[0] print("best_epoch by validation loss: %d" % best_epoch)
def run_single( feature_file, label_file, weight_file=None, task_file=None, train_pattern="", valid_pattern="", **model_kwargs_wrap ): """ using previously split data for experiment :param feature_file: :param label_file: :param weight_file: :param task_file: :param train_pattern: pattern to distinguish training files :param valid_pattern: pattern to distinguish valid files :param model_kwargs_wrap: { "model_kwargs": { Model: config_file: model_name: partial_restore_paths: restore_path: full_split_saver: } "model_primary_kwargs": { } } :return: """ np.random.seed(RANDOM_SEED_NP) # initialize data # datas = [] for data_file_pattern in [train_pattern, valid_pattern]: datas.append( DataGeneratorFull( feature_file=feature_file + data_file_pattern, label_file=label_file + data_file_pattern, weight_file=weight_file if weight_file is None else weight_file + data_file_pattern, task_file=task_file if task_file is None else task_file + data_file_pattern ) ) data_train, data_valid = datas # initialize model # model_kwargs = model_kwargs_wrap["model_kwargs"] model_primary_kwargs = model_kwargs_wrap["model_primary_kwargs"] model_spec = model_kwargs["model_spec"] if model_primary_kwargs is not None: # train primary model to initialize model # model_spec_primary_ = json_reader(model_primary_kwargs['config_file']) model_spec_primary = dict_conservative_update( dict_base=model_spec_primary_, dict_new=model_spec ) # make sure the shared hps are consistent model_spec_primary["optim_params"] = model_spec_primary_["optim_params"] model_spec_primary["max_epoch"] = model_spec_primary_["max_epoch"] [best_epoch_primary, results_], model_primary_save_path, _m = train_model( data_train=data_train, data_valid=data_valid, model_spec=model_spec_primary, **model_primary_kwargs ) # for models use parts of primary model (e.g., cross_stitch) for path_name in model_kwargs["partial_restore_paths"]: name_pattern = model_kwargs["partial_restore_paths"][path_name] if name_pattern is None: partial_restore_path = None else: partial_restore_path = name_pattern.format(best_epoch_primary) model_kwargs["partial_restore_paths"][path_name] = partial_restore_path # for models use full of primary model for graph definition (e.g., dmtrl_Tucker) if "primary_model_ckpt" in model_spec: model_spec["primary_model_ckpt"] = model_primary_save_path.format(best_epoch_primary) [best_epoch, train_final_results], model_best_save_path, model = train_model( data_train=data_train, data_valid=data_valid, **model_kwargs ) # test # data_test = data_valid model = init_model( data=data_test, Model=model_kwargs["Model"], model_spec=model_spec, model_name=model_kwargs["model_name"] ) model.restore( save_path=model_best_save_path.format(int(best_epoch)) ) results = simple_evaluate( model=model, data=data_test ) return { "training_final": train_final_results, "best_epoch": best_epoch, "valid_best": results }
print("base '%s': %s" % (key_name, str(dict_base[key_name]))) print("new '%s': %s" % (key_name, str(dict_new[key_name]))) raise RuntimeError if isinstance(dict_base[key_name], dict): dict_updated[key_name] = dict_update( dict_base=dict_base[key_name], dict_new=dict_new[key_name]) else: dict_updated[key_name] = dict_new[key_name] else: dict_updated[key_name] = dict_new[key_name] return dict_updated if __name__ == "__main__": ht = HyperparameterTuner(save_path="../ht_log/test") config = json_reader("../ht_log/test_config.json") print(config) config_model = json_reader("../models/dmtrl_Tucker_config.json") print(config_model) print("conservative update") config_model_updated = dict_conservative_update(dict_base=config_model, dict_new=config) print(config_model_updated) ht.initialize(hps=config_model_updated) print(ht.id2hps) for id_ in ht.id2hps.keys(): print("id: %d" % id_) print(ht.out_pattern(keys=ht.hp_names, values=ht.id2hps[id_])) break print("start testing")
name="task_hidden_a_dropout") return hidden_a_dropout if __name__ == "__main__": from experiment import DataGeneratorTrainTest, json_reader data_dir = "../data/MNIST_MTL/" data = DataGeneratorTrainTest(feature_file=data_dir + "feature_train", label_file=data_dir + "label_train", weight_file=None, task_file=data_dir + "id_train", train_ratio=0.8, valid_ratio=0.2, stage_wise=True) data_train = data.train model_spec = json_reader( "../models/topic_task_sparse_layer_wise_single_layer_config.json") model = TopicTaskSparseLayerWiseSingleLayer( feature_dim=data_train.feature_dim, label_dim=data_train.label_dim, task_dim=data_train.task_dim, model_spec=model_spec, model_name= "topic_task_sparse_layer_wise_single_layer_MNIST_MTL_nan_test") model.initialization() model.restore( save_path= "../ckpt/topic_task_sparse_layer_wise_single_layer_MNIST_MTL_nan_test/epoch_087" ) for weight in model.task_weight_list + model.task_bias_list: print(weight.name) print(model.sess.run(weight))