# initialize experiment / load dataset logger = initialize_exp(params) data, attributes, data2, attributes2 = load_images(params) train_data = DataSampler(data[0], attributes[0], data2, attributes2, params) valid_data = DataSampler(data[1], attributes[1], None, None, params) # build the model ae = AutoEncoder(params).cuda() lat_dis = LatentDiscriminator(params).cuda() if params.n_lat_dis else None ptc_dis = PatchDiscriminator(params).cuda() if params.n_ptc_dis else None clf_dis = Classifier(params).cuda() if params.n_clf_dis else None eval_clf = torch.load(params.eval_clf).cuda().eval() # trainer / evaluator trainer = Trainer(ae, lat_dis, ptc_dis, clf_dis, train_data, params) evaluator = Evaluator(ae, lat_dis, ptc_dis, clf_dis, eval_clf, valid_data, params) for n_epoch in range(params.n_epochs): logger.info('Starting epoch %i...' % n_epoch) for n_iter in range(0, params.epoch_size, params.batch_size): # latent discriminator training for _ in range(params.n_lat_dis): trainer.lat_dis_step() # patch discriminator training for _ in range(params.n_ptc_dis): trainer.ptc_dis_step()
for n in range(params.num_samples): mappings.append( sample_from_multivariate_gaussian(interpolated_mapping, var=params.var)) for m, sampled_mapping in enumerate(mappings): trainer.set_mapping_weights(weights=sampled_mapping) # compute the discriminator loss logger.info('----> COMPUTING DISCRIMINATOR LOSS <----\n\n') logger.info('alpha={}, m={}'.format(alpha, m)) losses = [] for n_iter in range(0, params.epoch_size, params.batch_size): loss = trainer.compute_loss() losses.append(loss.cpu().data.numpy()) #print(losses) logger.info('Discriminator loss {}: {}\n'.format(m, np.mean(losses))) evaluator = Evaluator(trainer) # run evaluations to_log = OrderedDict({'n_iter': 0}) evaluator.monolingual_wordsim(to_log) # evaluator.monolingual_wordanalogy(to_log) if params.tgt_lang: evaluator.crosslingual_wordsim(to_log) evaluator.word_translation(to_log) #evaluator.sent_translation(to_log) evaluator.dist_mean_cosine(to_log)
def main(): VALIDATION_METRIC_SUP = 'precision_at_1-csls_knn_10' VALIDATION_METRIC_UNSUP = 'mean_cosine-csls_knn_10-S2T-10000' # main parser = argparse.ArgumentParser(description='Supervised training') parser.add_argument("--seed", type=int, default=-1, help="Initialization seed") parser.add_argument("--verbose", type=int, default=2, help="Verbose level (2:debug, 1:info, 0:warning)") parser.add_argument("--exp_path", type=str, default="", help="Where to store experiment logs and models") parser.add_argument("--exp_name", type=str, default="debug", help="Experiment name") parser.add_argument("--exp_id", type=str, default="", help="Experiment ID") parser.add_argument("--cuda", type=bool_flag, default=True, help="Run on GPU") parser.add_argument("--export", type=str, default="txt", help="Export embeddings after training (txt / pth)") # data parser.add_argument("--src_lang", type=str, default='en', help="Source language") parser.add_argument("--tgt_lang", type=str, default='es', help="Target language") parser.add_argument("--aux_lang", type=str, default='', help="Auxiliary language") parser.add_argument("--emb_dim", type=int, default=300, help="Embedding dimension") parser.add_argument("--max_vocab", type=int, default=200000, help="Maximum vocabulary size (-1 to disable)") # training refinement parser.add_argument("--n_refinement", type=int, default=5, help="Number of refinement iterations (0 to disable the refinement procedure)") # dictionary creation parameters (for refinement) parser.add_argument("--dico_train", type=str, default="default", help="Path to training dictionary (default: use identical character strings)") parser.add_argument("--dico_eval", type=str, default="default", help="Path to evaluation dictionary") parser.add_argument("--dico_method", type=str, default='csls_knn_10', help="Method used for dictionary generation (nn/invsm_beta_30/csls_knn_10)") parser.add_argument("--dico_build", type=str, default='S2T&T2S', help="S2T,T2S,S2T|T2S,S2T&T2S") parser.add_argument("--dico_threshold", type=float, default=0, help="Threshold confidence for dictionary generation") parser.add_argument("--dico_max_rank", type=int, default=10000, help="Maximum dictionary words rank (0 to disable)") parser.add_argument("--dico_min_size", type=int, default=0, help="Minimum generated dictionary size (0 to disable)") parser.add_argument("--dico_max_size", type=int, default=0, help="Maximum generated dictionary size (0 to disable)") # reload pre-trained embeddings parser.add_argument("--src_emb", type=str, default='', help="Reload source embeddings") parser.add_argument("--tgt_emb", type=str, default='', help="Reload target embeddings") parser.add_argument("--aux_emb", type=str, default='', help="Reload auxiliary embeddings") parser.add_argument("--normalize_embeddings", type=str, default="", help="Normalize embeddings before training") parser.add_argument("--fitting_method", type=str, default="non_iterative", help="Method of fitting, one of [non_iterative, em, gauss_seidel, gradient_based]") # parse parameters params = parser.parse_args() # check parameters assert not params.cuda or torch.cuda.is_available() assert params.dico_train in ["identical_char", "default"] or os.path.isfile(params.dico_train) assert params.dico_build in ["S2T", "T2S", "S2T|T2S", "S2T&T2S"] assert params.dico_max_size == 0 or params.dico_max_size < params.dico_max_rank assert params.dico_max_size == 0 or params.dico_max_size > params.dico_min_size print(params.src_emb, params.tgt_emb, params.aux_emb) assert os.path.isfile(params.src_emb) assert os.path.isfile(params.tgt_emb) assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) assert params.export in ["", "txt", "pth"] # build logger / model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, aux_emb, mapping, _ = build_model(params, False) trainer = Trainer(src_emb, tgt_emb, aux_emb, mapping, None, params) # load a training dictionary. if a dictionary path is not provided, use a default # one ("default") or create one based on identical character strings ("identical_char") trainer.load_training_dico(params.dico_train) # define the validation metric VALIDATION_METRIC = VALIDATION_METRIC_UNSUP if params.dico_train == 'identical_char' else VALIDATION_METRIC_SUP logger.info("Validation metric: %s" % VALIDATION_METRIC) # apply the PCCA solution trainer.fit(fitting_method=params.fitting_method) # IMPORTANT: EVALUATOR SHOULD BE CREATED AFTER TRAINER HAS BEEN FITTED evaluator = Evaluator(trainer) # embeddings evaluation to_log = OrderedDict({}) evaluator.all_eval(to_log) logger.info("__log__:%s" % json.dumps(to_log))
assert not params.cuda or torch.cuda.is_available() assert 0 <= params.dis_dropout < 1 assert 0 <= params.dis_input_dropout < 1 assert 0 <= params.dis_smooth < 0.5 assert params.dis_lambda > 0 and params.dis_steps > 0 assert 0 < params.lr_shrink <= 1 assert os.path.isfile(params.src_emb) assert os.path.isfile(params.tgt_emb) assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) assert params.export in ["", "txt", "pth"] # build model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, discriminator = build_model(params, True) trainer = Trainer(src_emb, tgt_emb, mapping, discriminator, params) evaluator = Evaluator(trainer) writer = SummaryWriter() """ Learning loop for Adversarial Training """ if params.adversarial: logger.info('----> ADVERSARIAL TRAINING <----\n\n') # training loop for n_epoch in range(params.n_epochs): logger.info('Starting adversarial training epoch %i...' % n_epoch) tic = time.time()
assert 0 <= params.dis_dropout < 1 assert 0 <= params.dis_input_dropout < 1 assert 0 <= params.dis_smooth < 0.5 assert params.dis_lambda > 0 and params.dis_steps > 0 assert 0 < params.lr_shrink <= 1 assert os.path.isfile(params.src_emb) assert os.path.isfile(params.tgt_emb) assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) assert params.export in ["", "txt", "pth"] # build model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, discriminator = build_model(params, True) trainer = Trainer(src_emb, tgt_emb, mapping, discriminator, params) evaluator = Evaluator(trainer) writer = SummaryWriter() with open("data/results/wordresult_" + params.exp_id + ".txt", "wr") as w: """ Learning loop for Adversarial Training """ if params.adversarial: logger.info('----> ADVERSARIAL TRAINING <----\n\n') # training loop for n_epoch in range(params.n_epochs): logger.info('Starting adversarial training epoch %i...' % n_epoch) tic = time.time()
def evaluate_real_datasets(): REAL_DATASET_GROUP_PATH = 'data/raw/' real_dataset_groups = glob.glob(REAL_DATASET_GROUP_PATH + '*') seeds = np.random.randint(np.iinfo(np.uint32).max, size=RUNS, dtype=np.uint32) results = pd.DataFrame() datasets = [KDDCup(seed=1)] for real_dataset_group in real_dataset_groups: for data_set_path in glob.glob(real_dataset_group + '/labeled/train/*'): data_set_name = data_set_path.split('/')[-1].replace('.pkl', '') dataset = RealPickledDataset(data_set_name, data_set_path) datasets.append(dataset) for seed in seeds: datasets[0] = KDDCup(seed) evaluator = Evaluator(datasets, detectors, seed=seed) evaluator.evaluate() result = evaluator.benchmarks() evaluator.plot_roc_curves() evaluator.plot_threshold_comparison() evaluator.plot_scores() results = results.append(result, ignore_index=True) avg_results = results.groupby(['dataset', 'algorithm'], as_index=False).mean() evaluator.set_benchmark_results(avg_results) evaluator.export_results('run_real_datasets') evaluator.create_boxplots(runs=RUNS, data=results, detectorwise=False) evaluator.create_boxplots(runs=RUNS, data=results, detectorwise=True)
assert 0 < params.lr_shrink <= 1 assert os.path.isfile(params.src_emb) assert os.path.isfile(params.tgt_emb) assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) assert params.export in ["", "txt", "pth"] # build model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, _, mapping, discriminator = build_model(params, True) if params.glo_emb: glo_emb = KeyedVectors.load_word2vec_format(params.glo_emb) else: glo_emb = None trainer = Trainer(src_emb, tgt_emb, glo_emb, params.eval_emb, mapping, discriminator, params) evaluator = Evaluator(trainer) """ Learning loop for Adversarial Training """ if params.adversarial: logger.info('----> ADVERSARIAL TRAINING <----\n\n') # training loop for n_epoch in range(params.n_epochs): logger.info('Starting adversarial training epoch %i...' % n_epoch) tic = time.time() n_words_proc = 0 stats = {'DIS_COSTS': []} for n_iter in range(0, params.epoch_size, params.batch_size):
assert os.path.isfile(params.tgt_emb) assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) assert params.export in ["", "txt", "pth"] params.verbose = 0 params.leave_one_out = True # build logger / model / trainer / evaluator #do all this stuff once to get the word2id's logger = initialize_exp(params) src_emb, tgt_emb, eval_emb, mapping, _ = build_model(params, False) if params.glo_emb: glo_emb = KeyedVectors.load_word2vec_format(params.glo_emb) if params.glo_emb else None else: glo_emb = None trainer = Trainer(src_emb, tgt_emb, glo_emb, eval_emb, mapping, None, params) evaluator = Evaluator(trainer) # load a training dictionary. if a dictionary path is not provided, use a default # one ("default") or create one based on identical character strings ("identical_char") trainer.load_training_dico(params.dico_train, params.subsample) eval_dico = load_dictionary(params.dico_eval, trainer.src_dico.word2id, trainer.tgt_dico.word2id) if params.cuda: eval_dico.cuda() dico = trainer.dico.clone() # define the validation metric logger.info("Validation metric: %s" % params.val_metric) cos = torch.nn.CosineSimilarity(dim=1) ranks = []
assert not params.cuda or torch.cuda.is_available() assert 0 <= params.dis_dropout < 1 assert 0 <= params.dis_input_dropout < 1 assert 0 <= params.dis_smooth < 0.5 assert params.dis_lambda > 0 and params.dis_steps > 0 assert 0 < params.lr_shrink <= 1 assert os.path.isfile(params.src_emb) assert all(os.path.isfile(emb) for emb in params.tgt_emb) assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) assert params.export in ["", "txt", "pth"] # build model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, discriminator = build_model(params, True) trainer = Trainer(src_emb, tgt_emb, mapping, discriminator, params) evaluator = Evaluator(trainer) """ Learning loop for Adversarial Training """ if params.adversarial: logger.info('----> ADVERSARIAL TRAINING <----\n\n') # training loop for n_epoch in range(params.n_epochs): logger.info('Starting adversarial training epoch %i...' % n_epoch) tic = time.time() n_words_proc = 0 stats = {'DIS_COSTS': []} for n_iter in range(0, params.epoch_size, params.batch_size):
assert not params.cuda or torch.cuda.is_available() assert 0 <= params.dis_dropout < 1 assert 0 <= params.dis_input_dropout < 1 assert 0 <= params.dis_smooth < 0.5 assert params.dis_lambda > 0 and params.dis_steps > 0 assert 0 < params.lr_shrink <= 1 assert os.path.isfile(params.src_emb) assert os.path.isfile(params.tgt_emb) assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) assert params.export in ["", "txt", "pth"] # build model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, discriminator = build_model(params, True) trainer = Trainer(src_emb, tgt_emb, mapping, discriminator, params) evaluator = Evaluator(trainer) """ Learning loop for Adversarial Training """ if params.adversarial: logger.info('----> ADVERSARIAL TRAINING <----\n\n') # training loop for n_epoch in range(params.n_epochs): logger.info('Starting adversarial training epoch %i...' % n_epoch) tic = time.time() n_words_proc = 0 stats = {'DIS_COSTS': []} for n_iter in range(0, params.epoch_size, params.batch_size):
assert not params.cuda or torch.cuda.is_available() assert 0 <= params.dis_dropout < 1 assert 0 <= params.dis_input_dropout < 1 assert 0 <= params.dis_smooth < 0.5 assert params.dis_lambda > 0 and params.dis_steps > 0 assert 0 < params.lr_shrink <= 1 assert os.path.isfile(params.src_emb) assert os.path.isfile(params.tgt_emb) assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) assert params.export in ["", "txt", "pth"] # build model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, discriminator = build_model(params, True) trainer = Trainer(src_emb, tgt_emb, mapping, discriminator, params) evaluator = Evaluator(trainer) # load a training dictionary (supervision) trainer.load_training_dico(params.dico_train) """ Learning loop for Adversarial Training """ if params.adversarial: logger.info('----> ADVERSARIAL TRAINING <----\n\n') # training loop for n_epoch in range(params.n_epochs): logger.info('Starting adversarial training epoch %i...' % n_epoch) tic = time.time() n_words_proc = 0
def learning(params, src_data, tgt_data, options): VALIDATION_METRIC = 'mean_cosine-csls_knn_10-S2T-10000' logger = logging.getLogger('{}Log'.format(src_data.dataname)) for i in range(10): # tic = time.time() if i == 0: options.initialize = True else: options.initialize = False logger.info("src_learning {}回目".format(i + 1)) src_data = RVSML_OT_Learning(src_data, options, params) logger.info("tgt_learning {}回目".format(i + 1)) tgt_data = RVSML_OT_Learning(tgt_data, options, params) # build model / trainer / evaluator src_emb, tgt_emb, mapping, discriminator = build_model( params, src_data, tgt_data, True) trainer = Trainer(src_emb, tgt_emb, mapping, discriminator, params) evaluator = Evaluator(trainer) """ Learning loop for Adversarial Training """ logger.info('----> ADVERSARIAL TRAINING <----\n\n') # training loop for n_epoch in range(params.n_epochs): logger.info('Starting adversarial training epoch %i...' % n_epoch) tic = time.time() n_words_proc = 0 stats = {'DIS_COSTS': []} for n_iter in range(0, params.epoch_size, params.batch_size): # discriminator training for _ in range(params.dis_steps): trainer.dis_step(stats) # mapping training (discriminator fooling) n_words_proc += trainer.mapping_step(stats) # log stats if n_iter % 500 == 0: stats_str = [('DIS_COSTS', 'Discriminator loss')] stats_log = [ '%s: %.4f' % (v, np.mean(stats[k])) for k, v in stats_str if len(stats[k]) > 0 ] stats_log.append('%i samples/s' % int(n_words_proc / (time.time() - tic))) logger.info(('%06i - ' % n_iter) + ' - '.join(stats_log)) # reset tic = time.time() n_words_proc = 0 for k, _ in stats_str: del stats[k][:] # embeddings / discriminator evaluation to_log = OrderedDict({'n_epoch': n_epoch}) evaluator.all_eval(to_log) evaluator.eval_dis(to_log) # JSON log / save best model / end of epoch logger.info("__log__:%s" % json.dumps(to_log)) trainer.save_best(to_log, VALIDATION_METRIC) logger.info('End of epoch %i.\n\n' % n_epoch) # update the learning rate (stop if too small) trainer.update_lr(to_log, VALIDATION_METRIC) if trainer.map_optimizer.param_groups[0]['lr'] < params.min_lr: logger.info('Learning rate < 1e-6. BREAK.') break """ Learning loop for Procrustes Iterative Refinement """ # if params.n_refinement > 0: # Get the best mapping according to VALIDATION_METRIC logger.info('----> ITERATIVE PROCRUSTES REFINEMENT <----\n\n') trainer.reload_best() # training loop for n_iter in range(params.n_refinement): logger.info('Starting refinement iteration %i...' % n_iter) # build a dictionary from aligned embeddings trainer.build_dictionary() # apply the Procrustes solution trainer.procrustes() # embeddings evaluation to_log = OrderedDict({'n_iter': n_iter}) evaluator.all_eval(to_log) # JSON log / save best model / end of epoch logger.info("__log__:%s" % json.dumps(to_log)) trainer.save_best(to_log, VALIDATION_METRIC) logger.info('End of refinement iteration %i.\n\n' % n_iter) src_data.trans_mat = torch.mm(src_data.trans_mat, trainer.mapping.weight.data) return src_data, tgt_data
assert 0 <= params.dis_dropout < 1 assert 0 <= params.dis_input_dropout < 1 assert 0 <= params.dis_smooth < 0.5 assert params.dis_lambda > 0 and params.dis_steps > 0 assert 0 < params.lr_shrink <= 1 assert os.path.isfile(params.src_emb) assert os.path.isfile(params.tgt_emb) # assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) assert params.dico_eval == '' or os.path.isfile(params.dico_eval) assert params.export in ["", "txt", "pth"] # build model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, discriminator = build_model(params, True) trainer = Trainer(src_emb, tgt_emb, mapping, discriminator, params) evaluator = Evaluator(trainer) """ Learning loop for Adversarial Training """ if params.adversarial: logger.info('----> ADVERSARIAL TRAINING <----\n\n') # training loop for n_epoch in range(params.n_epochs): logger.info('Starting adversarial training epoch %i...' % n_epoch) tic = time.time() n_words_proc = 0 stats = {'DIS_COSTS': []} for n_iter in range(0, params.epoch_size, params.batch_size):
help="Normalize embeddings before training") # inference parameters parser.add_argument("--multilingual_inference_method", nargs='+', help="which inference methods to use", default=['BI', 'NT', 'CNT', 'CAT']) # parse parameters params = parser.parse_args() # check parameters assert not params.cuda or torch.cuda.is_available() assert all(os.path.isfile(emb) for emb in params.embs) assert len(params.langs) == len(params.embs) assert all([ inf_met in ['BI', 'NT', 'CNT', 'CAT'] or inf_met.startswith('CAT') for inf_met in params.multilingual_inference_method ]) # build logger / model / trainer / evaluator logger = initialize_exp(params) embs, mapping = build_model(params) trainer = Trainer(embs, mapping, params) evaluator = Evaluator(trainer) """ Inference with MWT (Multilingual Word Translation) """ logger.info('Starting inference...') # embeddings evaluation evaluator.word_translation() logger.info('End of inference.\n\n')
params = parser.parse_args() # check parameters assert not params.cuda or torch.cuda.is_available() assert params.dico_train in ["identical_char", "default"] or os.path.isfile(params.dico_train) assert params.dico_build in ["S2T", "T2S", "S2T|T2S", "S2T&T2S"] assert params.dico_max_size == 0 or params.dico_max_size < params.dico_max_rank assert params.dico_max_size == 0 or params.dico_max_size > params.dico_min_size assert os.path.isfile(params.src_emb) assert os.path.isfile(params.tgt_emb) # build logger / model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, _ = build_model(params, False) trainer = Trainer(src_emb, tgt_emb, mapping, None, params) evaluator = Evaluator(trainer) # load a training dictionary. if a dictionary path is not provided, use a default # one ("default") or create one based on identical character strings ("identical_char") trainer.load_training_dico(params.dico_train) """ Learning loop for Procrustes Iterative Refinement """ for n_iter in range(params.n_iters): logger.info('Starting refinement iteration %i...' % n_iter) # build a dictionary from aligned embeddings (unless # it is the first iteration and we use the init one) if n_iter > 0 or not hasattr(trainer, 'dico'):
# parse parameters params = parser.parse_args() # check parameters assert params.src_lang, "source language undefined" assert os.path.isfile(params.src_emb) assert not params.tgt_lang or os.path.isfile(params.tgt_emb) assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) # build logger / model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, _ = build_model(params, False) trainer = Trainer(src_emb, tgt_emb, mapping, None, params) evaluator = Evaluator(trainer) # run evaluations to_log = OrderedDict({'n_iter': 0}) evaluator.cluster_accuracy(to_log) # evaluator.monolingual_wordsim(to_log) # evaluator.monolingual_wordanalogy(to_log) if params.tgt_lang: evaluator.cluster_accuracy(to_log, cl="separately") evaluator.cluster_accuracy(to_log, cl="multilingual") # evaluator.crosslingual_wordsim(to_log) # evaluator.word_translation(to_log) # evaluator.sent_translation(to_log) # evaluator.dist_mean_cosine(to_log) logger.info(" === EVALUTATION RESULTS === ")
# build the trainable model ae = AutoEncoder(params).cuda() # build the Teacher model (if required) if params.lambda_jacobian > 0: params2 = params params2.max_fm = 512 + params.n_attr ae_teacher = AutoEncoder(params2).cuda() params.max_fm = params.max_fm_orig else: ae_teacher = None # trainer / evaluator trainer = Trainer(ae, ae_teacher, train_data, params) evaluator = Evaluator(ae, ae_teacher, valid_data, params) for n_epoch in range(params.n_epochs): logger.info('Starting epoch %i...' % n_epoch) for eee in range(25): evaluator.autoencoder_step(iterno=eee, epoch=n_epoch) evaluator.step(n_epoch) for n_iter in range(0, params.epoch_size, params.batch_size): # autoencoder training trainer.autoencoder_step() # print training statistics
path = os.path.join(trainer.params.exp_path, 'translation.tsv') logger.info('Saving translations to %s', path) with open(path, 'w') as out, open('corpora/voynich.txt') as inp: for idx, line in enumerate(inp): line = line.rstrip() translation = ' '.join([txt_dict[w] for w in line.split()]) if idx < 10: logger.info(translation) out.write(translation + '\n') # build model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, discriminator = build_model(params, True) trainer = Trainer(src_emb, tgt_emb, mapping, discriminator, params) evaluator = Evaluator(trainer) """ Learning loop for Adversarial Training """ if params.adversarial: logger.info('----> ADVERSARIAL TRAINING <----\n\n') # training loop for n_epoch in range(params.n_epochs): logger.info('Starting adversarial training epoch %i...' % n_epoch) tic = time.time() n_words_proc = 0 stats = {'DIS_COSTS': []} for n_iter in range(0, params.epoch_size, params.batch_size):
assert 0 <= params.dis_input_dropout < 1 assert 0 <= params.dis_smooth < 0.5 assert params.dis_lambda > 0 and params.dis_steps > 0 assert 0 < params.lr_shrink <= 1 assert os.path.isfile(params.src_emb) assert os.path.isfile(params.tgt_emb) assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) assert params.export in ["", "txt", "pth"] # build model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, discriminator = build_model(params, True) trainer = Trainer(src_emb, tgt_emb, mapping, discriminator, params) evaluator = Evaluator(trainer) np.random.seed(params.seed) # init generator parameters with artetxe's methods if params.map_init == "second_order": m_init = extract_initial_mapping(params, src_emb, tgt_emb) trainer.set_mapping_weights(torch.from_numpy(m_init)) # if we initialize the generator from a supervised mapping, evaluate before training for sanity check if not params.map_id_init: # embeddings / discriminator evaluation to_log = OrderedDict({'n_epoch': -1}) evaluator.all_eval(to_log) evaluator.eval_dis(to_log) """ Learning loop for Adversarial Training
assert params.dico_build in ["S2T", "T2S", "S2T|T2S", "S2T&T2S"] assert params.dico_max_size == 0 or params.dico_max_size < params.dico_max_rank assert params.dico_max_size == 0 or params.dico_max_size > params.dico_min_size assert os.path.isfile(params.src_emb) assert all(os.path.isfile(emb) for emb in params.tgt_emb) #assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) assert params.export in ["", "txt", "pth"] assert len(params.tgt_lang) == len(params.tgt_emb) assert len(params.tgt_lang) == 1 or params.generalized assert params.fine_tuning <= params.n_refinement # build logger / model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, _ = build_model(params, False) trainer = Trainer(src_emb, tgt_emb, mapping, None, params) evaluator = Evaluator(trainer) #start generalized training with support support = True if params.generalized else False # load a training dictionary. if a dictionary path is not provided, use a default # one ("default") or create one based on identical character strings ("identical_char") trainer.load_training_dico(params.dico_train, support) """ Learning loop for Procrustes Iterative Learning """ for n_iter in range(params.n_refinement + 1): if n_iter > params.n_refinement - params.fine_tuning: support = False logger.info('Starting iteration %i...' % n_iter)
parser.add_argument("--normalize_embeddings", type=str, default="", help="Normalize embeddings before training") parser.add_argument("--validate_path", type=str, default="", help="Validation words") # parse parameters params = parser.parse_args() # build model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, discriminator = build_model(params, True) trainer = Trainer(src_emb, tgt_emb, mapping, discriminator, params) trainer.reload_best() evaluator = Evaluator(trainer) if params.validate_path == "": while True: word = raw_input("Query word: ") print word to_log2 = OrderedDict({'n_epoch': 0}) evaluator.validate_words(to_log2, [word]) print to_log2["Word_dict"].keys() for wordKey in to_log2["Word_dict"].keys(): print to_log2["Word_dict"][wordKey] else: en_words = []
assert params.dis_lambda > 0 and params.dis_steps > 0 assert 0 < params.lr_shrink <= 1 assert os.path.isfile(params.src_emb) assert os.path.isfile(params.tgt_emb) assert params.dico_eval == 'default' or params.dico_eval == 'vecmap' or os.path.isfile( params.dico_eval) assert params.export in ["", "txt", "pth"] # build model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping_G, mapping_F, discriminator_A, discriminator_B, encoder_A, decoder_A, encoder_B, decoder_B = build_model( params, True) trainer = Trainer(src_emb, tgt_emb, mapping_G, mapping_F, discriminator_A, discriminator_B, encoder_A, decoder_A, encoder_B, decoder_B, params) evaluator = Evaluator(trainer) """ Learning loop for Adversarial Training """ if params.adversarial: # first train the autoencoder to become mature trainer.train_autoencoder_A() trainer.train_autoencoder_B() logger.info('----> ADVERSARIAL TRAINING <----\n\n') # adversarial training loop for n_epoch in range(params.n_epochs): logger.info('Starting adversarial training epoch %i...' % n_epoch)
# check parameters assert not params.cuda or torch.cuda.is_available() assert 0 <= params.dis_dropout < 1 assert 0 <= params.dis_input_dropout < 1 assert 0 <= params.dis_smooth < 0.5 assert params.dis_lambda > 0 and params.dis_steps > 0 assert 0 < params.lr_shrink <= 1 assert os.path.isfile(params.src_emb) assert os.path.isfile(params.tgt_emb) # build model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, discriminator = build_model(params, True) trainer = Trainer(src_emb, tgt_emb, mapping, discriminator, params) evaluator = Evaluator(trainer) """ Learning loop for Adversarial Training """ if params.adversarial: logger.info('----> ADVERSARIAL TRAINING <----\n\n') # training loop for n_epoch in range(params.n_epochs): logger.info('Starting adversarial training epoch %i...' % n_epoch) tic = time.time() n_words_proc = 0 stats = {'DIS_COSTS': []} for n_iter in range(0, params.epoch_size, params.batch_size):
# reload pre-trained embeddings parser.add_argument("--src_emb", type=str, default="", help="Reload source embeddings") parser.add_argument("--tgt_emb", type=str, default="", help="Reload target embeddings") parser.add_argument("--max_vocab", type=int, default=200000, help="Maximum vocabulary size") parser.add_argument("--emb_dim", type=int, default=300, help="Embedding dimension") parser.add_argument("--normalize_embeddings", type=str, default="", help="Normalize embeddings before training") # parse parameters params = parser.parse_args() # check parameters assert params.src_lang, "source language undefined" assert os.path.isfile(params.src_emb) assert not params.tgt_lang or os.path.isfile(params.tgt_emb) # build logger / model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, _ = build_model(params, False) trainer = Trainer(src_emb, tgt_emb, mapping, None, params) evaluator = Evaluator(trainer) # run evaluations to_log = OrderedDict({'n_iter': 0}) evaluator.monolingual_wordsim(to_log) if params.tgt_lang: evaluator.crosslingual_wordsim(to_log) evaluator.word_translation(to_log) evaluator.sent_translation(to_log) # evaluator.dist_mean_cosine(to_log)
parser.add_argument("--model_name",type=str, default="", help="Model name") # parse parameters params = parser.parse_args() # check parameters assert params.src_lang, "source language undefined" assert os.path.isfile(params.src_emb) assert not params.tgt_lang or os.path.isfile(params.tgt_emb) assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) # build logger / model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, _ = build_model(params, False) trainer = Trainer(src_emb, tgt_emb, mapping, None, params) evaluator = Evaluator(trainer) # run evaluations to_log = OrderedDict({'n_iter': 0}) log_export = { "name" : params.model_name } evaluator.monolingual_wordsim(to_log,log_export) evaluator.monolingual_wordanalogy(to_log,log_export) if params.tgt_lang: # evaluator.crosslingual_wordsim(to_log) evaluator.word_translation(to_log) # evaluator.sent_translation(to_log) evaluator.dist_mean_cosine(to_log) log_export['nn-1'] = to_log['precision_at_1-nn']
def main(): params = load_args() logger = create_logger( os.path.join(params.exp_path, "lnmap-experiment.log")) logger.info("{}".format( jsbeautifier.beautify(json.dumps(params.__dict__), opts))) set_seed(params) src_emb, tgt_emb, mapping_G, mapping_F, encoder_A, decoder_A, encoder_B, decoder_B = build_model( params) trainer = Trainer(src_emb, tgt_emb, mapping_G, mapping_F, encoder_A, decoder_A, encoder_B, decoder_B, params) evaluator = Evaluator(trainer) trainer.load_training_dico(logger) trainer.load_training_dico(logger, src2tgt=False) logger.info("Seed dictionary size: {}".format(trainer.dico_AB.shape[0])) trainer.dico_AB_original = trainer.dico_AB.clone() trainer.dico_BA_original = trainer.dico_BA.clone() if params.load_autoenc_weights: load_autoenc_weights(params, trainer, logger) else: trainer.train_autoencoder_A(logger) trainer.train_autoencoder_B(logger) if params.save_autoenc_weights: save_autoenc_weights(params, trainer, logger) # Source to Target Training logger.info("\n \n Training for {} to {}".format(params.src_lang, params.tgt_lang)) for i in range(params.iteration): trainer.train_A2B() emb1 = (trainer.mapping_G( trainer.encoder_A( trainer.src_emb.weight.data)).data)[0:params.dico_max_rank] emb2 = (trainer.encoder_B( trainer.tgt_emb.weight.data).data)[0:params.dico_max_rank] emb1 = emb1 / emb1.norm(2, 1, keepdim=True).expand_as(emb1) emb2 = emb2 / emb2.norm(2, 1, keepdim=True).expand_as(emb2) all_pairs, all_scores = generate_new_dictionary_bidirectional( emb1, emb2) add_size = params.induced_dico_c * (i + 1) trainer.dico_AB = torch.cat( (trainer.dico_AB_original, all_pairs[:add_size].cuda()), 0) if i == 0: logger.info( "After first iteration train dictionary size: {}".format( trainer.dico_AB.shape[0])) logger.info("Final iteration train dictionary size: {}".format( trainer.dico_AB.shape[0])) trainer.set_eval() precision_at_1 = get_word_translation_accuracy( params, trainer.mapping_G(trainer.encoder_A( trainer.src_emb.weight.data).data).data, trainer.encoder_B(trainer.tgt_emb.weight.data).data, src2tgt=True) if params.save_model_weights: save_model_weights(params, trainer, src2tgt=True) # Target to Source Training logger.info("\n \n Training for {} to {}".format(params.tgt_lang, params.src_lang)) n_iter = 0 for i in range(params.iteration): trainer.train_B2A() emb1 = ((trainer.encoder_A( trainer.src_emb.weight.data)).data)[0:params.dico_max_rank] emb2 = (trainer.mapping_F( trainer.encoder_B( trainer.tgt_emb.weight.data)).data)[0:params.dico_max_rank] emb1 = emb1 / emb1.norm(2, 1, keepdim=True).expand_as(emb1) emb2 = emb2 / emb2.norm(2, 1, keepdim=True).expand_as(emb2) all_pairs, all_scores = generate_new_dictionary_bidirectional( emb2, emb1) add_size = params.induced_dico_c * (i + 1) trainer.dico_BA = torch.cat( (trainer.dico_BA_original, all_pairs[:add_size].cuda()), 0) if i == 0: logger.info( "After first iteration train dictionary size: {}".format( trainer.dico_BA.shape[0])) logger.info("Final iteration train dictionary size: {}".format( trainer.dico_BA.shape[0])) trainer.set_eval() precision_at_1 = get_word_translation_accuracy( params, trainer.mapping_F(trainer.encoder_B( trainer.tgt_emb.weight.data).data).data, trainer.encoder_A(trainer.src_emb.weight.data).data, src2tgt=False) if params.save_model_weights: save_model_weights(params, trainer, src2tgt=False)
type=bool_flag, default=False, help="print predictions") # parse parameters params = parser.parse_args() # check parameters assert params.src_lang, "source language undefined" assert os.path.isfile(params.src_emb) assert not params.tgt_lang or os.path.isfile(params.tgt_emb) assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) # build logger / model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, _ = build_model(params, False) trainer = Trainer(src_emb, tgt_emb, mapping, None, params) evaluator = Evaluator(trainer) # run evaluations to_log = OrderedDict({'n_iter': 0}) evaluator.monolingual_wordsim(to_log) # evaluator.monolingual_wordanalogy(to_log) if params.tgt_lang: evaluator.crosslingual_wordsim(to_log) evaluator.word_translation(to_log, print_trans=params.print, result_path=trainer.params.exp_path) evaluator.sent_translation(to_log) # evaluator.dist_mean_cosine(to_log)
assert params.dico_train in [ "identical_char", "default", "identical_num", "MAT" ] or os.path.isfile(params.dico_train) assert params.dico_build in ["S2T", "T2S", "S2T|T2S", "S2T&T2S"] assert params.dico_max_size == 0 or params.dico_max_size < params.dico_max_rank assert params.dico_max_size == 0 or params.dico_max_size > params.dico_min_size assert all(os.path.isfile(emb) for emb in params.embs) assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) assert params.export in ["", "txt", "pth"] assert len(params.langs) == len(params.embs) # build logger / model / trainer / evaluator logger = initialize_exp(params) embs, mapping = build_model(params) trainer = Trainer(embs, mapping, params) evaluator = Evaluator(trainer) final_results = { '{}-{}'.format(src, tgt): 0 for src, tgt in permutations(params.langs, 2) } # load a training dictionary. if a dictionary path is not provided, use a default # one ("default") or create one based on identical character strings ("identical_char") trainer.load_multi_pairwise_training_dico(params.dico_train, params.dicts_train_path) """ Learning loop for Multilingual Pairwise Procrustes Analysis (MPPA) """ for n_iter in range(params.n_refinement): logger.info('Starting iteration %i...' % n_iter)
assert not params.cuda or torch.cuda.is_available() assert 0 <= params.dis_dropout < 1 assert 0 <= params.dis_input_dropout < 1 assert 0 <= params.dis_smooth < 0.5 assert params.dis_lambda > 0 and params.dis_steps > 0 assert 0 < params.lr_shrink <= 1 assert os.path.isfile(params.src_emb) assert os.path.isfile(params.tgt_emb) assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) assert params.export in ["", "txt", "pth"] # build model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, discriminator = build_model(params, True) trainer = Trainer(src_emb, tgt_emb, mapping, discriminator, params) evaluator = Evaluator(trainer) compute_candidates_for_method_similarity(params.src_dico.word2id, params.tgt_dico.word2id, params, "dict/candidates_dict.txt") logger.info("Num of synthetic class-method dictionary : " + str(len(params.src_candidate_indices))) """ Learning loop for Adversarial Training """ if params.adversarial: logger.info('----> ADVERSARIAL TRAINING <----\n\n') # training loop for n_epoch in range(params.n_epochs):
# check parameters assert not params.cuda or torch.cuda.is_available() assert 0 <= params.dis_dropout < 1 assert 0 <= params.dis_input_dropout < 1 assert 0 <= params.dis_smooth < 0.5 assert params.dis_lambda > 0 and params.dis_steps > 0 assert 0 < params.lr_shrink <= 1 assert os.path.isfile(params.src_emb) assert os.path.isfile(params.tgt_emb) assert params.export in ["", "txt", "pth"] # build model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, discriminator = build_model(params, True) trainer = Trainer(src_emb, tgt_emb, mapping, discriminator, params) evaluator = Evaluator(trainer) """ Learning loop for Adversarial Training """ if params.adversarial: logger.info('----> ADVERSARIAL TRAINING <----\n\n') # training loop for n_epoch in range(params.n_epochs): logger.info('Starting adversarial training epoch %i...' % n_epoch) tic = time.time() n_words_proc = 0 stats = {'DIS_COSTS': []}
parser.add_argument("--use_earlystop", action="store_true", help="Early stop") # parse parameters params = parser.parse_args() # check parameters assert params.src_lang, "source language undefined" assert os.path.isfile(params.src_emb) assert not params.tgt_lang or os.path.isfile(params.tgt_emb) assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) # build logger / model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, _ = build_model(params, False) trainer = Trainer(src_emb, tgt_emb, mapping, None, params) evaluator = Evaluator(trainer) # run evaluations to_log = OrderedDict({'n_iter': 0}) evaluator.monolingual_wordsim(to_log) # evaluator.monolingual_wordanalogy(to_log) if params.tgt_lang: evaluator.crosslingual_wordsim(to_log) examples = evaluator.word_translation(to_log, use_csls=False, output_results=True) for method, lists in examples.items(): for k, values in zip([1, 5, 10], lists): with open( os.path.join(params.exp_path, '{}_at_{}.tsv'.format(method, k)), 'w') as f:
help="Embedding dimension") parser.add_argument("--normalize_embeddings", type=str, default="", help="Normalize embeddings before training") # parse parameters params = parser.parse_args() # check parameters assert params.src_lang, "source language undefined" assert os.path.isfile(params.src_emb) assert not params.tgt_lang or os.path.isfile(params.tgt_emb) assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) # build logger / model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, _ = build_model(params, False) trainer = Trainer(src_emb, tgt_emb, mapping, None, params) evaluator = Evaluator(trainer) # run evaluations to_log = OrderedDict({'n_iter': 0}) evaluator.monolingual_wordsim(to_log) # evaluator.monolingual_wordanalogy(to_log) if params.tgt_lang: evaluator.crosslingual_wordsim(to_log) evaluator.word_translation(to_log) evaluator.sent_translation(to_log) # evaluator.dist_mean_cosine(to_log)
assert not params.cuda or torch.cuda.is_available() assert params.dico_train in ["identical_char", "default"] or os.path.isfile( params.dico_train) assert params.dico_build in ["S2T", "T2S", "S2T|T2S", "S2T&T2S"] assert params.dico_max_size == 0 or params.dico_max_size < params.dico_max_rank assert params.dico_max_size == 0 or params.dico_max_size > params.dico_min_size assert os.path.isfile(params.src_emb) assert os.path.isfile(params.tgt_emb) assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval) assert params.export in ["", "txt", "pth"] # build logger / model / trainer / evaluator logger = initialize_exp(params) src_emb, tgt_emb, mapping, _ = build_model(params, False) trainer = Trainer(src_emb, tgt_emb, mapping, None, params) evaluator = Evaluator(trainer) # load a training dictionary. if a dictionary path is not provided, use a default # one ("default") or create one based on identical character strings ("identical_char") trainer.load_training_dico(params.dico_train) # define the validation metric VALIDATION_METRIC = VALIDATION_METRIC_UNSUP if params.dico_train == 'identical_char' else VALIDATION_METRIC_SUP logger.info("Validation metric: %s" % VALIDATION_METRIC) """ Learning loop for Procrustes Iterative Learning """ for n_iter in range(params.n_refinement + 1): logger.info('Starting iteration %i...' % n_iter)