parser.add_argument( "--disable-skip", default=False, action='store_true', help="Disable skipping of samples with high argmax BLEU") args = parser.parse_args() device = torch.device("cuda" if args.cuda else "cpu") saves_path = os.path.join(SAVES_DIR, args.name) os.makedirs(saves_path, exist_ok=True) phrase_pairs, emb_dict = \ data.load_data(genre_filter=args.data) log.info("Obtained %d phrase pairs with %d uniq words", len(phrase_pairs), len(emb_dict)) data.save_emb_dict(saves_path, emb_dict) end_token = emb_dict[data.END_TOKEN] train_data = data.encode_phrase_pairs(phrase_pairs, emb_dict) rand = np.random.RandomState(data.SHUFFLE_SEED) rand.shuffle(train_data) train_data, test_data = data.split_train_test(train_data) log.info("Training data converted, got %d samples", len(train_data)) train_data = data.group_train_data(train_data) test_data = data.group_train_data(test_data) log.info("Train set has %d phrases, test %d", len(train_data), len(test_data)) rev_emb_dict = {idx: word for word, idx in emb_dict.items()} net = model.PhraseModel(emb_size=model.EMBEDDING_DIM, dict_size=len(emb_dict),
parser = argparse.ArgumentParser() parser.add_argument("--data", required=True, help="Category to use for training. " "Empty string to train on full dataset") parser.add_argument("--cuda", action='store_true', default=False, help="Enable cuda") parser.add_argument("-n", "--name", required=True, help="Name of the run") args = parser.parse_args() device = torch.device("cuda" if args.cuda else "cpu") saves_path = os.path.join(SAVES_DIR, args.name) os.makedirs(saves_path, exist_ok=True) phrase_pairs, emb_dict = data.load_data(genre_filter=args.data) log.info("Obtained %d phrase pairs with %d uniq words", len(phrase_pairs), len(emb_dict)) data.save_emb_dict(saves_path, emb_dict) end_token = emb_dict[data.END_TOKEN] train_data = data.encode_phrase_pairs(phrase_pairs, emb_dict) rand = np.random.RandomState(data.SHUFFLE_SEED) rand.shuffle(train_data) log.info("Training data converted, got %d samples", len(train_data)) train_data, test_data = data.split_train_test(train_data) log.info("Train set has %d phrases, test %d", len(train_data), len(test_data)) net = model.PhraseModel(emb_size=model.EMBEDDING_DIM, dict_size=len(emb_dict), hid_size=model.HIDDEN_STATE_SIZE).to(device) log.info("Model: %s", net) writer = SummaryWriter(comment="-" + args.name) optimiser = optim.Adam(net.parameters(), lr=LEARNING_RATE)