def run_test(): V = 11 criterion = utils.LabelSmoothing(size=V, padding_idx=0, smoothing=0.0) model = make_model(V, V, N=2, d_model=256, d_ff=256, dropout=.0) model_opt = NoamOpt( model.src_embed[0].d_model, 1, 400, torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9)) for epoch in range(10): model.train() run_epoch(data_gen(V, 30, 20), model, SimpleLossCompute(model.generator, criterion, model_opt)) model.eval() print( run_epoch(data_gen(V, 30, 5), model, SimpleLossCompute(model.generator, criterion, None)))
for line in fin: name, id = line.strip().split("\t") ents.append(name) if (method == "train"): filename = sys.argv[2] is_char_level = sys.argv[3] == 'char' if is_char_level: # TODO dataloader = DataLoader_char(filename, chunk_len, device) V = dataloader.vocabularyLoader.n_chars # vocabolary size criterion = LabelSmoothing(size=V, padding_idx=0, smoothing=0.0) criterion.cuda() model = make_model(V, V, N=transformer_size) model.cuda() model_opt = NoamOpt(model.src_embed[0].d_model, 1, 400, torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9)) for epoch in range(epoch_number): if epoch % epoches_of_loss_record == 0: f = open("procedure.txt", "a+") f.write("step:%d \n" % epoch) f.close() print("step: ", epoch) model.train() run_epoch("train", data_gen_char(dataloader, batch, nbatches), model, SimpleLossCompute(model.generator, criterion, model_opt), nbatches, epoch) model.eval() run_epoch("test ", data_gen_char(dataloader, batch, 1), model,
ax = fig.add_subplot(rows, cols, i * rows + j + 1) ax.grid('off') ax.axis('off') ax.imshow(X_train[random_index, :]) ax.set_title(y_train[random_index]) plt.show() from keras.utils import to_categorical import keras from keras import backend as K y_train2 = to_categorical(y_train) y_test2 = to_categorical(y_test) y_validate2 = to_categorical(y_validate) INIT_LR = 1e-2 BATCH_SIZE = 32 EPOCHS = 10 model = make_model() model.compile(loss='categorical_crossentropy', optimizer=keras.optimizers.adamax(lr=INIT_LR), metrics=['accuracy']) def lr_scheduler(epoch): return INIT_LR * 0.9**epoch class LrHistory(keras.callbacks.Callback): def on_epoch_begin(self, epoch, logs={}): print("Learning rate:", K.get_value(model.optimizer.lr)) from keras.callbacks import TensorBoard
def create_models(bracket, game_stats, target, exclude_list): team_models={} for team in bracket['Team']: team_models[team]=make_model(team, game_stats, target, exclude_list) return team_models
def main(): args = parse_args() SRC_DIR = args.SRC_DIR MODEL_DIR = args.MODEL_DIR DATA = args.DATA EN_EMB = args.EN_EMB DE_EMB = args.DE_EMB SEQ_TRAIN = True if DE_EMB == 'elmo' else False # TODO currently hidden size is fixed, should be able to adjust # based on src and trg embeddings respectively # EMB_DIM should be multiple of h (default 8), look at MultiHeadedAttention if 'glove' in EN_EMB: EMB_DIM = 200 elif 'elmo' in EN_EMB: EMB_DIM = 1024 else: EMB_DIM = 512 BATCH_SIZE = 1500 EPOCHES = 100 options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json" weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5" vocab_file = os.path.join(MODEL_DIR, '%s.vocab' % (DATA)) model_file = os.path.join( MODEL_DIR, '%s.%s.%s.transformer.pt' % (DATA, EN_EMB, DE_EMB)) if not os.path.exists(MODEL_DIR): os.makedirs(MODEL_DIR) # GPU to use device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # device = ("cpu") # devices = [0, 1, 2, 3] ##################### # Data Loading # ##################### BOS_WORD = '<s>' EOS_WORD = '</s>' BLANK_WORD = "<blank>" MIN_FREQ = 2 spacy_en = spacy.load('en') def tokenize_en(text): return [tok.text for tok in spacy_en.tokenizer(text)] TEXT = data.Field(tokenize=tokenize_en, init_token=BOS_WORD, eos_token=EOS_WORD, pad_token=BLANK_WORD) train = datasets.TranslationDataset(path=os.path.join(SRC_DIR, DATA), exts=('.train.src', '.train.trg'), fields=(TEXT, TEXT)) val = datasets.TranslationDataset(path=os.path.join(SRC_DIR, DATA), exts=('.val.src', '.val.trg'), fields=(TEXT, TEXT)) train_iter = MyIterator(train, batch_size=BATCH_SIZE, device=device, repeat=False, sort_key=lambda x: (len(x.src), len(x.trg)), batch_size_fn=batch_size_fn, train=True) valid_iter = MyIterator(val, batch_size=BATCH_SIZE, device=device, repeat=False, sort_key=lambda x: (len(x.src), len(x.trg)), batch_size_fn=batch_size_fn, train=False) random_idx = random.randint(0, len(train) - 1) print(train[random_idx].src) print(train[random_idx].trg) ############### # Vocabuary # ############### if os.path.exists(vocab_file): TEXT.vocab = torch.load(vocab_file) else: print("Save %s vocabuary..." % (DATA), end='\t') TEXT.build_vocab(train.src, min_freq=MIN_FREQ, vectors='glove.6B.200d') print("vocab size = %d" % (len(TEXT.vocab))) torch.save(TEXT.vocab, vocab_file) pad_idx = TEXT.vocab.stoi["<blank>"] ##################### # Word Embedding # ##################### encoder_emb, decoder_emb = get_emb(EN_EMB, DE_EMB, TEXT.vocab, device, d_model=EMB_DIM, elmo_options=options_file, elmo_weights=weight_file) ########################## # Training the System # ########################## model = make_model(len(TEXT.vocab), encoder_emb, decoder_emb, d_model=EMB_DIM).to(device) if os.path.exists(model_file): print("Restart from last checkpoint...") model.load_state_dict(torch.load(model_file)) criterion = LabelSmoothing(size=len(TEXT.vocab), padding_idx=pad_idx, smoothing=0.1).to(device) model_opt = NoamOpt( EMB_DIM, 1, 2000, torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9)) # calculate parameters total_params = sum(p.numel() for p in model.parameters()) // 1000000 trainable_params = sum( p.numel() for p in model.parameters() if p.requires_grad) // 1000000 rate = trainable_params / total_params print("Model parameters trainable (%d M) / total (%d M) = %f" % (trainable_params, total_params, rate)) print("Training %s %s %s..." % (DATA, EN_EMB, DE_EMB)) ### SINGLE GPU for epoch in range(EPOCHES): model.train() loss_compute = SimpleLossCompute(model.generator, criterion, opt=model_opt) run_epoch((rebatch(pad_idx, b) for b in train_iter), model, loss_compute, TEXT.vocab, seq_train=SEQ_TRAIN) model.eval() total_loss, total_tokens = 0, 0 for batch in (rebatch(pad_idx, b) for b in valid_iter): out = greedy_decode(model, TEXT.vocab, batch.src, batch.src_mask, trg=batch.trg) loss = loss_compute(out, batch.trg_y, batch.ntokens) total_loss += loss total_tokens += batch.ntokens print("Save model...") torch.save(model.state_dict(), model_file) print("Epoch %d/%d - Loss: %f" % (epoch + 1, EPOCHES, total_loss / total_tokens))
bBoxForSequences = np.array( np.load( '/home/hrishi/1Hrishi/0Thesis/Human-Pose-Prediction-GAN/codebase/Normalizing_Test_BBOX_Params.npy' )) print(bBoxForSequences.shape) print(sequences.shape) input_dim = sequences.shape[-1] output_dim = input_dim PATH = '/media/hrishi/OS/1Hrishi/1Cheese/0Thesis/checkpoints/BIG_WU2900_Transformer_loss_0.00044433000110201377.ckpt' model = make_model(src_vocab=input_dim, tgt_vocab=output_dim, N=6, d_model=1024, d_ff=4096, h=16, dropout=0.3) model.load_state_dict(torch.load(PATH, map_location='cpu')) model.eval() model.cuda() batch_size = 5 batch_index = 0 seq_length = 16 # for batch_index in range(0, train_sequences.shape[0], batch_size): X_test = torch.from_numpy(sequences[batch_index:batch_index + batch_size, :-1, :]).float().to(device) # X_test = X_test.view((-1, seq_length - 1, input_dim))
"/media/hrishi/OS/1Hrishi/1Cheese/0Thesis/Data/Penn_Action/preprocessed/frames/", data=frame_sequences, batch_size=batch_size, shuffle=False, random_state=random_state, transform=transforms.Compose([ToTensor()])) dataloader = DataLoader(train_loader, batch_size=batch_size, shuffle=True, num_workers=1) model = make_model(src_vocab=input_dim, tgt_vocab=output_dim, N=2, d_model=512, d_ff=2048, h=8, dropout=0.1) model.train() model.float().cuda() model_opt = NoamOpt( model.src_embed[0].d_model, 1, 3000, torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9)) mse_loss_fn = nn.MSELoss() n_epochs = 2 train_losses = []
def main(): args = parse_args() SRC_DIR = args.SRC_DIR MODEL_DIR = args.MODEL_DIR EVAL_DIR = args.EVAL_DIR DATA = args.DATA EN_EMB = args.EN_EMB DE_EMB = args.DE_EMB if 'glove' in EN_EMB: EMB_DIM = 200 elif 'elmo' in EN_EMB: EMB_DIM = 1024 else: EMB_DIM = 512 BATCH_SIZE = 30 options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json" weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5" vocab_file = os.path.join(MODEL_DIR, '%s.vocab' % (DATA)) model_file = os.path.join(MODEL_DIR, '%s.%s.%s.transformer.pt' % (DATA, EN_EMB, DE_EMB)) if not os.path.exists(EVAL_DIR): os.makedirs(EVAL_DIR) # GPU to use device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # device = ("cpu") ##################### # Data Loading # ##################### BOS_WORD = '<s>' EOS_WORD = '</s>' BLANK_WORD = "<blank>" MIN_FREQ = 2 spacy_en = spacy.load('en') def tokenize_en(text): return [tok.text for tok in spacy_en.tokenizer(text)] TEXT = data.Field(tokenize=tokenize_en, init_token = BOS_WORD, eos_token = EOS_WORD, pad_token=BLANK_WORD) test = datasets.TranslationDataset(path=os.path.join(SRC_DIR, DATA), exts=('.test.src', '.test.trg'), fields=(TEXT, TEXT)) # use the same order as original data test_iter = data.Iterator(test, batch_size=BATCH_SIZE, device=device, sort=False, repeat=False, train=False) random_idx = random.randint(0, len(test) - 1) print(test[random_idx].src) print(test[random_idx].trg) ############### # Vocabuary # ############### TEXT.vocab = torch.load(vocab_file) pad_idx = TEXT.vocab.stoi["<blank>"] print("Load %s vocabuary; vocab size = %d" % (DATA, len(TEXT.vocab))) ##################### # Word Embedding # ##################### encoder_emb, decoder_emb = get_emb(EN_EMB, DE_EMB, TEXT.vocab, device, d_model=EMB_DIM, elmo_options=options_file, elmo_weights=weight_file) ########################## # Translation # ########################## model = make_model(len(TEXT.vocab), encoder_emb, decoder_emb, d_model=EMB_DIM).to(device) model.load_state_dict(torch.load(model_file)) model.eval() print("Predicting %s %s %s ..." % (DATA, EN_EMB, DE_EMB)) src, trg, pred = [], [], [] for batch in (rebatch(pad_idx, b) for b in test_iter): out = greedy_decode(model, TEXT.vocab, batch.src, batch.src_mask) # print("SRC OUT: ", src.shape, out.shape) probs = model.generator(out) _, prediction = torch.max(probs, dim = -1) source = [[TEXT.vocab.itos[word] for word in words[1:]] for words in batch.src] target = [[TEXT.vocab.itos[word] for word in words[1:]] for words in batch.trg] translation = [[TEXT.vocab.itos[word] for word in words] for words in prediction] for i in range(len(translation)): src.append(' '.join(source[i]).split('</s>')[0]) trg.append(' '.join(target[i]).split('</s>')[0]) pred.append(' '.join(translation[i]).split('</s>')[0]) # eliminate data with unkonwn words in src trg if '<unk>' in src[-1] or '<unk>' in trg[-1]: continue print("Source:", src[-1]) print("Target:", trg[-1]) print("Translation:", pred[-1]) print() prefix = os.path.join(EVAL_DIR, '%s.%s.%s.eval' % (DATA, EN_EMB, DE_EMB)) for sentences, ext in zip([src, trg, pred], ['.src', '.trg', '.pred']): with open(prefix + ext, 'w+') as f: f.write('\n'.join(sentences))