def run_test():
V = 11
criterion = utils.LabelSmoothing(size=V, padding_idx=0, smoothing=0.0)
model = make_model(V, V, N=2, d_model=256, d_ff=256, dropout=.0)
model_opt = NoamOpt(
model.src_embed[0].d_model, 1, 400,
torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98),
eps=1e-9))
for epoch in range(10):
model.train()
run_epoch(data_gen(V, 30, 20), model,
SimpleLossCompute(model.generator, criterion, model_opt))
model.eval()
print(
run_epoch(data_gen(V, 30, 5), model,
SimpleLossCompute(model.generator, criterion, None)))
for line in fin:
name, id = line.strip().split("\t")
ents.append(name)
if (method == "train"):
filename = sys.argv[2]
is_char_level = sys.argv[3] == 'char'
if is_char_level:
# TODO
dataloader = DataLoader_char(filename, chunk_len, device)
V = dataloader.vocabularyLoader.n_chars # vocabolary size
criterion = LabelSmoothing(size=V, padding_idx=0, smoothing=0.0)
criterion.cuda()
model = make_model(V, V, N=transformer_size)
model.cuda()
model_opt = NoamOpt(model.src_embed[0].d_model, 1, 400,
torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9))
for epoch in range(epoch_number):
if epoch % epoches_of_loss_record == 0:
f = open("procedure.txt", "a+")
f.write("step:%d \n" % epoch)
f.close()
print("step: ", epoch)
model.train()
run_epoch("train", data_gen_char(dataloader, batch, nbatches), model,
SimpleLossCompute(model.generator, criterion, model_opt), nbatches, epoch)
model.eval()
run_epoch("test ", data_gen_char(dataloader, batch, 1), model,
ax = fig.add_subplot(rows, cols, i * rows + j + 1)
ax.grid('off')
ax.axis('off')
ax.imshow(X_train[random_index, :])
ax.set_title(y_train[random_index])
plt.show()
from keras.utils import to_categorical
import keras
from keras import backend as K
y_train2 = to_categorical(y_train)
y_test2 = to_categorical(y_test)
y_validate2 = to_categorical(y_validate)
INIT_LR = 1e-2
BATCH_SIZE = 32
EPOCHS = 10
model = make_model()
model.compile(loss='categorical_crossentropy',
optimizer=keras.optimizers.adamax(lr=INIT_LR),
metrics=['accuracy'])
def lr_scheduler(epoch):
return INIT_LR * 0.9**epoch
class LrHistory(keras.callbacks.Callback):
def on_epoch_begin(self, epoch, logs={}):
print("Learning rate:", K.get_value(model.optimizer.lr))
from keras.callbacks import TensorBoard
def create_models(bracket, game_stats, target, exclude_list):
team_models={}
for team in bracket['Team']:
team_models[team]=make_model(team, game_stats, target, exclude_list)
return team_models
def main():
args = parse_args()
SRC_DIR = args.SRC_DIR
MODEL_DIR = args.MODEL_DIR
DATA = args.DATA
EN_EMB = args.EN_EMB
DE_EMB = args.DE_EMB
SEQ_TRAIN = True if DE_EMB == 'elmo' else False
# TODO currently hidden size is fixed, should be able to adjust
# based on src and trg embeddings respectively
# EMB_DIM should be multiple of h (default 8), look at MultiHeadedAttention
if 'glove' in EN_EMB:
EMB_DIM = 200
elif 'elmo' in EN_EMB:
EMB_DIM = 1024
else:
EMB_DIM = 512
BATCH_SIZE = 1500
EPOCHES = 100
options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json"
weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5"
vocab_file = os.path.join(MODEL_DIR, '%s.vocab' % (DATA))
model_file = os.path.join(
MODEL_DIR, '%s.%s.%s.transformer.pt' % (DATA, EN_EMB, DE_EMB))
if not os.path.exists(MODEL_DIR):
os.makedirs(MODEL_DIR)
# GPU to use
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# device = ("cpu")
# devices = [0, 1, 2, 3]
#####################
# Data Loading #
#####################
BOS_WORD = '<s>'
EOS_WORD = '</s>'
BLANK_WORD = "<blank>"
MIN_FREQ = 2
spacy_en = spacy.load('en')
def tokenize_en(text):
return [tok.text for tok in spacy_en.tokenizer(text)]
TEXT = data.Field(tokenize=tokenize_en,
init_token=BOS_WORD,
eos_token=EOS_WORD,
pad_token=BLANK_WORD)
train = datasets.TranslationDataset(path=os.path.join(SRC_DIR, DATA),
exts=('.train.src', '.train.trg'),
fields=(TEXT, TEXT))
val = datasets.TranslationDataset(path=os.path.join(SRC_DIR, DATA),
exts=('.val.src', '.val.trg'),
fields=(TEXT, TEXT))
train_iter = MyIterator(train,
batch_size=BATCH_SIZE,
device=device,
repeat=False,
sort_key=lambda x: (len(x.src), len(x.trg)),
batch_size_fn=batch_size_fn,
train=True)
valid_iter = MyIterator(val,
batch_size=BATCH_SIZE,
device=device,
repeat=False,
sort_key=lambda x: (len(x.src), len(x.trg)),
batch_size_fn=batch_size_fn,
train=False)
random_idx = random.randint(0, len(train) - 1)
print(train[random_idx].src)
print(train[random_idx].trg)
###############
# Vocabuary #
###############
if os.path.exists(vocab_file):
TEXT.vocab = torch.load(vocab_file)
else:
print("Save %s vocabuary..." % (DATA), end='\t')
TEXT.build_vocab(train.src, min_freq=MIN_FREQ, vectors='glove.6B.200d')
print("vocab size = %d" % (len(TEXT.vocab)))
torch.save(TEXT.vocab, vocab_file)
pad_idx = TEXT.vocab.stoi["<blank>"]
#####################
# Word Embedding #
#####################
encoder_emb, decoder_emb = get_emb(EN_EMB,
DE_EMB,
TEXT.vocab,
device,
d_model=EMB_DIM,
elmo_options=options_file,
elmo_weights=weight_file)
##########################
# Training the System #
##########################
model = make_model(len(TEXT.vocab),
encoder_emb,
decoder_emb,
d_model=EMB_DIM).to(device)
if os.path.exists(model_file):
print("Restart from last checkpoint...")
model.load_state_dict(torch.load(model_file))
criterion = LabelSmoothing(size=len(TEXT.vocab),
padding_idx=pad_idx,
smoothing=0.1).to(device)
model_opt = NoamOpt(
EMB_DIM, 1, 2000,
torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98),
eps=1e-9))
# calculate parameters
total_params = sum(p.numel() for p in model.parameters()) // 1000000
trainable_params = sum(
p.numel() for p in model.parameters() if p.requires_grad) // 1000000
rate = trainable_params / total_params
print("Model parameters trainable (%d M) / total (%d M) = %f" %
(trainable_params, total_params, rate))
print("Training %s %s %s..." % (DATA, EN_EMB, DE_EMB))
### SINGLE GPU
for epoch in range(EPOCHES):
model.train()
loss_compute = SimpleLossCompute(model.generator,
criterion,
opt=model_opt)
run_epoch((rebatch(pad_idx, b) for b in train_iter),
model,
loss_compute,
TEXT.vocab,
seq_train=SEQ_TRAIN)
model.eval()
total_loss, total_tokens = 0, 0
for batch in (rebatch(pad_idx, b) for b in valid_iter):
out = greedy_decode(model,
TEXT.vocab,
batch.src,
batch.src_mask,
trg=batch.trg)
loss = loss_compute(out, batch.trg_y, batch.ntokens)
total_loss += loss
total_tokens += batch.ntokens
print("Save model...")
torch.save(model.state_dict(), model_file)
print("Epoch %d/%d - Loss: %f" %
(epoch + 1, EPOCHES, total_loss / total_tokens))
bBoxForSequences = np.array(
np.load(
'/home/hrishi/1Hrishi/0Thesis/Human-Pose-Prediction-GAN/codebase/Normalizing_Test_BBOX_Params.npy'
))
print(bBoxForSequences.shape)
print(sequences.shape)
input_dim = sequences.shape[-1]
output_dim = input_dim
PATH = '/media/hrishi/OS/1Hrishi/1Cheese/0Thesis/checkpoints/BIG_WU2900_Transformer_loss_0.00044433000110201377.ckpt'
model = make_model(src_vocab=input_dim,
tgt_vocab=output_dim,
N=6,
d_model=1024,
d_ff=4096,
h=16,
dropout=0.3)
model.load_state_dict(torch.load(PATH, map_location='cpu'))
model.eval()
model.cuda()
batch_size = 5
batch_index = 0
seq_length = 16
# for batch_index in range(0, train_sequences.shape[0], batch_size):
X_test = torch.from_numpy(sequences[batch_index:batch_index +
batch_size, :-1, :]).float().to(device)
# X_test = X_test.view((-1, seq_length - 1, input_dim))
"/media/hrishi/OS/1Hrishi/1Cheese/0Thesis/Data/Penn_Action/preprocessed/frames/",
data=frame_sequences,
batch_size=batch_size,
shuffle=False,
random_state=random_state,
transform=transforms.Compose([ToTensor()]))
dataloader = DataLoader(train_loader,
batch_size=batch_size,
shuffle=True,
num_workers=1)
model = make_model(src_vocab=input_dim,
tgt_vocab=output_dim,
N=2,
d_model=512,
d_ff=2048,
h=8,
dropout=0.1)
model.train()
model.float().cuda()
model_opt = NoamOpt(
model.src_embed[0].d_model, 1, 3000,
torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9))
mse_loss_fn = nn.MSELoss()
n_epochs = 2
train_losses = []
def main():
args = parse_args()
SRC_DIR = args.SRC_DIR
MODEL_DIR = args.MODEL_DIR
EVAL_DIR = args.EVAL_DIR
DATA = args.DATA
EN_EMB = args.EN_EMB
DE_EMB = args.DE_EMB
if 'glove' in EN_EMB:
EMB_DIM = 200
elif 'elmo' in EN_EMB:
EMB_DIM = 1024
else:
EMB_DIM = 512
BATCH_SIZE = 30
options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json"
weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5"
vocab_file = os.path.join(MODEL_DIR, '%s.vocab' % (DATA))
model_file = os.path.join(MODEL_DIR, '%s.%s.%s.transformer.pt' % (DATA, EN_EMB, DE_EMB))
if not os.path.exists(EVAL_DIR):
os.makedirs(EVAL_DIR)
# GPU to use
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# device = ("cpu")
#####################
# Data Loading #
#####################
BOS_WORD = '<s>'
EOS_WORD = '</s>'
BLANK_WORD = "<blank>"
MIN_FREQ = 2
spacy_en = spacy.load('en')
def tokenize_en(text):
return [tok.text for tok in spacy_en.tokenizer(text)]
TEXT = data.Field(tokenize=tokenize_en, init_token = BOS_WORD,
eos_token = EOS_WORD, pad_token=BLANK_WORD)
test = datasets.TranslationDataset(path=os.path.join(SRC_DIR, DATA),
exts=('.test.src', '.test.trg'), fields=(TEXT, TEXT))
# use the same order as original data
test_iter = data.Iterator(test, batch_size=BATCH_SIZE, device=device,
sort=False, repeat=False, train=False)
random_idx = random.randint(0, len(test) - 1)
print(test[random_idx].src)
print(test[random_idx].trg)
###############
# Vocabuary #
###############
TEXT.vocab = torch.load(vocab_file)
pad_idx = TEXT.vocab.stoi["<blank>"]
print("Load %s vocabuary; vocab size = %d" % (DATA, len(TEXT.vocab)))
#####################
# Word Embedding #
#####################
encoder_emb, decoder_emb = get_emb(EN_EMB, DE_EMB, TEXT.vocab, device,
d_model=EMB_DIM,
elmo_options=options_file,
elmo_weights=weight_file)
##########################
# Translation #
##########################
model = make_model(len(TEXT.vocab), encoder_emb, decoder_emb,
d_model=EMB_DIM).to(device)
model.load_state_dict(torch.load(model_file))
model.eval()
print("Predicting %s %s %s ..." % (DATA, EN_EMB, DE_EMB))
src, trg, pred = [], [], []
for batch in (rebatch(pad_idx, b) for b in test_iter):
out = greedy_decode(model, TEXT.vocab, batch.src, batch.src_mask)
# print("SRC OUT: ", src.shape, out.shape)
probs = model.generator(out)
_, prediction = torch.max(probs, dim = -1)
source = [[TEXT.vocab.itos[word] for word in words[1:]] for words in batch.src]
target = [[TEXT.vocab.itos[word] for word in words[1:]] for words in batch.trg]
translation = [[TEXT.vocab.itos[word] for word in words] for words in prediction]
for i in range(len(translation)):
src.append(' '.join(source[i]).split('</s>')[0])
trg.append(' '.join(target[i]).split('</s>')[0])
pred.append(' '.join(translation[i]).split('</s>')[0])
# eliminate data with unkonwn words in src trg
if '<unk>' in src[-1] or '<unk>' in trg[-1]:
continue
print("Source:", src[-1])
print("Target:", trg[-1])
print("Translation:", pred[-1])
print()
prefix = os.path.join(EVAL_DIR, '%s.%s.%s.eval' % (DATA, EN_EMB, DE_EMB))
for sentences, ext in zip([src, trg, pred], ['.src', '.trg', '.pred']):
with open(prefix + ext, 'w+') as f:
f.write('\n'.join(sentences))
