def main():
lstm_size = 1024
lstm_layers = 3
with monit.section("Loading data"):
files = parser.load.load_files()
train_files, valid_files = parser.load.split_train_valid(
files, is_shuffle=False)
with monit.section("Create model"):
model = SimpleLstmModel(encoding_size=tokenizer.VOCAB_SIZE,
embedding_size=tokenizer.VOCAB_SIZE,
lstm_size=lstm_size,
lstm_layers=lstm_layers)
model.to(device)
experiment.add_pytorch_models({'base': model})
experiment.load("94ab8470e6a711ea9703c1dbf199539e", 5654)
# For debugging with a specific piece of source code
# predictor = Predictor(model, lstm_layers, lstm_size)
# for s in ['""" """\n', "from __future__"]:
# predictor.add(s)
# s = predictor.get_suggestion()
# Evaluate all the files in validation set
for file in valid_files:
logger.log(str(file.path), Text.heading)
evaluator = Evaluator(model,
file,
lstm_layers,
lstm_size,
skip_spaces=True)
evaluator.eval()
def main():
lstm_size = 1024
lstm_layers = 3
with logger.section("Loading data"):
files = parser.load.load_files()
train_files, valid_files = parser.load.split_train_valid(files, is_shuffle=False)
with logger.section("Create model"):
model = SimpleLstmModel(encoding_size=tokenizer.VOCAB_SIZE,
embedding_size=tokenizer.VOCAB_SIZE,
lstm_size=lstm_size,
lstm_layers=lstm_layers)
model.to(device)
EXPERIMENT.add_models({'base': model})
EXPERIMENT.start_replay()
# For debugging with a specific piece of source code
# predictor = Predictor(model, lstm_layers, lstm_size)
# for s in ['""" """\n', "from __future__"]:
# predictor.add(s)
# s = predictor.get_suggestion()
# Evaluate all the files in validation set
for file in valid_files:
logger.log(str(file.path), color=colors.BrightColor.orange)
evaluator = Evaluator(model, file,
lstm_layers, lstm_size,
skip_spaces=True)
keys_saved = evaluator.eval()
logger.info(keys_saved=keys_saved)
def main_train():
lstm_size = 1024
lstm_layers = 3
batch_size = 32
seq_len = 32
is_half = False
with logger.section("Create model"):
# Create model
model = SimpleLstmModel(encoding_size=tokenizer.VOCAB_SIZE,
embedding_size=tokenizer.VOCAB_SIZE,
lstm_size=lstm_size,
lstm_layers=lstm_layers)
# Use half precision
if is_half:
model.half()
# Move model to `device`
model.to(device)
# Create loss function and optimizer
loss_func = torch.nn.CrossEntropyLoss()
if is_half:
optimizer = torch.optim.Adam(model.parameters(), eps=1e-5)
else:
optimizer = torch.optim.Adam(model.parameters())
# Initial state is 0
if is_half:
dtype = torch.float16
else:
dtype = torch.float32
h0 = torch.zeros((lstm_layers, batch_size, lstm_size),
device=device,
dtype=dtype)
c0 = torch.zeros((lstm_layers, batch_size, lstm_size),
device=device,
dtype=dtype)
# Specify the model in [lab](https://github.com/vpj/lab) for saving and loading
EXPERIMENT.add_models({'base': model})
# Start training scratch (step '0')
EXPERIMENT.start_train(True)
# Setup logger indicators
logger.add_indicator("train_loss", queue_limit=500, is_histogram=True)
logger.add_indicator("valid_loss", queue_limit=500, is_histogram=True)
for epoch in range(100):
if not run_epoch(epoch, model, loss_func, optimizer, seq_len,
batch_size, h0, c0):
break
def main_train():
lstm_size = 1024
lstm_layers = 3
batch_size = 32
seq_len = 32
with logger.section("Loading data"):
# Load all python files
files = parser.load.load_files()
# Split training and validation data
train_files, valid_files = parser.load.split_train_valid(files, is_shuffle=False)
with logger.section("Create model"):
# Create model
model = SimpleLstmModel(encoding_size=tokenizer.VOCAB_SIZE,
embedding_size=tokenizer.VOCAB_SIZE,
lstm_size=lstm_size,
lstm_layers=lstm_layers)
# Move model to `device`
model.to(device)
# Create loss function and optimizer
loss_func = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters())
# Initial state is 0
h0 = torch.zeros((lstm_layers, batch_size, lstm_size), device=device)
c0 = torch.zeros((lstm_layers, batch_size, lstm_size), device=device)
# Setup logger indicators
logger.add_indicator("train_loss", queue_limit=500, is_histogram=True)
logger.add_indicator("valid_loss", queue_limit=500, is_histogram=True)
# Specify the model in [lab](https://github.com/vpj/lab) for saving and loading
EXPERIMENT.add_models({'base': model})
# Start training scratch (step '0')
EXPERIMENT.start_train(0)
# Number of batches per epoch
batches = math.ceil(sum([len(f[1]) + 1 for f in train_files]) / (batch_size * seq_len))
# Number of steps per epoch. We train and validate on each step.
steps_per_epoch = 200
# Train for 100 epochs
for epoch in logger.loop(range(100)):
# Create trainer
trainer = Trainer(files=train_files,
model=model,
loss_func=loss_func,
optimizer=optimizer,
batch_size=batch_size,
seq_len=seq_len,
is_train=True,
h0=h0,
c0=c0,
eof=0)
# Create validator
validator = Trainer(files=train_files,
model=model,
loss_func=loss_func,
optimizer=optimizer,
is_train=False,
seq_len=seq_len,
batch_size=batch_size,
h0=h0,
c0=c0,
eof=0)
# Next batch to train and validation
train_batch = 0
valid_batch = 0
# Loop through steps
for i in range(1, steps_per_epoch):
# Set global step
global_step = epoch * batches + min(batches, (batches * i) // steps_per_epoch)
logger.set_global_step(global_step)
# Last batch to train and validate
train_batch_limit = trainer.x.shape[0] * min(1., (i + 1) / steps_per_epoch)
valid_batch_limit = validator.x.shape[0] * min(1., (i + 1) / steps_per_epoch)
try:
with logger.delayed_keyboard_interrupt():
with logger.section("train", total_steps=trainer.x.shape[0], is_partial=True):
model.train()
# Train
while train_batch < train_batch_limit:
trainer.run(train_batch)
logger.progress(train_batch + 1)
train_batch += 1
with logger.section("valid", total_steps=validator.x.shape[0], is_partial=True):
model.eval()
# Validate
while valid_batch < valid_batch_limit:
validator.run(valid_batch)
logger.progress(valid_batch + 1)
valid_batch += 1
# Output results
logger.write()
# 10 lines of logs per epoch
if (i + 1) % (steps_per_epoch // 10) == 0:
logger.new_line()
except KeyboardInterrupt:
logger.save_progress()
logger.save_checkpoint()
logger.new_line()
return