]), download = True)
dataset_eval = opts.dataset(opts.data, train = False, transform = transforms.Compose([
T.Scale(256),
T.CenterCrop(base_model.input_side),
normalize
]), download = True)
adapt_sampler = lambda batch, dataset, sampler, **kwargs: type('', (torch.utils.data.Sampler, ), dict(__len__ = dataset.__len__, __iter__ = lambda _: itertools.chain.from_iterable(sampler(batch, dataset, **kwargs))))()
loader_train = torch.utils.data.DataLoader(dataset_train, sampler = adapt_sampler(opts.batch, dataset_train, opts.sampler), num_workers = opts.threads, batch_size = opts.batch, drop_last = True, pin_memory = True)
loader_eval = torch.utils.data.DataLoader(dataset_eval, shuffle = False, num_workers = opts.threads, batch_size = opts.batch, pin_memory = True)
model = opts.model(base_model, dataset_train.num_training_classes).cuda()
model_weights, model_biases, base_model_weights, base_model_biases = [[p for k, p in model.named_parameters() if p.requires_grad and ('bias' in k) == is_bias and ('base' in k) == is_base] for is_base in [False, True] for is_bias in [False, True]]
base_model_lr_mult = model.optimizer_params.pop('base_model_lr_mult', 1.0)
optimizer = model.optimizer([dict(params = base_model_weights, lr = base_model_lr_mult * model.optimizer_params['lr']), dict(params = base_model_biases, lr = base_model_lr_mult * model.optimizer_params['lr'], weight_decay = 0.0), dict(params = model_biases, weight_decay = 0.0)], **model.optimizer_params)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, **model.lr_scheduler_params)
log = open(opts.log, 'w', 0)
for epoch in range(opts.epochs):
scheduler.step()
model.train()
loss_all, norm_all = [], []
for batch_idx, batch in enumerate(loader_train if model.criterion is not None else []):
tic = time.time()
images, labels = [tensor.cuda() for tensor in batch]
loss = model.criterion(model(images), labels)
loss_all.append(float(loss))
optimizer.zero_grad()
loss.backward()
optimizer.step()
def main(): # pylint: disable=too-many-locals, too-many-statements
"""Create the RNN model and train it, outputting the text results.
Periodically:
(1) the training/evaluation set cost and accuracies are printed, and
(2) the RNN is given a random input feed to output its own
self-generated output text for our amusement.
"""
text = utils.retrieve_text(params.TEXT_FILE)
chars = set(text)
chars_size = len(chars)
dictionary, reverse_dictionary = utils.build_dataset(chars)
train_one_hots, eval_one_hots = utils.create_one_hots(text, dictionary)
x = tf.placeholder(tf.float32, [None, params.N_INPUT * chars_size])
labels = tf.placeholder(tf.float32, [None, chars_size])
logits = model.inference(x, chars_size)
cost = model.cost(logits, labels)
optimizer = model.optimizer(cost)
accuracy = model.accuracy(logits, labels)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(
params.SUMMARIES_DIR + '/train', sess.graph)
eval_writer = tf.summary.FileWriter(params.SUMMARIES_DIR + '/eval')
loss_total = 0
accuracy_total = 0
for epoch in range(params.EPOCHS):
for index in range(0,
len(train_one_hots) -
params.N_INPUT - params.BATCH_SIZE,
params.BATCH_SIZE):
input_x, output_y = utils.create_training_io(
train_one_hots, index, params.BATCH_SIZE, chars_size)
_, acc, loss, summary = sess.run(
[optimizer, accuracy, cost, merged],
feed_dict={x: input_x, labels: output_y})
step = epoch * (len(train_one_hots) - params.N_INPUT) + index
train_writer.add_summary(summary, step)
loss_total += loss
accuracy_total += acc
if index % params.TRAINING_DISPLAY_STEP == 0 and index:
print(
'Epoch: {} Training Step: {}\n'
'Training Set: Loss: {:.3f} '
'Accuracy: {:.3f}'.format(
epoch,
index,
loss_total *
params.BATCH_SIZE / params.TRAINING_DISPLAY_STEP,
accuracy_total *
params.BATCH_SIZE / params.TRAINING_DISPLAY_STEP,
)
)
loss_total = accuracy_total = 0
evaluation.evaluation(sess, step, eval_one_hots,
x, labels, accuracy, cost,
eval_writer, chars_size, merged)
utils.create_example_text(sess, x, logits, chars,
dictionary, reverse_dictionary)
feature_size = data_embeddings.shape[-1]
seed = 88
# Creating bi-lstm based computational graph and attaching
# loss and optimizer to the graph
outputs, inputs, labels, drop_rate = model.lstm_model(input_shape=(None, num_time, feature_size), label_shape=(None, num_time, num_labels),
num_layers=NUM_LAYERS, cell_size=CELL_SIZE)
loss, mask = model.build_loss(labels, outputs, loss_name=ARGS.loss)
patient_pred = model.compute_patient_prediction(labels, outputs, mask)
train_loss = tf.summary.scalar('train_loss', loss)
validation_loss = tf.summary.scalar('val_loss', loss)
train_op, gradient_norm = model.optimizer(loss, lr=ARGS.lr)
grad_norm = tf.summary.scalar('grad_norm', gradient_norm)
train_summary = tf.summary.merge([train_loss, grad_norm])
validation_summary = tf.summary.merge([validation_loss])
saver = tf.train.Saver(tf.global_variables(), max_to_keep=1, save_relative_paths=True)
# Dividing training data into train and validation data
train_embedding, val_embedding, train_labels, val_labels, train_mask, val_mask = train_test_split(data_embeddings, data_labels,
data_mask, test_size=VAL_RATIO,
random_state=seed)
train_data_num = train_embedding.shape[0]
# Get a batch from the reader
reader = ImageReader(images_dir, labels_dir, data_list_file, (config.image_height, config.image_width),
False, False, 0)
inputs = reader.dequeue(batch_size)
image_batch = inputs[0] #batch size x height x width x 3
label_batch = inputs[1] #batch size x height x width x 1
# Load the model
mapping = model.get_mapping_model(image_batch, name='mapping')
# Initialize variables
sess.run(tf.global_variables_initializer())
# Loss and optimizer
loss = model.loss(mapping, label_batch)
optimizer = model.optimizer(loss)
# Run the data queue
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
# Run the session
i = 0
while i < n_iters:
# Running optimization on a single batch
sess.run(optimizer)
# Print loss
print("Iter {}: loss = {}".format(i, loss))