def transfer(model, decoder, sess, args, vocab, data0, data1, out_path):
batches, order0, order1 = get_batches(data0, data1, vocab.word2id,
args.batch_size)
data0_tsf, data1_tsf = [], []
losses = Losses(len(batches))
for batch in batches:
ori, tsf = decoder.rewrite(batch)
half = batch['size'] / 2
data0_tsf += tsf[:half]
data1_tsf += tsf[half:]
loss, loss_g, loss_d, loss_d0, loss_d1 = sess.run(
[
model.loss, model.loss_g, model.loss_d, model.loss_d0,
model.loss_d1
],
feed_dict=feed_dictionary(model, batch, args.rho, args.gamma_min))
losses.add(loss, loss_g, loss_d, loss_d0, loss_d1)
n0, n1 = len(data0), len(data1)
data0_tsf = reorder(order0, data0_tsf)[:n0]
data1_tsf = reorder(order1, data1_tsf)[:n1]
if out_path:
write_sent(data0_tsf, out_path + '.0' + '.tsf')
write_sent(data1_tsf, out_path + '.1' + '.tsf')
return losses
def __init__(self,
model,
optimizer,
train_loader,
test_loader,
args,
epoch=-1,
global_step=0,
test_mode=False):
if args.fp16:
try:
from apex import amp
global amp
amp = amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
self.model = model
self.args = args
self.optimizer = optimizer
self.train_loader = train_loader
self.test_loader = test_loader
self.epoch = epoch
self.module = model.module if hasattr(
model, 'module') else model # for data parallel
self.masking_policies = [
'random', 'seen_noun', 'seen_verb',
'seen_combo_seen_noun_seen_verb', 'new_noun', 'new_verb',
'new_combo_seen_noun_seen_verb', 'new_combo_new_noun_new_verb',
'seen_combo_seen_noun_seen_verb_merge',
'new_combo_seen_noun_seen_verb_merge',
'new_combo_new_noun_new_verb_merge'
]
if test_mode and not args.pointing:
self.masker = TestMasker(annotation_root=args.annotation_root,
masking_policy=args.test_masking_policy,
tok=self.train_loader.dataset.tokenizer,
p_mask_img=args.p_mask_img,
p_mask_txt=args.p_mask_txt)
else:
self.masker = Masker(self.train_loader.dataset.tokenizer,
**vars(args))
self.losses = Losses(self.module.cfg, args, **vars(args))
self.global_step = global_step
def transfer(model, sess, args, vocab, data0, data1, out_path):
batches, order0, order1 = get_batches(data0, data1, vocab.word2id,
args.batch_size)
data0_tsf, data1_tsf = [], []
losses = Losses(len(batches))
for batch in batches:
ori, tsf, loss, loss_g, loss_d, loss_d0, loss_d1 = rewrite(
model, sess, args, vocab, batch)
half = batch['size'] / 2
data0_tsf += tsf[:half]
data1_tsf += tsf[half:]
losses.add(loss, loss_g, loss_d, loss_d0, loss_d1)
n0, n1 = len(data0), len(data1)
data0_tsf = reorder(order0, data0_tsf)[:n0]
data1_tsf = reorder(order1, data1_tsf)[:n1]
if out_path:
write_sent(data0_tsf, out_path + '.0' + '.tsf')
write_sent(data1_tsf, out_path + '.1' + '.tsf')
return losses
with tf.Session(config=config) as sess:
model = create_model(sess, args, vocab)
if args.beam > 1:
decoder = beam_search.Decoder(sess, args, vocab, model)
else:
decoder = greedy_decoding.Decoder(sess, args, vocab, model)
if args.train:
batches, _, _ = get_batches(train0, train1, vocab.word2id,
args.batch_size)
random.shuffle(batches)
start_time = time.time()
step = 0
losses = Losses(args.steps_per_checkpoint)
best_dev = float('inf')
learning_rate = args.learning_rate
rho = args.rho
gamma = args.gamma_init
dropout = args.dropout_keep_prob
for epoch in range(1, 1 + args.max_epochs):
print '--------------------epoch %d--------------------' % epoch
print 'learning_rate:', learning_rate, ' gamma:', gamma
for batch in batches:
feed_dict = feed_dictionary(model, batch, rho, gamma,
dropout, learning_rate)
loss_d0, _ = sess.run([model.loss_d0, model.optimizer_d0],
def train(**args):
"""
Evaluate selected model
Args:
rerun (Int): Integer indicating number of repetitions for the select experiment
seed (Int): Integer indicating set seed for random state
save_dir (String): Top level directory to generate results folder
model (String): Name of selected model
dataset (String): Name of selected dataset
exp (String): Name of experiment
debug (Int): Debug state to avoid saving variables
load_type (String): Keyword indicator to evaluate the testing or validation set
pretrained (Int/String): Int/String indicating loading of random, pretrained or saved weights
opt (String): Int/String indicating loading of random, pretrained or saved weights
lr (Float): Learning rate
momentum (Float): Momentum in optimizer
weight_decay (Float): Weight_decay value
final_shape ([Int, Int]): Shape of data when passed into network
Return:
None
"""
print(
"\n############################################################################\n"
)
print("Experimental Setup: ", args)
print(
"\n############################################################################\n"
)
for total_iteration in range(args['rerun']):
# Generate Results Directory
d = datetime.datetime.today()
date = d.strftime('%Y%m%d-%H%M%S')
result_dir = os.path.join(
args['save_dir'], args['model'], '_'.join(
(args['dataset'], args['exp'], date)))
log_dir = os.path.join(result_dir, 'logs')
save_dir = os.path.join(result_dir, 'checkpoints')
if not args['debug']:
os.makedirs(result_dir, exist_ok=True)
os.makedirs(log_dir, exist_ok=True)
os.makedirs(save_dir, exist_ok=True)
# Save copy of config file
with open(os.path.join(result_dir, 'config.yaml'), 'w') as outfile:
yaml.dump(args, outfile, default_flow_style=False)
# Tensorboard Element
writer = SummaryWriter(log_dir)
# Check if GPU is available (CUDA)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# Load Network
model = create_model_object(**args).to(device)
# Load Data
loader = data_loader(model_obj=model, **args)
if args['load_type'] == 'train':
train_loader = loader['train']
valid_loader = loader[
'train'] # Run accuracy on train data if only `train` selected
elif args['load_type'] == 'train_val':
train_loader = loader['train']
valid_loader = loader['valid']
else:
sys.exit('Invalid environment selection for training, exiting')
# END IF
# Training Setup
params = [p for p in model.parameters() if p.requires_grad]
if args['opt'] == 'sgd':
optimizer = optim.SGD(params,
lr=args['lr'],
momentum=args['momentum'],
weight_decay=args['weight_decay'])
elif args['opt'] == 'adam':
optimizer = optim.Adam(params,
lr=args['lr'],
weight_decay=args['weight_decay'])
else:
sys.exit('Unsupported optimizer selected. Exiting')
# END IF
scheduler = MultiStepLR(optimizer,
milestones=args['milestones'],
gamma=args['gamma'])
if isinstance(args['pretrained'], str):
ckpt = load_checkpoint(args['pretrained'])
model.load_state_dict(ckpt)
start_epoch = load_checkpoint(args['pretrained'],
key_name='epoch') + 1
optimizer.load_state_dict(
load_checkpoint(args['pretrained'], key_name='optimizer'))
for quick_looper in range(start_epoch):
scheduler.step()
# END FOR
else:
start_epoch = 0
# END IF
model_loss = Losses(device=device, **args)
acc_metric = Metrics(**args)
best_val_acc = 0.0
############################################################################################################################################################################
# Start: Training Loop
for epoch in range(start_epoch, args['epoch']):
running_loss = 0.0
print('Epoch: ', epoch)
# Setup Model To Train
model.train()
# Start: Epoch
for step, data in enumerate(train_loader):
if step % args['pseudo_batch_loop'] == 0:
loss = 0.0
optimizer.zero_grad()
# END IF
x_input = data['data'].to(device)
annotations = data['annots']
assert args['final_shape'] == list(x_input.size(
)[-2:]), "Input to model does not match final_shape argument"
outputs = model(x_input)
loss = model_loss.loss(outputs, annotations)
loss = loss * args['batch_size']
loss.backward()
running_loss += loss.item()
if np.isnan(running_loss):
import pdb
pdb.set_trace()
# END IF
if not args['debug']:
# Add Learning Rate Element
for param_group in optimizer.param_groups:
writer.add_scalar(
args['dataset'] + '/' + args['model'] +
'/learning_rate', param_group['lr'],
epoch * len(train_loader) + step)
# END FOR
# Add Loss Element
writer.add_scalar(
args['dataset'] + '/' + args['model'] +
'/minibatch_loss',
loss.item() / args['batch_size'],
epoch * len(train_loader) + step)
# END IF
if ((epoch * len(train_loader) + step + 1) % 100 == 0):
print('Epoch: {}/{}, step: {}/{} | train loss: {:.4f}'.
format(
epoch, args['epoch'], step + 1,
len(train_loader), running_loss /
float(step + 1) / args['batch_size']))
# END IF
if (epoch * len(train_loader) +
(step + 1)) % args['pseudo_batch_loop'] == 0 and step > 0:
# Apply large mini-batch normalization
for param in model.parameters():
param.grad *= 1. / float(
args['pseudo_batch_loop'] * args['batch_size'])
optimizer.step()
# END IF
# END FOR: Epoch
if not args['debug']:
# Save Current Model
save_path = os.path.join(
save_dir, args['dataset'] + '_epoch' + str(epoch) + '.pkl')
save_checkpoint(epoch, step, model, optimizer, save_path)
# END IF: Debug
scheduler.step(epoch=epoch)
print('Schedulers lr: %f', scheduler.get_lr()[0])
## START FOR: Validation Accuracy
running_acc = []
running_acc = valid(valid_loader, running_acc, model, device,
acc_metric)
if not args['debug']:
writer.add_scalar(
args['dataset'] + '/' + args['model'] +
'/validation_accuracy', 100. * running_acc[-1],
epoch * len(valid_loader) + step)
print('Accuracy of the network on the validation set: %f %%\n' %
(100. * running_acc[-1]))
# Save Best Validation Accuracy Model Separately
if best_val_acc < running_acc[-1]:
best_val_acc = running_acc[-1]
if not args['debug']:
# Save Current Model
save_path = os.path.join(
save_dir, args['dataset'] + '_best_model.pkl')
save_checkpoint(epoch, step, model, optimizer, save_path)
# END IF
# END IF
# END FOR: Training Loop
############################################################################################################################################################################
if not args['debug']:
# Close Tensorboard Element
writer.close()
def __init__(
self,
seed=None,
optimizer=Adam,
optimizer_kwargs={},
learning_rate_init=0.04,
gamma=0.995, # learning rate decay factor
considered_groups=list(
range(12)), ## group layers to be considered from start
sample_variance_threshold=0.002,
weight_loss_sample_variance=0, # 10.
evaluation_steps=250, # number of batches between loss tracking
N_batches_test=1, # number of batches considered for evaluation
):
super(ImageClassifier,
self).__init__(considered_groups=considered_groups)
if seed is not None:
torch.manual_seed(seed)
#'''
resnet = models.resnet18(pretrained=False)
self.conv = Sequential(
*(list(resnet.children())[:-1]),
Flatten(),
)
''' architecture used by Dielemann et al 2015
self.conv = Sequential(
# Conv2dUntiedBias(41, 41, 3, 32, kernel_size=6),
Conv2d(3,32, kernel_size=6),
ReLU(),
MaxPool2d(2),
# Conv2dUntiedBias(16, 16, 32, 64, kernel_size=5),
Conv2d(32, 64, kernel_size=5),
ReLU(),
MaxPool2d(2),
# Conv2dUntiedBias(6, 6, 64, 128, kernel_size=3),
Conv2d(64, 128, kernel_size=3),
ReLU(),
# Conv2dUntiedBias(4, 4, 128, 128, kernel_size=3), #weight_std=0.1),
Conv2d(128, 128, kernel_size=3),
ReLU(),
MaxPool2d(2),
Flatten(),
)
#'''
self.dense1 = MaxOut(8192, 2048, bias=0.01)
self.dense2 = MaxOut(2048, 2048, bias=0.01)
self.dense3 = Sequential(
MaxOut(2048, 37, bias=0.1),
# LeakyReLU(negative_slope=1e-7),
ALReLU(negative_slope=1e-2),
)
self.dropout = Dropout(p=0.5)
self.augment = Compose([
Lambda(lambda img: torch.cat([img, hflip(img)], 0)),
Lambda(lambda img: torch.cat([img, rotate(img, 45)], 0)),
FiveCrop(45),
Lambda(lambda crops: torch.cat([
rotate(crop, ang)
for crop, ang in zip(crops, (0, 90, 270, 180))
], 0)),
])
self.N_augmentations = 16
self.N_conv_outputs = 512
self.set_optimizer(optimizer,
lr=learning_rate_init,
**optimizer_kwargs)
# self.scheduler = ExponentialLR(self.optimizer, gamma=gamma)
self.scheduler = MultiStepLR(self.optimizer,
milestones=[292, 373],
gamma=gamma)
self.make_labels_hierarchical = False # if True, output probabilities are renormalized to fit the hierarchical label structure
self.N_batches_test = N_batches_test
self.evaluation_steps = evaluation_steps # number of batches between loss tracking
self.weight_loss_sample_variance = weight_loss_sample_variance
self.sample_variance_threshold = sample_variance_threshold
self.iteration = 0
self.epoch = 0
self.losses_train = Losses("loss", "train")
self.losses_valid = Losses("loss", "valid")
self.sample_variances_train = Losses("sample variance", "train")
self.sample_variances_valid = Losses("sample variance", "valid")
for g in range(1, 12):
setattr(self, f"accuracies_Q{g}_train",
Accuracies("accuracy train", f"Q{g}"))
setattr(self, f"accuracies_Q{g}_valid",
Accuracies("accuracy valid", f"Q{g}"))
self.losses_regression = Losses("loss", "regression")
self.losses_variance = Losses("loss", "sample variance")
## return to random seed
if seed is not None:
sd = np.random.random() * 10000
torch.manual_seed(sd)
def unet_model_fn(features, labels, mode, params):
tf.local_variables_initializer()
loss, train_op, = None, None
eval_metric_ops, training_hooks, evaluation_hooks = None, None, None
predictions_dict = None
unet = Unet(params=params)
logits = unet.model(input_tensor=features['image'])
y_pred = tf.math.softmax(logits, axis=-1)
output_img = tf.expand_dims(tf.cast(tf.math.argmax(y_pred, axis=-1) * 255, dtype=tf.uint8), axis=-1)
if mode in (estimator.ModeKeys.TRAIN, estimator.ModeKeys.EVAL):
with tf.name_scope('Loss_Calculation'):
loss = Losses(logits=logits, labels=labels['label'])
loss = loss.custom_loss()
with tf.name_scope('Dice_Score_Calculation'):
dice = f1(labels=labels['label'], predictions=y_pred)
with tf.name_scope('Images_{}'.format(mode)):
with tf.name_scope('Reformat_Outputs'):
label = tf.expand_dims(tf.cast(tf.argmax(labels['label'], -1) * 255, dtype=tf.uint8), axis=-1)
image = tf.math.divide(features['image'] - tf.reduce_max(features['image'], [0, 1, 2]),
tf.reduce_max(features['image'], [0, 1, 2]) - tf.reduce_min(features['image'],
[0, 1, 2]))
summary.image('1_Medical_Image', image, max_outputs=1)
summary.image('2_Output', output_img, max_outputs=1)
summary.image('3_Output_pred', tf.expand_dims(y_pred[:, :, :, 1], -1), max_outputs=1)
summary.image('4_Output_label', label, max_outputs=1)
if mode == estimator.ModeKeys.TRAIN:
with tf.name_scope('Learning_Rate'):
global_step = tf.compat.v1.train.get_or_create_global_step()
learning_rate = tf.compat.v1.train.exponential_decay(params['lr'], global_step=global_step,
decay_steps=params['decay_steps'],
decay_rate=params['decay_rate'], staircase=False)
with tf.name_scope('Optimizer_conf'):
train_op = Adam(learning_rate=learning_rate).minimize(loss=loss, global_step=global_step)
with tf.name_scope('Metrics'):
summary.scalar('Output_DSC', dice[1])
summary.scalar('Learning_Rate', learning_rate)
if mode == estimator.ModeKeys.EVAL:
eval_metric_ops = {'Metrics/Output_DSC': dice}
eval_summary_hook = tf.estimator.SummarySaverHook(output_dir=params['eval_path'],
summary_op=summary.merge_all(),
save_steps=params['eval_steps'])
evaluation_hooks = [eval_summary_hook]
if mode == estimator.ModeKeys.PREDICT:
predictions_dict = {'image': features['image'],
'y_preds': y_pred[:, :, :, 1],
'output_img': output_img,
'path': features['path']}
return estimator.EstimatorSpec(mode,
predictions=predictions_dict,
loss=loss,
train_op=train_op,
eval_metric_ops=eval_metric_ops,
training_hooks=training_hooks,
evaluation_hooks=evaluation_hooks)