def epochTrain (epochID, model, dataLoader, optimizer, loss, test_model):
model.train()
for batchID, (input, target) in enumerate (dataLoader):
# print(args.branch_type+' Training...'+'epoch:'+str(epochID)+'||'+'train:'+str(batchID))
target = target.cuda(async = True)
if args.branch_type == 'local':
output, heatmap, pool = test_model(torch.autograd.Variable(input.cuda(async = True)))
input = heatmap_crop_origin(heatmap.cpu(), input)
if args.branch_type == 'fusion':
test_model1, test_model2 = test_model
output, heatmap, pool1 = test_model1(torch.autograd.Variable(input.cuda(async = True)))
inputs = heatmap_crop_origin(heatmap.cpu(), input)
output, heatmap, pool2 = test_model2(torch.autograd.Variable(inputs.cuda(async = True)))
pool1 = pool1.view(pool1.size(0), -1)
pool2 = pool2.view(pool2.size(0), -1)
pool3 = pool3.view(pool3.size(0), -1)
input = torch.cat((pool1.cpu(), pool2.cpu(), pool3.cpu()), dim=1)
input = input.cuda(async = True)
varInput = torch.autograd.Variable(input)
varTarget = torch.autograd.Variable(target)
varOutput, heatmap, pool = model(varInput)
lossvalue = loss(varOutput, varTarget)
optimizer.zero_grad()
lossvalue.backward()
optimizer.step()
def test_loss(self):
_, loss, _ = linear(self.l1, self.l2)
actual = loss(self.theta, self.x, self.y)
self.assertTrue(isinstance(actual, float))
expected_error = 22 - 6
expected_loss = expected_error * expected_error + 1.0 + 1.3
self.assertAlmostEqual(actual, expected_loss)
def attack3():
attack_loader = loader.attack_loader
loss.start_log()
for batch, (inputs, labels) in enumerate(attack_loader):
inputs = inputs.to('cuda')
inputs.requires_grad = True
labels = labels.to('cuda')
print("333333333333333333")
outputs = model(inputs)
lossx = loss(outputs, labels)
gradients = torch.autograd.grad(lossx, inputs)[0]
per = 0.1 * gradients.sign()
for (t) in range(14):
adversarial_examples = inputs + per
#adversarial_examples = torch.clamp(adversarial_examples, min=0, max=1).detach()
#adversarial_examples.requires_grad = True
outputs = model(adversarial_examples)
lossx = loss(outputs, labels)
gradients = torch.autograd.grad(lossx, adversarial_examples)[0]
per = per + (0.1 / 14) * gradients.sign()
per = torch.clamp(per, min=-0.1, max=0.1)
#loss.backward()
save_img(adversarial_examples, inputs.shape[0])
loss.end_log(len(attack_loader))
def _info_gain(self, y, threshold, col_data):
# 计算当前节点上的所有样本，在当前特征的当前分割点下，所选择的criterion的信息增益
if self.criterion == 'mse':
loss = mse
elif self.criterion == 'entropy':
loss = entropy
elif self.criterion == 'gini':
loss = gini
parent_loss = loss(y)
left_idx_array = np.argwhere(col_data <= threshold).flatten()
right_idx_array = np.argwhere(col_data > threshold).flatten()
# 计算子节点的loss的时候，简单按照样本个数比例做加权，惩罚一下样本数太少的子节点
left_num = len(left_idx_array)
right_num = len(right_idx_array)
n = left_num + right_num
left_loss = loss(y[left_idx_array])
right_loss = loss(y[right_idx_array])
child_loss = (left_num / n) * left_loss + (right_num / n) * right_loss
return parent_loss - child_loss
def validate(opt, shared, m, val_data, val_idx):
m.train(False)
val_loss = 0.0
num_ex = 0
loss = None
if opt.loss == 'loss':
loss = Loss(opt, shared)
elif opt.loss == 'crf':
loss = CRFLoss(opt, shared)
elif opt.loss == 'luka': # Ashim : For Lukasiewicz Loss
print('Using Lukasiewicz Loss')
loss = LukaLoss(opt, shared)
elif opt.loss == 'godel':
print('Using Godel Loss')
loss = GodelLoss(opt, shared)
loss.train(False)
print('validating on the {0} batches...'.format(len(val_idx)))
m.begin_pass()
for i in range(len(val_idx)):
(data_name, source, char_source, batch_ex_idx, batch_l, source_l,
label, res_map) = val_data[val_idx[i]]
wv_idx = Variable(source, requires_grad=False)
cv_idx = Variable(char_source, requires_grad=False)
y_gold = Variable(label, requires_grad=False)
# update network parameters
m.update_context(batch_ex_idx, batch_l, source_l, res_map)
# forward pass
pred = m.forward(wv_idx, cv_idx)
# loss
batch_loss = loss(pred, y_gold)
# stats
val_loss += float(batch_loss.data)
num_ex += batch_l
perf, extra_perf = loss.get_epoch_metric()
m.end_pass()
return (perf, extra_perf, val_loss / num_ex, num_ex)
def epochVal (epochID, model, dataLoader, optimizer, loss, test_model):
with torch.no_grad():
model.eval ()
lossVal = 0
lossValNorm = 0
losstensorMean = 0
for i, (input, target) in enumerate (dataLoader):
# print(args.branch_type+' Valing...''epoch:'+str(epochID)+'||'+'eval:'+str(i))
target = target.cuda(async = True)
if args.branch_type == 'local':
output, heatmap, pool = test_model(torch.autograd.Variable(input.cuda(async = True)))
input = heatmap_crop_origin(heatmap.cpu(), input)
if args.branch_type == 'fusion':
test_model1, test_model2 = test_model
output, heatmap, pool1 = test_model1(torch.autograd.Variable(input.cuda(async = True)))
inputs = heatmap_crop_origin(heatmap.cpu(), input)
output, heatmap, pool2 = test_model2(torch.autograd.Variable(inputs.cuda(async = True)))
pool1 = pool1.view(pool1.size(0), -1)
pool2 = pool2.view(pool2.size(0), -1)
input = torch.cat((pool1.cpu(), pool2.cpu()), dim=1)
input = input.cuda(async = True)
varInput = torch.autograd.Variable(input, volatile=True)
varTarget = torch.autograd.Variable(target, volatile=True)
varOutput, heatmap, pool = model(varInput)
losstensor = loss(varOutput, varTarget)
losstensorMean += losstensor
lossVal += losstensor.data[0]
lossValNorm += 1
outLoss = lossVal / lossValNorm
losstensorMean = losstensorMean / lossValNorm
return outLoss, losstensorMean
def evaluate(opt, shared, m, data):
m.train(False)
val_loss = 0.0
num_ex = 0
verbose = opt.verbose==1
loss = None
if opt.loss == 'loss':
loss = Loss(opt, shared)
elif opt.loss == 'crf':
loss = CRFLoss(opt, shared)
loss.train(False)
loss.verbose = verbose
m.begin_pass()
loss.begin_pass()
for i in range(data.size()):
(data_name, source, char_source, batch_ex_idx, batch_l, source_l, label, res_map) = data[i]
wv_idx = Variable(source, requires_grad=False)
cv_idx = Variable(char_source, requires_grad=False)
y_gold = Variable(label, requires_grad=False)
# update network parameters
m.update_context(batch_ex_idx, batch_l, source_l, res_map)
# forward pass
pred = m.forward(wv_idx, cv_idx)
# loss
batch_loss = loss(pred, y_gold)
# stats
val_loss += float(batch_loss.data)
num_ex += batch_l
perf, extra_perf = loss.get_epoch_metric()
loss.end_pass()
m.end_pass()
return (perf, extra_perf, val_loss / num_ex, num_ex)
def validation(data_loader, net, loss, epoch, save_dir,visual=None):
global min_loss
net.eval()
metrics=[]
loss_value=0
count=0
for i, (data,label) in enumerate(data_loader):
data=Variable(data.cuda(async=True))
label=Variable(label.cuda(async=True))
output=net(data.float())
loss_output=loss(output,label)
loss_value+=loss_output.item()
print("the loss is",loss_output.item())
count+=1
output=output>0
label=label>0
for j in range(data.size()[0]):
metrics.append(torch.equal(output[j],label[j]))
metrics=np.array(metrics)*1.0
acc=metrics.sum()/metrics.shape[0]
print("------------------------------------------------")
print("the epoch is %d, and the validation acc is %f"%(epoch,acc))
print("the loss value is ",loss_value)
print("------------------------------------------------")
if min_loss>loss_value:
#-------save the best-------------
min_loss=loss_value
state_dict=net.module.state_dict()
for key in state_dict.keys():
state_dict[key]=state_dict[key].cpu()
torch.save({
'epoch': epoch,
'save_dir': save_dir,
'state_dict': state_dict},
os.path.join(save_dir, 'the_min_loss_%f_epoch_%d_acc_%f.ckpt' % (min_loss,epoch,acc)))
def train(data_loader, net, loss, epoch, optimizer, get_lr, save_freq, save_dir,visual=None):
net.train()
lr=get_lr(epoch)
for param_group in optimizer.param_groups:
param_group['lr']=lr
metrics=[]
for i, (data,label) in enumerate(data_loader):
data=Variable(data.cuda(async=True))
label=Variable(label.cuda(async=True))
output=net(data.float())
loss_output=loss(output,label)
print("the loss is",loss_output.data)
optimizer.zero_grad()
loss_output.backward()
optimizer.step()
output=output>0 #最后一层没有sigmoid激活，所以要大于0
label=label>0
for j in range(data.size()[0]):
metrics.append(torch.equal(output[j],label[j]))
metrics=np.array(metrics)*1.0
acc=metrics.sum()/metrics.shape[0]
print("------------------------------------------------")
print("the epoch is %d, and the acc is %f"%(epoch,acc))
print("------------------------------------------------")
if not os.path.exists(save_dir):
os.makedirs(save_dir)
if epoch%save_freq==0:
#-----------for training restart---------------
state_dict=net.module.state_dict()
for key in state_dict.keys():
state_dict[key]=state_dict[key].cpu()
torch.save({
'epoch': epoch,
'save_dir': save_dir,
'state_dict': state_dict},
os.path.join(save_dir, 'train_%d_acc_%f.ckpt' % (epoch,acc)))
def run_epoch(config,optimizer,loss,epoch,pbar,model,configuration):
if configuration=='train':
model.train()
elif configuration=='val':
model.eval()
epoch_logger = {'criterion':torch.zeros(1).cuda(gpus_list[0]),'dice':torch.zeros(1).cuda(gpus_list[0]),'bce':torch.zeros(1).cuda(gpus_list[0]),'rce':torch.zeros(1).cuda(gpus_list[0])}
for iteration,batch in enumerate(pbar, 1):
if configuration=='train':
optimizer.zero_grad()
logits = model(batch['input_rd'].cuda(gpus_list[0]))
elif configuration=='val':
with torch.no_grad():
logits = model(batch['input_rd'].cuda(gpus_list[0]))
logger = loss(logits,batch['label'].cuda(gpus_list[0]))
if configuration=='train':
logger['criterion'].backward()
optimizer.step()
epoch_logger['criterion'] += logger['criterion'].detach()
epoch_logger['dice'] += logger['dice'].detach()
epoch_logger['bce'] += logger['bce'].detach()
epoch_logger['rce'] += logger['rce'].detach()
stats = {'criterion':epoch_logger['criterion'].detach().item()/(iteration*batch_size),'dice':epoch_logger['dice'].detach().item()/(iteration*batch_size),
'bce':epoch_logger['bce'].detach().item()/(iteration*batch_size),'rce':epoch_logger['rce'].detach().item()/(iteration*batch_size)}
pbar.set_postfix(ordered_dict=stats, refresh=True)
# Debug
if debug_mode and iteration==4: break
epoch_logger['criterion'] /= (iteration*batch_size)
epoch_logger['dice'] /= (iteration*batch_size)
epoch_logger['bce'] /= (iteration*batch_size)
epoch_logger['rce'] /= (iteration*batch_size)
return epoch_logger
def evaluate():
eval_batch_count = 50 # faster validation, change to 40000/batch_size for the report
validation_error = 0
validation_accuracy = 0
with tf.device('/cpu:0'):
with tf.Graph().as_default() as g:
images, labels = cifar_input.build_input(
'cifar100', '../../cifar/cifar100/validation.bin', batch_size,
'eval') #TEST.BIN OR VALIDATION.BIN
logits = inference(images, NUM_CLASSES=100)
saver = tf.train.Saver()
losses = loss(logits, labels)
accuracies = accuracy(logits, labels)
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
tf.train.start_queue_runners(sess)
ckpt_state = tf.train.get_checkpoint_state(train_dir)
saver.restore(sess, ckpt_state.model_checkpoint_path)
for _ in six.moves.range(eval_batch_count):
(value_losses, value_accuracy) = sess.run([losses, accuracies])
validation_error += value_losses
validation_accuracy += value_accuracy
validation_error /= 50
validation_accuracy /= 50
step = str(ckpt_state.model_checkpoint_path).split('-')[1]
tf.logging.info('loss: %.3f, best accuracy: %.3f' %
(validation_error, validation_accuracy))
f = open(train_dir + "validation_data.csv", 'ab')
f.write('{0},{1},{2}\n'.format(step, validation_error,
validation_accuracy))
f.close()
f = open(train_dir + "log.txt", 'ab')
f.write('loss: {0}, best accuracy: {1}\n'.format(
validation_error, validation_accuracy))
f.close()
def attack2():
attack_loader = loader.attack_loader
loss.start_log()
for batch, (inputs, labels) in enumerate(attack_loader):
inputs = inputs.to('cuda')
inputs.requires_grad = True
labels = labels.to('cuda')
outputs = model(inputs)
print(inputs.shape)
print(labels.shape)
print(outputs[0].shape)
print("22222222222222222222222")
lossx = loss(outputs, labels)
#loss.backward()
gradients = torch.autograd.grad(lossx, inputs)[0]
adversarial_examples = inputs + (0.1 * gradients.sign())
save_img(adversarial_examples, inputs.shape[0])
loss.end_log(len(attack_loader))
def train(opt):
# Load Dataset
content_image = image_load(opt.content)
style_image = image_load(opt.style)
generate_image = torch.randn_like(content_image).requires_grad_(True)
# Set Optimizer
optim = torch.optim.Adam([generate_image], lr=0.01)
# Set Loss
loss = Loss(alpha=1, beta=1000)
writer = SummaryWriter()
if not osp.isdir(opt.result):
os.makedirs(opt.result)
for epoch in range(opt.epoch):
optim.zero_grad()
total_loss, c_loss, s_loss = loss(generate_image, content_image,
style_image)
total_loss.backward()
optim.step()
writer.add_scalar('loss/total', total_loss, epoch)
writer.add_scalar('loss/content', c_loss, epoch)
writer.add_scalar('loss/style', s_loss, epoch)
if (epoch + 1) % opt.display_epoch == 0:
writer.add_images('image',
generate_image,
epoch,
dataformats="NCHW")
print('[Epoch {}] Total : {:.2} | C_loss : {:.2} | S_loss : {:.2}'.
format(epoch + 1, total_loss, c_loss, s_loss))
imsave(generate_image,
osp.join(opt.result, '{}.png'.format(epoch + 1)))
imsave(content_image, 'content.png')
imsave(style_image, 'style.png')
def plot_prediction(i):
labels = {
0: "cross",
1: "circle",
2: "triangle",
3: "square",
4: "pentagon"
}
pred = net.forward(data_test)
print(pred.shape)
pred_label = np.where(pred[:, i] == np.max(pred[:, i]))[0][0]
plot_title = labels[int(pred_label)] + " prob : " + str(
round(np.max(pred[:, i]), 2))
plt.title(plot_title)
plt.imshow(data_test[:, i].reshape(size, size), cmap="Greys")
plot_img_derivative(net, loss(), data_test[:, i], labels_test[:,
i])
plt.colorbar()
plt.show()
def faster_ssd(self):
'''
定义faster-ssd的计算图谱
'''
with self.graph.as_default():
# Training computation.
self.small_rpn_train = InceptionV3_facenet_112(
batch_seize=train_batch,
num_channels=self.input_image_size[0],
image_size_h=self.input_image_size[1],
image_size_w=self.input_image_size[2],
trainable=True,
resueable=False,
BN_decay=0.99,
BN_epsilon=0.00001,
anchor_num=15,
name="train_rpn/")
self.rpn_target_layer = anchor_target_layer(
self.small_rpn_train.roi_data,
batch=train_batch,
img_height=self.input_image_size[1],
img_width=self.input_image_size[2],
FrcnnParam=self.Work_Param)
'''
self.small_rpn_val=InceptionV3_facenet_112(batch_seize=val_batch,num_channels=self.input_image_size[0],image_size_h=self.input_image_size[1],image_size_w=self.input_image_size[2],
trainable=False,resueable=True,BN_decay=0.99,BN_epsilon=0.00001,anchor_num=15,name="val/")
self.small_rpn_test=InceptionV3_facenet_112(batch_seize=test_batch,num_channels=self.input_image_size[0],image_size_h=self.input_image_size[1],image_size_w=self.input_image_size[2],
trainable=False,resueable=True,BN_decay=0.99,BN_epsilon=0.00001,anchor_num=15,name="test/")
'''
#self.merged_train_weights_summary= tf.summary.merge(summaries)
#self.summary_placeholders, self.update_ops, self.summary_op = self.setup_summary()
for var in tf.trainable_variables():
if var.op.name.find('train_rpn') > 0:
self.rpn_variable.append(var)
train = loss(self.small_rpn_train.rpn_cls_score_reshape,
self.small_rpn_train.rpn_bbox_pred, self.rpn_variable)
return train
def build_train_graph(self):
opt = self.opt
print("ALL Ok GIRL")
print(opt.dataset_dir)
# class weights Sitting
class_weight = [
1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0
]
class_weight = tf.reshape(
tf.convert_to_tensor(class_weight, dtype=tf.float32), [12])
self.train_image_batch = tf.placeholder(
tf.float32, [None, opt.img_height, opt.img_width, 3])
self.train_label_batch = tf.placeholder(
tf.float32,
[None, opt.img_height * opt.img_width, opt.numberClasses])
self.tgt_image = self.train_image_batch
self.tgt_label = self.train_label_batch
self.tgt_label = self.train_label_batch
# train_image_batch, train_label_batch = self.camvid_batches()
self.N_classes = opt.numberClasses
self.keep_prob = opt.dropout
self.batch_size = opt.batch_size
self.width = opt.img_width
self.height = opt.img_height
self.Training = True
self.random = False
self.total_steps = opt.max_steps
self.dataset = opt.dataset
self.version_net = opt.version_net
self.configuration = opt.configuration
with tf.name_scope("Net_prediction"):
if self.version_net == 'FCN_Seg':
print("Computing FCN_Seg encoder and decoder")
segMap = FCN_Seg(self, is_training=self.Training)
print("Output FCN_Seg")
print(segMap)
with tf.name_scope("Output_Metrics"):
segmentationMask = tf.argmax(segMap, axis=1)
# print("Segmentation mask")
# print(segmentationMask)
Smask = tf.placeholder(tf.float32,
[None, opt.img_height * opt.img_width])
Smask = tf.reshape(
segmentationMask,
(self.batch_size, opt.img_height * opt.img_width))
segmentationLabel = tf.argmax(self.train_label_batch, axis=2)
# print("Segmentation Label")
# print(segmentationLabel)
print(Smask)
print(segmentationLabel)
equality = tf.equal(Smask, segmentationLabel)
accuracy = tf.reduce_mean(tf.cast(equality, tf.float32))
with tf.name_scope("compute_loss"):
# Compute Softmax loss of D-Net
Reshaped_labels = tf.reshape(self.train_label_batch,
(-1, self.N_classes))
if opt.version_net == 'FCN_Seg':
TotalLoss = loss(segMap, Reshaped_labels, self.N_classes,
class_weight)
print(TotalLoss)
with tf.name_scope("Training"):
print(
"==================== Training ================================="
)
train_vars = [var for var in tf.trainable_variables()]
global_step = tf.train.get_or_create_global_step()
self.global_step = global_step
self.incr_global_step = tf.assign(self.global_step,
self.global_step + 1)
# DECAY ############################
print("Exponential Decay .......")
Decay = 0.90
# Decay the learning rate exponentially based on the number of
# steps.
lr = tf.train.exponential_decay(opt.learning_rate,
global_step,
(self.total_steps / 2),
Decay,
staircase=True)
# OPTIMIZER #########################
# select Adam as optimizer
optimizer = tf.train.AdamOptimizer(lr, opt.beta1)
self.grads_and_vars = optimizer.compute_gradients(
TotalLoss, var_list=train_vars)
self.Training = optimizer.apply_gradients(self.grads_and_vars)
self.learning_rate = lr
# Collect tensors that are useful later (tf summary)
self.predMask = segMap
self.total_loss = TotalLoss
self.steps_per_epoch = opt.steps_per_epoch
self.accuracy = accuracy
self.version_net = opt.version_net
# merge all summaries into a single "operation" which we can execute in a session
# merged = tf.summary.merge_all()
self.logs_path = opt.logs_path
print("DONE BUILDING GRAPH!")
def DifferentialEvolution_f(self,model):
self.emin,self.emax =model.normalise_val()
model=self.model
re = ""
frontier=[model.candidates(0,False,0) for _ in range(self.max_changes)]
best_energy=model.Energy(frontier[0],self.emax,self.emin)
best_solution=frontier[0]
k=0
list_energy=[]
a12_small=0.56
prev_energy=[0 for _ in range(21)]
while k<self.kmax :
re=""
if(best_energy==self.threshold):
break
for i, solution in enumerate(frontier):
seen = []
while len(seen) < 3:
rand_index = random.randint(0, 99)
if rand_index == i:
continue
if rand_index not in seen:
seen.append(rand_index)
mutation=frontier[seen[0]]
current_energy=model.Energy(solution,self.emax,self.emin)
append=" ."
if(self.crossover < random.random()):
if(model.Energy(mutation,self.emax,self.emin)<current_energy):
current_energy=model.Energy(mutation,self.emax,self.emin)
frontier[i] = mutation
append = " +"
else:
mutation=[]
for j in xrange(model.n):
l=model.lo[j]
m=model.hi[j]
inter = (frontier[seen[0]][j] + self.f* (frontier[seen[1]][j] - frontier[seen[2]][j]))
if inter >= l and inter <= m:
mutation.append(inter)
else:
mutation.append(frontier[seen[random.randint(0, 2)]][j])
if(model.ok(mutation) and model.Energy(mutation,self.emax,self.emin)<current_energy) :
frontier[i]=mutation
current_energy=model.Energy(mutation,self.emax,self.emin)
append=" +"
if(current_energy<best_energy and current_energy>=self.threshold):
append=" ?"
best_energy=current_energy
best_solution=frontier[i]
list_energy.append(current_energy)
re=re+append
k+=1
if (k>0):
if (k==25):
first_energy=prev_energy
if k % 25 is 0:
print k,best_energy,re
re = ""
l1=list_energy
l2=prev_energy
# print l1,l2
# print l1
#print l2
a12_result=a12(l1,l2)
#print a12_result
if (a12_result>a12_small):
k =k +5
else :
k=k -1
prev_energy=list_energy
list_energy=[]
l=''
final_energy=prev_energy
x=loss(final_energy,first_energy)
global_variable.loss_inter=x
print("\nBest Solution : " + str(best_solution))
print("Best Energy : " + str((model.Energy(best_solution,self.emax,self.emin))))
def train_epoch(opt, shared, m, optim, ema, data, epoch_id, sub_idx):
train_loss = 0.0
num_ex = 0
start_time = time.time()
min_grad_norm2 = 1000000000000.0
max_grad_norm2 = 0.0
loss = None
if opt.loss == 'loss':
loss = Loss(opt, shared)
elif opt.loss == 'crf':
loss = CRFLoss(opt, shared)
elif opt.loss == 'luka': # Ashim : For Lukasiewicz Loss
print('Using Lukasiewicz Loss')
loss = LukaLoss(opt, shared)
elif opt.loss == 'godel':
print('Using Godel Loss')
loss = GodelLoss(opt, shared)
#if opt.loss == 'godel' and epoch_id < 100:
# print('Using R-Prod')
# loss = Loss(opt, shared)
data_size = len(sub_idx)
batch_order = torch.randperm(data_size)
batch_order = [sub_idx[idx] for idx in batch_order]
acc_batch_size = 0
m.train(True)
loss.train(True)
loss.begin_pass()
m.begin_pass()
for i in range(data_size):
(data_name, source, char_source, batch_ex_idx, batch_l, source_l,
label, res_map) = data[batch_order[i]]
#print('Batch_l ' + str(batch_l))
#print('Source_l ' +str(source_l))
#print(source)
wv_idx = Variable(source, requires_grad=False)
cv_idx = Variable(char_source, requires_grad=False)
y_gold = Variable(label, requires_grad=False)
# update network parameters
shared.epoch = epoch_id
m.update_context(batch_ex_idx, batch_l, source_l, res_map)
# forward pass
output = m.forward(wv_idx, cv_idx)
# loss
if opt.loss == 'luka':
batch_loss = loss(output,
y_gold,
constraints_lambda=opt.constraints_lambda)
else:
batch_loss = loss(output,
y_gold,
constraints_lambda=opt.constraints_lambda)
# stats
train_loss += float(batch_loss.data)
num_ex += batch_l
time_taken = time.time() - start_time
acc_batch_size += batch_l
# accumulate grads
batch_loss.backward()
# accumulate current batch until the rolled up batch size exceeds threshold or meet certain boundary
if i == data_size - 1 or acc_batch_size >= opt.acc_batch_size or (
i + 1) % opt.print_every == 0:
grad_norm2 = optim.step(m, acc_batch_size)
if opt.ema == 1:
ema.step(m)
# clear up grad
m.zero_grad()
acc_batch_size = 0
# stats
grad_norm2_avg = grad_norm2
min_grad_norm2 = min(min_grad_norm2, grad_norm2_avg)
max_grad_norm2 = max(max_grad_norm2, grad_norm2_avg)
time_taken = time.time() - start_time
loss_stats = loss.print_cur_stats()
if (i + 1) % opt.print_every == 0:
stats = '{0}, Batch {1:.1f}k '.format(epoch_id + 1,
float(i + 1) / 1000)
stats += 'Grad {0:.1f}/{1:.1f} '.format(
min_grad_norm2, max_grad_norm2)
stats += 'Loss {0:.4f} '.format(train_loss / num_ex)
stats += loss.print_cur_stats()
stats += 'Time {0:.1f}'.format(time_taken)
print(stats)
perf, extra_perf = loss.get_epoch_metric()
m.end_pass()
loss.end_pass()
return perf, extra_perf, train_loss / num_ex, num_ex
def lr_find(model, lr_max, lr_min, trn_dataloader, linear=True):
"""
"""
torch.save(model.state_dict(), "tmp.pth")
lr_array, loss_array = [], []
num_batch = len(trn_dataloader)
ratio = lr_max / lr_min
step = (lr_max - lr_min) / num_batch
optimizer = torch.optim.SGD(model.parameters(), lr=lr_min, momentum=0.9)
print("SGD")
use_cuda = torch.cuda.is_available()
device = torch.device("cuda:0" if use_cuda else "cpu")
model.freeze_basenet()
model = model.to(device)
prior_box = get_prior_box()
min_loss = float("inf")
for i, batch in enumerate(tqdm(trn_dataloader)):
if linear:
lr = lr_min + step * i
else:
lr = lr_min * ratio ** (i / num_batch)
optimizer.param_groups[0]["lr"] = lr
imgs, bboxes, labels = batch
imgs = imgs.to(device)
cls_preds, loc_preds = model(imgs)
model.zero_grad()
total_loss = 0
total_loc_loss, total_cls_loss = 0, 0
for idx in range(imgs.shape[0]):
img, bbox, label = imgs[idx], bboxes[idx], labels[idx]
cls_pred, loc_pred = cls_preds[idx], loc_preds[idx]
iou = get_iou(bbox, prior_box)
pos_mask, cls_target, bbox_target = get_target(
iou, prior_box, img, bbox, label
)
pos_mask, cls_target, bbox_target = (
pos_mask.to(device),
cls_target.to(device),
bbox_target.to(device),
)
loss_loc, loss_cls = loss(
cls_pred, loc_pred, pos_mask, cls_target, bbox_target
)
total_loc_loss += loss_loc
total_cls_loss += loss_cls
total_loss += loss_cls + loss_loc
total_loss /= float(imgs.shape[0])
# use min_loss to terminate the lr find more quickly
if min_loss * 4 <= float(total_loss):
break
min_loss = min(float(min_loss), float(total_loss.data))
# print(min_loss)
total_loss.backward()
optimizer.step()
lr_array.append(lr)
loss_array.append(round(float(total_loss), 3))
model.state_dict = torch.load("tmp.pth")
return lr_array, loss_array
dataset = dataset(batch_size=cfg.total_bs, imgsize=cfg.img_size)
dataset.show_batch(rows=3, figsize=(10,10))
plt.show()
except:
print("Dataset Type {} are Not Implemented. ".format(cfg.dataset_type))
exit()
metrics = [getattr(fastai.metrics, met) for met in cfg.metric]
try:
loss = getattr(nn, cfg.loss)
except AttributeError as error:
loss = getattr(loss, cfg.loss)
optimizer = get_optimizer(cfg.optimizer)
learner = Learner(dataset, model, opt_func=optimizer, loss_func=loss().to(device),
metrics=metrics, bn_wd=False, true_wd=True,
wd=cfg.optimizer.weight_decay, path=cfg.work_dir) # Custom Learner
# models : Darknet, resnet18,34,50,101,152,xresnet18,34,50,101,152,squeezenet1_0,squeezenet1_1, densenet121
# learner = cnn_learner(dataset, models.resnet18, wd=cfg.optimizer.weight_decay,metrics=metrics)
# learner = unet_learner(dataset, models.resnet34, metrics=partial(foreground_acc,void_code=30), wd=cfg.optimizer.weight_decay)
# GAN
# generator = basic_generator(in_size=64, n_channels=3, n_extra_layers=1)
# critic = basic_critic(in_size=64, n_channels=3, n_extra_layers=1)
# learner = GANLearner.wgan(dataset, generator, critic, switch_eval=False,opt_func=partial(optim.Adam, betas=(0., 0.99)), wd=0.)
# NLP
# languagemodellearner()
# textclassifierlearner()
def train(image_width,
dim_x,
dim_z,
encoder_type,
decoder,
dataset,
learning_rate=0.0001,
optimizer=tf.train.AdamOptimizer,
loss=elbo_loss,
batch_size=100,
results_dir='results',
max_epochs=10,
n_view=10,
results_file=None,
bn=False,
**kwargs):
saved_variables = kwargs.pop('saved_variables', None)
anneal_lr = kwargs.pop('anneal_lr', False)
learning_rate_temperature = kwargs.pop('learning_rate_temperature', None)
global_step = tf.Variable(0, trainable=False) # for checkpoint saving
on_epoch = tf.placeholder(tf.float32, name='on_epoch')
dt = datetime.datetime.now()
results_file = results_file if results_file is not None else '/{}_{:02d}-{:02d}-{:02d}'.format(
dt.date(), dt.hour, dt.minute, dt.second)
results_dir += results_file
os.mkdir(results_dir)
# Get all the settings and save them.
with open(results_dir + '/settings.txt', 'w') as f:
args = inspect.getargspec(train).args
#print("locals= ", locals())
#print("args= ", args)
#print("locals()['image_width'] = ", locals()['image_width'])
#print("locals()['image_width'] = ", locals()[args[0]])
#for arg in args: # ERROR SOMEWHERE
#print("arg= ", arg, ", locals= ", locals()[arg] )
lll = locals(
) # BUG in Python 3? Cannot write: locals()[arg] in a comprehensive list. locals()['image_width'] works in a print statement
#settings = print("locals= ", [lll[arg] for arg in args])
settings = [lll[arg] for arg in args]
for s, arg in zip(settings, args):
setting = '{}: {}'.format(arg, s)
f.write('{}\n'.format(setting))
print(setting)
settings = locals()[inspect.getargspec(train).keywords]
for kw, val in settings.items():
setting = '{}: {}'.format(kw, val)
f.write('{}\n'.format(setting))
print(setting)
# Make the neural neural_networks
# GE: There is also a cnn (not used)
# GE: change from tanh to relu or elu?
# GE: nn and cnn are defined in neural_network.py
is_training = tf.placeholder(tf.bool)
if bn:
encoder_net = lambda x: nn(x,
enc_dims,
name='encoder',
act=tf.nn.tanh,
is_training=is_training)
else: # no training
encoder_net = lambda x: nn(
x, enc_dims, name='encoder', act=tf.nn.tanh, is_training=None)
# GE: returns a lambda function:
# lambda x, e: _nf_encoder(x, e, neural_net, dim_z, flow, use_c)
# GE: where _nf_encoder is "encoder_net"
# GE: encoder_net: nn, cnn, conv_net
# GE: encoder_type: nf_encoder, iaf_encoder, ...
# GE: what is flow? Number of NF layers.
encoder = encoder_type(encoder_net, dim_z, flow)
# Build computation graph and operations
x = tf.placeholder(tf.float32, [None, dim_x], 'x')
x_w = tf.placeholder(tf.float32, [None, dim_x], 'x_w')
e = tf.placeholder(tf.float32, (None, dim_z), 'noise')
z_params, z = encoder(x_w, e)
x_pred = decoder(z)
kl_weighting = 1.0 - tf.exp(
-on_epoch / kl_annealing_rate) if kl_annealing_rate is not None else 1
monitor_functions = loss(x_pred, x, kl_weighting=kl_weighting, **z_params)
#monitor_functions_sorted = sorted(monitor_functions.iteritems(), key=lambda x: x[0]) # python 2.x only
monitor_functions_sorted = sorted(monitor_functions.items(),
key=lambda x: x[0]) #python 2 and 3
#monitor_output_train = {name: [] for name in monitor_functions.iterkeys()} # python 2
#monitor_output_valid = {name: [] for name in monitor_functions.iterkeys()} # python 2
monitor_output_train = {name: [] for name in monitor_functions} # python 3
monitor_output_valid = {name: [] for name in monitor_functions} # python 3
monitor_function_names = [p[0] for p in monitor_functions_sorted]
monitor_function_list = [p[1] for p in monitor_functions_sorted]
for i in range(len(monitor_function_names)):
print(
"monitor_function_names/list= {0:20s}, ".format(
monitor_function_names[i]), monitor_function_list[i])
#print(monitor_functions)
train_loss, valid_loss = monitor_functions[
'train_loss'], monitor_functions['valid_loss']
out_op = x_pred
# Batch normalization stuff
# One of the default argumetns to batch_norm: updates_collections=ops.GraphKeys.UPDATE_OPS,
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if update_ops:
updates = tf.group(*update_ops)
# https://stackoverflow.com/questions/43060206/what-does-control-flow-ops-with-dependencies-mean-for-tensoflow
# only evaluate train_loss once updates is updated
train_loss = control_flow_ops.with_dependencies([updates], train_loss)
# Optimizer with gradient clipping
lr = tf.Variable(learning_rate)
optimizer = optimizer(lr)
gvs = optimizer.compute_gradients(
train_loss) # gvs is a list of dictionaries
#for k in range(len(gvs)):
#print("k= ", gvs[k])
# https://www.tensorflow.org/api_docs/python/tf/clip_by_norm
capped_gvs = [(tf.clip_by_norm(grad, 1), var) if grad is not None else
(grad, var) for grad, var in gvs]
train_op = optimizer.apply_gradients(capped_gvs)
# Make training and validation sets
training_data, validation_data = dataset['train'], dataset['valid']
n_train_batches = training_data.images.shape[
0] // batch_size, # python 3 (// integer division)
n_valid_batches = validation_data.images.shape[0] // batch_size,
print('Loaded training and validation data')
visualized = validation_data.images[:n_view]
e_visualized = np.random.normal(0, 1, (n_view, dim_z)) ## GE: ???
# Make summaries
# rec_summary = tf.image_summary("rec", vec2im(out_op, batch_size, image_width), max_images=10) # tf 0.12
# images are 4D: batch, heigh, width, channels (gray, RGB, RGBA)
rec_summary = tf.summary.image("rec",
vec2im(out_op, batch_size, image_width),
max_outputs=10) # tf 1.x
for fn_name, fn in monitor_functions.items():
#tf.scalar_summary(fn_name, fn) # python 2.x
tf.summary.scalar(fn_name, fn) # python 3.x
#summary_op = tf.merge_all_summaries() # python 2.x
summary_op = tf.summary.merge_all() # python 3.x
# Create a saver.
saver = tf.train.Saver(tf.all_variables())
# Create a session
sess = tf.InteractiveSession()
sess.run(tf.initialize_all_variables())
# Use pre-trained weight values
if saved_variables is not None:
restore.set_variables(sess, saved_variables)
#summary_writer = tf.train.SummaryWriter(results_dir, sess.graph) # TF 0.12
summary_writer = tf.summary.FileWriter(results_dir, sess.graph) # TF 1.x
samples_list = []
batch_counter = 0
best_validation_loss = 1e100
number_of_validation_failures = 0
feed_dict = {}
validation_losses, training_losses = [], []
for epoch in range(max_epochs):
feed_dict[on_epoch] = epoch
start_time = time.time()
l_t = 0
monitor_output_epoch = {name: 0
for name in monitor_function_names} # GE: ???
for _ in range(n_train_batches):
batch_counter += 1
# whitened: False (
feed_dict[x], feed_dict[x_w] = training_data.next_batch(
batch_size, whitened=False)
feed_dict[e] = np.random.normal(0, 1, (batch_size, dim_z))
feed_dict[is_training] = True
output = sess.run([train_op, train_loss] + monitor_function_list,
feed_dict=feed_dict)
l, monitor_output_batch = output[1], output[2:]
for name, out in zip(monitor_function_names, monitor_output_batch):
monitor_output_epoch[name] += out
if batch_counter % 100 == 0:
summary_str = sess.run(summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, batch_counter)
# Save the model checkpoint periodically.
if batch_counter % 1000 == 0 or epoch == max_epochs:
checkpoint_path = os.path.join(results_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=global_step)
l_t += l
l_t /= n_train_batches
for name in monitor_function_names:
monitor_output_train[name].append(monitor_output_epoch[name] /
n_train_batches)
training_losses.append(l_t)
# Validation loop
l_v = 0
monitor_output_epoch = {name: 0 for name in monitor_function_names}
#for _ in range(n_valid_batches):
for _ in range(n_valid_batches):
feed_dict[x], feed_dict[x_w] = validation_data.next_batch(
batch_size, whitened=False)
feed_dict[e] = np.random.normal(0, 1, (batch_size, dim_z))
feed_dict[is_training] = False
output = sess.run([valid_loss] + monitor_function_list,
feed_dict=feed_dict)
l_v_batched, monitor_output_batch = output[0], output[1:]
for name, out in zip(monitor_function_names, monitor_output_batch):
monitor_output_epoch[name] += out
l_v += l_v_batched
l_v /= n_valid_batches
for name in monitor_function_names:
monitor_output_valid[name].append(monitor_output_epoch[name] /
n_valid_batches)
validation_losses.append(l_v)
duration = time.time() - start_time
examples_per_sec = (n_valid_batches +
n_train_batches) * batch_size * 1.0 / duration
print(
'Epoch: {:d}\t Weighted training loss: {:.2f}, Validation loss {:.2f} ({:.1f} examples/sec, {:.1f} sec/epoch)'
.format(epoch, l, l_v, examples_per_sec, duration))
samples = sess.run([out_op],
feed_dict={
x: visualized,
x_w: visualized,
e: e_visualized,
is_training: False
})
samples = np.reshape(samples, (n_view, image_width, image_width))
samples_list.append(samples)
# show_samples(samples, image_width)
# Learning rate annealing
lr = lr / (1.0 + epoch * 1.0 / learning_rate_temperature
) if learning_rate_temperature is not None else lr
if epoch % 100 == 0:
np.save(results_dir + '/validation_losses_{}.npy'.format(epoch),
validation_losses)
np.save(results_dir + '/training_losses_{}.npy'.format(epoch),
training_losses)
np.save(
results_dir + '/sample_visualizations_{}.npy'.format(epoch),
np.array(samples_list))
np.save(results_dir + '/real_visualizations_{}.npy'.format(epoch),
np.reshape(visualized, (n_view, image_width, image_width)))
for name in monitor_function_names:
np.save(results_dir + '/{}_valid_{}.npy'.format(name, epoch),
monitor_output_valid[name])
np.save(results_dir + '/{}_train_{}.npy'.format(name, epoch),
monitor_output_train[name])
np.save(results_dir + '/validation_losses.npy', validation_losses)
np.save(results_dir + '/training_losses.npy', training_losses)
np.save(results_dir + '/sample_visualizations.npy', np.array(samples_list))
np.save(results_dir + '/real_visualizations.npy',
np.reshape(visualized, (n_view, image_width, image_width)))
for name in monitor_function_names:
np.save(results_dir + '/{}_valid.npy'.format(name),
monitor_output_valid[name])
np.save(results_dir + '/{}_train.npy'.format(name),
monitor_output_train[name])
visualize = False
if visualize:
for samples in samples_list:
together = np.hstack(
(np.reshape(visualized, (n_view, image_width, image_width)),
samples > 0.5))
plot_images_together(together)
sess.close()
def maxwalksat_f(self,model):
model = self.model
re=''
max_energy=-sys.maxint
current_energy=0.0
max_scored_dimension=-1
max_scored_point=[]
self.emin,self.emax =model.normalise_val()
list_energy=[]
a12_small=0.56
prev_energy=[0 for _ in range(21)]
#print self.emax , self.emin
for i in range(1,self.max_tries):
re=''
variable = self.model.candidates(0,False,0)
for j in range (1,self.max_changes):
append=' .'
current_energy=model.Energy(variable,self.emax,self.emin)
if (current_energy > self.threshold):
print "maximum energy is ",current_energy," maximum scored point is ",variable
return variable
r= random.random()
if (self.p <r ):
variable,current_energy,max_scored_dimension=self.roll_marbles_everywhere(variable,model)
if (current_energy>max_energy):
max_scored_point,max_energy=variable,current_energy
append=' !'
else:
append=' +'
list_energy.append(current_energy)
else:
if(max_scored_dimension>-1):
variable=self.roll_marbles_one_direction(max_scored_dimension,variable,model)
current_energy= model.Energy(variable,self.emax,self.emin)
append=' .'
if (current_energy>max_energy):
max_scored_point,max_energy=variable,current_energy
list_energy.append(current_energy)
re=re+append
if (j==25):
print i*j ,max_energy,re
re=''
l1=list_energy
l2=prev_energy
# print l1
#print l2
a12_result=a12(l2,l1)
#print a12_result
if (a12_result>a12_small):
j =j +5
else :
j=j -1
prev_energy=list_energy
list_energy=[]
l=''
if (i==1):
first_energy=prev_energy
final_energy=prev_energy
x=loss(final_energy,first_energy)
global_variable.loss_inter=x
print "maximum energy is ",max_energy," maximum scored point is ",max_scored_point
def simulated_annealer(self,model):
self.emin,self.emax =model.normalise_val()
#print self.emax , self.emin
sol0= self.model.candidates(0,False,0)
sol0Energy= model.Energy(sol0,self.emax,self.emin)
best=sol0
ebest=sol0Energy
l=''
k=1
# d_energy=dict()
# count_dict=0
list_energy=[]
a12_small=0.56
prev_energy=[0 for _ in range(21)]
list_loss= []
# print prev_energy
while(k<=self.kmax):
new=' .'
solNear=self.model.neighbour(sol0)
solNearEnergy=model.Energy(solNear,self.emax,self.emin)
if (sol0Energy<ebest):
new=' !'
best,ebest=sol0,sol0Energy
sol0=solNear
elif (solNearEnergy<sol0Energy):
new=' +'
sol0,sol0Energy= solNear,solNearEnergy
#if (sol0Energy<ebest):
# best,ebest=sol0,sol0Energy
elif (self.prob_jump(solNearEnergy,sol0Energy,((float(k)/float(self.kmax)*self.cooling)))):
new=' ?'
sol0,sol0Energy= solNear,solNearEnergy
#if (sol0Energy<ebest):
# best,ebest=sol0,sol0Energy
l=l+new
list_energy.append(sol0Energy)
if (k>0):
if ((k%25)==0):
print k,ebest,l
# d_energy[count_dict]=list_energy
l1=list_energy
l2=prev_energy
# print l1
#print l2
a12_result=a12(l2,l1)
if (a12_result>a12_small):
self.kmax =self.kmax +5
else :
self.kmax=self.kmax -1
prev_energy=list_energy
# print prev_energy
list_energy=[]
l=''
# count_dict=count_dict+1
if (k==25):
first_energy=prev_energy
k=k +1
final_energy=prev_energy
print final_energy,first_energy
x=loss(final_energy,first_energy)
global_variable.loss_inter=x
# print d_energy[count_dict]
print "kmax=" , self.kmax ,"cooling =" ,self.cooling ,"energy_minimum=" , ebest, "best point=" ,best
def GeneticAlgorithm_f(self, model, parameters):
model = self.model
self.emin, self.emax = self.model.normalise_val()
self.kmax = parameters[0]
self.population_Size = parameters[1]
self.p_crossover = parameters[2]
self.p_mutation = parameters[3]
self.frontier_distribution = parameters[4]
#print self.emin,self.emax
population_frontier = [
self.model.candidates(0, False, 0)
for _ in range(self.population_Size)
]
population_frontier_old = self.calculate_best_frontier(
population_frontier)
first_frontier = population_frontier_old
pop_front_best = population_frontier_old[:]
#print len(population_frontier_old)
#print population_frontier_old
k = 0
life = 0
while k < self.kmax:
i = 0
children = []
for i in xrange(self.population_Size):
parent1, parent2 = self.selectParents(
population_frontier_old, len(population_frontier_old))
child1, child2 = self.crossover(parent1, parent2,
self.p_crossover, model)
child1 = self.mutate(child1, self.p_mutation, model)
child2 = self.mutate(child2, self.p_mutation, model)
if (self.bin_dom(child1, child2)):
children.append(child1)
elif (self.bin_dom(child1, child2)):
children.append(child2)
else:
x = random.randint(0, 100)
if (x % 2 == 0):
children.append(child2)
else:
children.append(child1)
population_frontier_new = []
for can1 in population_frontier:
flag = True
for can2 in population_frontier:
if (can1 == can2):
continue
if (self.bin_dom(can1, can2)):
flag = False
break
if flag:
population_frontier_new.append(can1)
tmp = []
new_frontier = False
for a in population_frontier_new:
for b in pop_front_best:
if self.bin_dom(a, b):
tmp.append(a)
pop_front_best.remove(b)
if tmp:
pop_front_best.extend(tmp)
new_frontier = True
if new_frontier:
life = 0
else:
life = life + 1
if (life == self.lifes):
break
population_frontier = children
population_frontier_old = population_frontier_new
k = k + 1
# print population_frontier
#print first_frontier
x = loss(population_frontier[0], first_frontier[0])
global_variable.pop_front_best = pop_front_best
'''name='Pareto_Fronts/pareto.txt'
f=open(name,'w')
i=0
for i in xrange(len(pop_front_best)):
k=0
for j in (self.model.objectives(pop_front_best[i])):
j=(j - self.emin[k]) / (self.emax[k]- self.emin[k])
k+=1
s=str(j)
f.write(s)
f.write(" ")
f.write("\n")
f.close();
# hypervol=HyperVolume_wrapper()
os.remove('./Pareto_Fronts/pareto.txt')'''
#print hypervol
return x
batch_size = 128
log_device_placement = False
with tf.Graph().as_default():
global_step = tf.contrib.framework.get_or_create_global_step()
# Generating images and its labels
# build_inputs('cifar10/cifar100', cifar dataset dir, batch size, mode)
images, labels = cifar_input.build_input(
'cifar10', '../../cifar/cifar10/data_batch*', batch_size, 'train')
# Creating graph. NUM_CLASESS=10 (CIFAR-10) or NUM_CLASESS=100 (CIFAR-100)
logits = inference(images, NUM_CLASSES=10)
# Loss/Error and Accuracy
losses = loss(logits, labels)
accuracies = accuracy(logits, labels)
# Our train_op (Only minimizing loss)
train_op = train(losses, global_step, batch_size)
# SessionRunHook. Logging will be done each x steps.
class _LoggerHook(tf.train.SessionRunHook):
def begin(self):
self._step = -1
# Creating train_dir if it does not exist and writing to log file
if not os.path.exists(train_dir):
os.makedirs(train_dir)
open(train_dir + 'training_data.csv', 'w').close()
f = open(train_dir + "log.txt", 'ab')
f.write('\n\n===============================\n')
def train_net(args):
ctx = []
for ctx_id in [int(x) for x in args.gpus.split(',')]:
ctx.append(mx.gpu(ctx_id))
print('gpu num:', len(ctx))
prefix = args.prefix
prefix_dir = os.path.dirname(prefix)
if not os.path.exists(prefix_dir):
os.makedirs(prefix_dir)
end_epoch = args.end_epoch
args.ctx_num = len(ctx)
args.num_layers = int(args.network[1:])
print('num_layers', args.num_layers)
assert args.batch_size % args.ctx_num == 0
args.per_batch_size = args.batch_size // args.ctx_num
args.image_channel = 3
data_dir = args.data_dir
print('data dir', data_dir)
path_imgrec = None
path_imglist = None
for line in open(os.path.join(data_dir, 'property')):
vec = line.strip().split(',')
assert len(vec) == 3
args.num_classes = int(vec[0])
image_size = [int(vec[1]), int(vec[2])]
args.image_h = image_size[0]
args.image_w = image_size[1]
print('image_size', image_size)
assert (args.num_classes > 0)
print('num_classes', args.num_classes)
path_imgrec = os.path.join(data_dir, "train.rec")
print('Called with argument:', args)
data_shape = (args.image_channel, image_size[0], image_size[1])
mean = None
begin_epoch = 0
#feat_net = fresnet.get(100, 256)
#margin_block = ArcMarginBlock(args)
net = TrainBlock(args)
net.collect_params().reset_ctx(ctx)
net.hybridize()
#initializer = mx.init.Xavier(rnd_type='gaussian', factor_type="out", magnitude=2) #resnet style
#feat_net.initialize(ctx=ctx, init=initializer)
#feat_net.hybridize()
#margin_block.initialize(ctx=ctx, init=mx.init.Normal(0.01))
#margin_block.hybridize()
ds = FaceDataset(data_shape=(3, 112, 112), path_imgrec=path_imgrec)
#print(len(ds))
#img, label = ds[0]
#print(img.__class__, label.__class__)
#print(img.shape, label)
loader = gluon.data.DataLoader(ds,
batch_size=args.batch_size,
shuffle=True,
num_workers=8,
last_batch='discard')
metric = CompositeEvalMetric([AccMetric()])
ver_list = []
ver_name_list = []
if args.task == '':
for name in args.eval.split(','):
path = os.path.join(data_dir, name + ".bin")
if os.path.exists(path):
print('loading ver-set:', name)
data_set = verification.load_bin(path, image_size)
ver_list.append(data_set)
ver_name_list.append(name)
def ver_test(nbatch, tnet):
results = []
for i in range(len(ver_list)):
xnorm, acc, thresh = verification.easytest(
ver_list[i], tnet, ctx, batch_size=args.batch_size)
print('[%s][%d]Accuracy-Thresh-XNorm: %.5f - %.5f - %.5f' %
(ver_name_list[i], nbatch, acc, thresh, xnorm))
#print('[%s][%d]Accuracy: %1.5f+-%1.5f' % (ver_name_list[i], nbatch, acc1, std1))
#print('[%s][%d]Accuracy-Flip: %1.5f+-%1.5f' % (ver_name_list[i], nbatch, acc2, std2))
results.append(acc)
return results
total_time = 0
num_epochs = 0
best_acc = [0]
highest_acc = [0.0, 0.0] #lfw and target
global_step = [0]
save_step = [0]
lr_steps = [20000, 28000, 32000]
if args.num_classes >= 20000:
lr_steps = [100000, 160000, 220000]
print('lr_steps', lr_steps)
kv = mx.kv.create('device')
trainer = gluon.Trainer(net.collect_params(),
'sgd', {
'learning_rate': args.lr,
'wd': args.wd,
'momentum': args.mom,
'multi_precision': True
},
kvstore=kv)
def _batch_callback():
mbatch = global_step[0]
global_step[0] += 1
for _lr in lr_steps:
if mbatch == _lr:
args.lr *= 0.1
if args.mode == 'gluon':
trainer.set_learning_rate(args.lr)
else:
opt.lr = args.lr
print('lr change to', args.lr)
break
#_cb(param)
if mbatch % 1000 == 0:
print('lr-batch-epoch:', args.lr, mbatch)
if mbatch > 0 and mbatch % args.verbose == 0:
save_step[0] += 1
msave = save_step[0]
do_save = False
is_highest = False
tnet = TestBlock(args, params=net.collect_params())
tnet.hybridize()
acc_list = ver_test(mbatch, tnet)
if len(acc_list) > 0:
lfw_score = acc_list[0]
if lfw_score > highest_acc[0]:
highest_acc[0] = lfw_score
if acc_list[-1] >= highest_acc[-1]:
highest_acc[-1] = acc_list[-1]
if args.ckpt == 0:
do_save = False
elif args.ckpt == 1:
do_save = True
msave = 1
elif args.ckpt > 1:
do_save = True
if do_save:
print('saving', msave)
#print('saving gluon params')
fname = args.prefix + "-gluon.params"
tnet.save_parameters(fname)
tnet.export(args.prefix, msave)
#arg, aux = model.get_params()
#mx.model.save_checkpoint(prefix, msave, model.symbol, arg, aux)
print('[%d]Accuracy-Highest: %1.5f' % (mbatch, highest_acc[-1]))
if args.max_steps > 0 and mbatch > args.max_steps:
sys.exit(0)
loss_weight = 1.0
#loss = gluon.loss.SoftmaxCrossEntropyLoss(weight = loss_weight)
#loss = nd.SoftmaxOutput
loss = gluon.loss.SoftmaxCrossEntropyLoss()
while True:
#trainer = update_learning_rate(opt.lr, trainer, epoch, opt.lr_factor, lr_steps)
tic = time.time()
#train_iter.reset()
metric.reset()
btic = time.time()
#for i, batch in enumerate(train_iter):
for batch_idx, (x, y) in enumerate(loader):
#print(x.shape, y.shape)
_batch_callback()
#data = gluon.utils.split_and_load(batch.data[0].astype(opt.dtype), ctx_list=ctx, batch_axis=0)
#label = gluon.utils.split_and_load(batch.label[0].astype(opt.dtype), ctx_list=ctx, batch_axis=0)
data = gluon.utils.split_and_load(x, ctx_list=ctx, batch_axis=0)
label = gluon.utils.split_and_load(y, ctx_list=ctx, batch_axis=0)
outputs = []
losses = []
with ag.record():
for _data, _label in zip(data, label):
#print(y.asnumpy())
fc7 = net(_data, _label)
#print(z[0].shape, z[1].shape)
losses.append(loss(fc7, _label))
outputs.append(fc7)
for l in losses:
l.backward()
#trainer.step(batch.data[0].shape[0], ignore_stale_grad=True)
#trainer.step(args.ctx_num)
n = x.shape[0]
#print(n,n)
trainer.step(n)
metric.update(label, outputs)
i = batch_idx
if i > 0 and i % 20 == 0:
name, acc = metric.get()
if len(name) == 2:
logger.info(
'Epoch[%d] Batch [%d]\tSpeed: %f samples/sec\t%s=%f, %s=%f'
% (num_epochs, i, args.batch_size /
(time.time() - btic), name[0], acc[0], name[1],
acc[1]))
else:
logger.info(
'Epoch[%d] Batch [%d]\tSpeed: %f samples/sec\t%s=%f' %
(num_epochs, i, args.batch_size /
(time.time() - btic), name[0], acc[0]))
#metric.reset()
btic = time.time()
epoch_time = time.time() - tic
# First epoch will usually be much slower than the subsequent epics,
# so don't factor into the average
if num_epochs > 0:
total_time = total_time + epoch_time
#name, acc = metric.get()
#logger.info('[Epoch %d] training: %s=%f, %s=%f'%(num_epochs, name[0], acc[0], name[1], acc[1]))
logger.info('[Epoch %d] time cost: %f' % (num_epochs, epoch_time))
num_epochs = num_epochs + 1
#name, val_acc = test(ctx, val_data)
#logger.info('[Epoch %d] validation: %s=%f, %s=%f'%(epoch, name[0], val_acc[0], name[1], val_acc[1]))
# save model if meet requirements
#save_checkpoint(epoch, val_acc[0], best_acc)
if num_epochs > 1:
print('Average epoch time: {}'.format(
float(total_time) / (num_epochs - 1)))
def train_eval(p_dict, phase='train'):
### 传入参数
epoch = p_dict['epoch']
model = p_dict['model'] # 模型
loss = p_dict['loss'] # loss 函数
if phase == 'train':
data_loader = p_dict['train_loader'] # 训练数据
optimizer = p_dict['optimizer'] # 优化器
else:
data_loader = p_dict['val_loader']
### 局部变量定义
classification_metric_dict = dict()
# if args.task == 'case1':
for i, data in enumerate(tqdm(data_loader)):
if args.use_visit:
if args.gpu:
data = [Variable(x.cuda()) for x in data]
visits, values, mask, master, labels, times, trends = data
if i == 0:
print 'input size', visits.size()
output = model(visits, master, mask, times, phase, values, trends)
else:
inputs = Variable(data[0].cuda())
labels = Variable(data[1].cuda())
output = model(inputs)
# if 0:
if args.task == 'task2':
output, mask, time = output
labels = labels.unsqueeze(-1).expand(output.size()).contiguous()
labels[mask == 0] = -1
else:
time = None
classification_loss_output = loss(output, labels, args.hard_mining)
loss_gradient = classification_loss_output[0]
# 计算性能指标
function.compute_metric(output, labels, time,
classification_loss_output,
classification_metric_dict, phase)
# print(outputs.size(), labels.size(),data[3].size(),segment_line_output.size())
# print('detection', detect_character_labels.size(), detect_character_output.size())
# return
# 训练阶段
if phase == 'train':
optimizer.zero_grad()
loss_gradient.backward()
optimizer.step()
# if i >= 10:
# break
print('\nEpoch: {:d} \t Phase: {:s} \n'.format(epoch, phase))
metric = function.print_metric('classification',
classification_metric_dict, phase)
if args.phase != 'train':
print 'metric = ', metric
print
print
return
if phase == 'val':
if metric > p_dict['best_metric'][0]:
p_dict['best_metric'] = [metric, epoch]
function.save_model(p_dict)
if 0:
# if args.task == 'task2':
preds = classification_metric_dict['preds']
labels = classification_metric_dict['labels']
times = classification_metric_dict['times']
fl = open('../result/tauc_label.csv', 'w')
fr = open('../result/tauc_result.csv', 'w')
fl.write('adm_id,last_event_time,mortality\n')
fr.write('adm_id,probability\n')
for i, (p, l, t) in enumerate(zip(preds, labels, times)):
if i % 30:
continue
fl.write(str(i) + ',')
fl.write(str(t) + ',')
fl.write(str(int(l)) + '\n')
fr.write(str(i) + ',')
fr.write(str(p) + '\n')
print('valid: metric: {:3.4f}\t epoch: {:d}\n'.format(metric, epoch))
print('\t\t\t valid: best_metric: {:3.4f}\t epoch: {:d}\n'.format(
p_dict['best_metric'][0], p_dict['best_metric'][1]))
else:
print('train: metric: {:3.4f}\t epoch: {:d}\n'.format(metric, epoch))
def train_ssd():
trn_id_fname, trn_id_annotation, trn_id_single_anno, idx_category, category_idx, imgs, imgs_id, imgs_bbox, imgs_class = (
get_anno_data())
config = Config("remote")
test_dataset = VOC_dataset(config.voc2007_root, config.voc2012_root,
config.voc2007_test_anno, "test")
trn_dataset = VOC_dataset(config.voc2007_root, config.voc2012_root,
config.anno_path, "trn")
test_dataloader = DataLoader(
test_dataset,
batch_size=1,
shuffle=False,
num_workers=8,
collate_fn=detection_collate_fn,
)
trn_dataloader = DataLoader(
trn_dataset,
batch_size=32,
shuffle=True,
num_workers=8,
collate_fn=detection_collate_fn,
)
ssd_model = get_SSD_model(1, config.vgg_weight_path,
config.vgg_reduced_weight_path)
use_cuda = torch.cuda.is_available()
device = torch.device("cuda:0" if use_cuda else "cpu")
ssd_model = ssd_model.to(device)
prior_box = get_prior_box()
# prior_box = prior_box.to(device)
loss_array = []
val_array = []
print("success build ssd model to train")
optimizer = torch.optim.SGD(ssd_model.parameters(),
lr=1e-3,
momentum=0.9,
weight_decay=5 * 1e-4)
lr_scheduler = optim.lr_scheduler.LambdaLR(optimizer, adjust_lr)
for epoch in range(120):
lr_scheduler.step()
# train
for i, batch in enumerate(trn_dataloader):
imgs, bboxes, labels, img_id, ignores, img_scale = batch
# bboxes, labels, img_id, ignores, img_scale = bboxes[0], labels[0], img_id[0], ignores[0], img_scale[0]
imgs = imgs.to(device)
# bboxes = bboxes.to(device)
cls_preds, loc_preds = ssd_model(imgs)
ssd_model.zero_grad()
total_loss = 0
total_loc_loss, total_cls_loss = 0, 0
for idx in range(imgs.shape[0]):
img, bbox, label = imgs[idx], bboxes[idx], labels[idx]
cls_pred, loc_pred = cls_preds[idx], loc_preds[idx]
iou = get_iou(bbox, prior_box)
pos_mask, cls_target, bbox_target = get_target(
iou, prior_box, img, bbox, label)
pos_mask, cls_target, bbox_target = (
pos_mask.to(device),
cls_target.to(device),
bbox_target.to(device),
)
loss_loc, loss_cls = loss(cls_pred, loc_pred, pos_mask,
cls_target, bbox_target)
total_loc_loss += loss_loc
total_cls_loss += loss_cls
total_loss += loss_cls + loss_loc
total_loss /= float(imgs.shape[0])
total_cls_loss /= float(imgs.shape[0])
total_loc_loss /= float(imgs.shape[0])
total_loss.backward()
optimizer.step()
cls_loss = round(float(total_cls_loss), 3)
loc_loss = round(float(total_loc_loss), 3)
t_loss = round(float(total_loss), 3)
if i % 5 == 0:
print(
epoch * 515 + i,
"cls_loss: {}, loc_loss: {}, loss: {}".format(
cls_loss, loc_loss, t_loss),
)
loss_array.append(t_loss)
# val and save every 5 epoch
if epoch % 5 == 0:
torch.save(ssd_model.state_dict(), "f_trained_{}_epoch".format(i))
print("val--------------------------")
for val_i, batch in enumerate(test_dataloader):
imgs, bboxes, labels, img_id, ignores, img_scale = batch
imgs = imgs.to(device)
cls_preds, loc_preds = ssd_model(imgs)
total_loss = 0
total_loc_loss, total_cls_loss = 0, 0
for idx in range(imgs.shape[0]):
img, bbox, label = imgs[idx], bboxes[idx], labels[idx]
cls_pred, loc_pred = cls_preds[idx], loc_preds[idx]
iou = get_iou(bbox, prior_box)
pos_mask, cls_target, bbox_target = get_target(
iou, prior_box, img, bbox, label)
pos_mask, cls_target, bbox_target = (
pos_mask.to(device),
cls_target.to(device),
bbox_target.to(device),
)
loss_loc, loss_cls = loss(cls_pred, loc_pred, pos_mask,
cls_target, bbox_target)
total_loc_loss += loss_loc
total_cls_loss += loss_cls
total_loss += loss_cls + loss_loc
total_loss /= float(imgs.shape[0])
total_cls_loss /= float(imgs.shape[0])
total_loc_loss /= float(imgs.shape[0])
cls_loss = round(float(total_cls_loss), 3)
loc_loss = round(float(total_loc_loss), 3)
t_loss = round(float(total_loss), 3)
if val_i % 100 == 0:
print(
val_i,
"cls_loss: {}, loc_loss: {}, loss: {}".format(
cls_loss, loc_loss, t_loss),
)
val_array.append(t_loss)
# valdiate the mAP
print("mAP")
mean_average_precision = mAP(config.voc2007_test_anno)
for i, batch in tqdm_notebook(enumerate(test_dataloader)):
imgs, bboxes, labels, img_id, ignores, img_scale = batch
bboxes, labels, img_id, ignores, img_scale = (
bboxes[0],
labels[0],
img_id[0],
ignores[0],
img_scale[0],
)
imgs = imgs.to(device)
cls_preds, loc_preds = ssd_model(imgs)
res_score, res_bbox, res_cls = mean_average_precision.nms(
cls_preds, loc_preds, prior_box, conf_threshold=0.01)
for _ in range(len(bboxes)):
bboxes[_][1] *= 300
bboxes[_][3] *= 300
bboxes[_][0] *= 300
bboxes[_][2] *= 300
for _ in range(len(bboxes)):
mean_average_precision.add_single_gt(
img_id, bboxes[_], labels[_], ignores[_])
for _ in range(len(res_score)):
mean_average_precision.add_single_pred(
img_id,
res_score[_].cpu().detach().numpy(),
res_bbox[_].cpu().detach().numpy(),
res_cls[_],
)
res = mean_average_precision.calculate_mAP()
# save loss array and map to log
with open("res.json", "a") as f:
json.dump({"epoch": epoch, "loss": loss_array, "map": res}, f)
f.write("\n")
print("mAP: ", np.mean([res[k] for k in res.keys()]))
print("finish val--------------------------")
def DifferentialEvolution_f(self, model):
self.emin, self.emax = model.normalise_val()
model = self.model
re = ""
frontier = [model.candidates(0, False, 0) for _ in range(self.max_changes)]
best_energy = model.Energy(frontier[0], self.emax, self.emin)
best_solution = frontier[0]
k = 0
list_energy = []
a12_small = 0.56
prev_energy = [0 for _ in range(21)]
while k < self.kmax:
re = ""
if best_energy == self.threshold:
break
for i, solution in enumerate(frontier):
seen = []
while len(seen) < 3:
rand_index = random.randint(0, 99)
if rand_index == i:
continue
if rand_index not in seen:
seen.append(rand_index)
mutation = frontier[seen[0]]
current_energy = model.Energy(solution, self.emax, self.emin)
append = " ."
if self.crossover < random.random():
if model.Energy(mutation, self.emax, self.emin) < current_energy:
current_energy = model.Energy(mutation, self.emax, self.emin)
frontier[i] = mutation
append = " +"
else:
mutation = []
for j in xrange(model.n):
l = model.lo[j]
m = model.hi[j]
inter = frontier[seen[0]][j] + self.f * (frontier[seen[1]][j] - frontier[seen[2]][j])
if inter >= l and inter <= m:
mutation.append(inter)
else:
mutation.append(frontier[seen[random.randint(0, 2)]][j])
if model.ok(mutation) and model.Energy(mutation, self.emax, self.emin) < current_energy:
frontier[i] = mutation
current_energy = model.Energy(mutation, self.emax, self.emin)
append = " +"
if current_energy < best_energy and current_energy >= self.threshold:
append = " ?"
best_energy = current_energy
best_solution = frontier[i]
list_energy.append(current_energy)
re = re + append
k += 1
if k > 0:
if k == 25:
first_energy = prev_energy
if k % 25 is 0:
print k, best_energy, re
re = ""
l1 = list_energy
l2 = prev_energy
# print l1,l2
# print l1
# print l2
a12_result = a12(l1, l2)
# print a12_result
if a12_result > a12_small:
k = k + 5
else:
k = k - 1
prev_energy = list_energy
list_energy = []
l = ""
final_energy = prev_energy
x = loss(final_energy, first_energy)
global_variable.loss_inter = x
print ("\nBest Solution : " + str(best_solution))
print ("Best Energy : " + str((model.Energy(best_solution, self.emax, self.emin))))
def train(
image_width,
dim_x,
dim_z,
encoder_type,
decoder,
dataset,
learning_rate=0.0001,
optimizer=tf.train.AdamOptimizer,
loss=elbo_loss,
batch_size=100,
results_dir='results',
max_epochs=10,
n_view=10,
results_file=None,
bn=False,
**kwargs
):
saved_variables = kwargs.pop('saved_variables', None)
anneal_lr = kwargs.pop('anneal_lr', False)
learning_rate_temperature = kwargs.pop('learning_rate_temperature', None)
global_step = tf.Variable(0, trainable=False) # for checkpoint saving
on_epoch = tf.placeholder(tf.float32, name='on_epoch')
dt = datetime.datetime.now()
results_file = results_file if results_file is not None else '/{}_{:02d}-{:02d}-{:02d}'.format(dt.date(), dt.hour, dt.minute, dt.second)
results_dir += results_file
os.mkdir(results_dir)
# Get all the settings and save them.
with open(results_dir + '/settings.txt', 'w') as f:
args = inspect.getargspec(train).args
settings = [locals()[arg] for arg in args]
for s, arg in zip(settings, args):
setting = '{}: {}'.format(arg, s)
f.write('{}\n'.format(setting))
print(setting)
settings = locals()[inspect.getargspec(train).keywords]
for kw, val in settings.items():
setting = '{}: {}'.format(kw, val)
f.write('{}\n'.format(setting))
print(setting)
# Make the neural neural_networks
is_training = tf.placeholder(tf.bool)
if bn:
encoder_net = lambda x: nn(x, enc_dims, name='encoder', act=tf.nn.tanh, is_training=is_training)
else:
encoder_net = lambda x: nn(x, enc_dims, name='encoder', act=tf.nn.tanh, is_training=None)
encoder = encoder_type(encoder_net, dim_z, flow)
# Build computation graph and operations
x = tf.placeholder(tf.float32, [None, dim_x], 'x')
x_w = tf.placeholder(tf.float32, [None, dim_x], 'x_w')
e = tf.placeholder(tf.float32, (None, dim_z), 'noise')
z_params, z = encoder(x_w, e)
x_pred = decoder(z)
kl_weighting = 1.0 - tf.exp(-on_epoch / kl_annealing_rate) if kl_annealing_rate is not None else 1
monitor_functions = loss(x_pred, x, kl_weighting=kl_weighting, **z_params)
monitor_functions_sorted = sorted(monitor_functions.iteritems(), key=lambda x: x[0])
monitor_output_train = {name: [] for name in monitor_functions.iterkeys()}
monitor_output_valid = {name: [] for name in monitor_functions.iterkeys()}
monitor_function_names = [p[0] for p in monitor_functions_sorted]
monitor_function_list = [p[1] for p in monitor_functions_sorted]
train_loss, valid_loss = monitor_functions['train_loss'], monitor_functions['valid_loss']
out_op = x_pred
# Batch normalization stuff
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if update_ops:
updates = tf.group(*update_ops)
train_loss = control_flow_ops.with_dependencies([updates], train_loss)
# Optimizer with gradient clipping
lr = tf.Variable(learning_rate)
optimizer = optimizer(lr)
gvs = optimizer.compute_gradients(train_loss)
capped_gvs = [(tf.clip_by_norm(grad, 1), var) if grad is not None else (grad, var)
for grad, var in gvs]
train_op = optimizer.apply_gradients(capped_gvs)
# Make training and validation sets
training_data, validation_data = dataset['train'], dataset['valid']
n_train_batches, n_valid_batches = training_data.images.shape[0] / batch_size, validation_data.images.shape[0] / batch_size,
print 'Loaded training and validation data'
visualized, e_visualized = validation_data.images[:n_view], np.random.normal(0, 1, (n_view, dim_z))
# Make summaries
rec_summary = tf.image_summary("rec", vec2im(out_op, batch_size, image_width), max_images=10)
for fn_name, fn in monitor_functions.items():
tf.scalar_summary(fn_name, fn)
summary_op = tf.merge_all_summaries()
# Create a saver.
saver = tf.train.Saver(tf.all_variables())
# Create a session
sess = tf.InteractiveSession()
sess.run(tf.initialize_all_variables())
# Use pre-trained weight values
if saved_variables is not None:
restore.set_variables(sess, saved_variables)
summary_writer = tf.train.SummaryWriter(results_dir, sess.graph)
samples_list = []
batch_counter = 0
best_validation_loss = 1e100
number_of_validation_failures = 0
feed_dict = {}
validation_losses, training_losses = [], []
for epoch in range(max_epochs):
feed_dict[on_epoch] = epoch
start_time = time.time()
l_t = 0
monitor_output_epoch = {name: 0 for name in monitor_function_names}
for _ in xrange(n_train_batches):
batch_counter += 1
feed_dict[x], feed_dict[x_w] = training_data.next_batch(batch_size, whitened=False)
feed_dict[e] = np.random.normal(0, 1, (batch_size, dim_z))
feed_dict[is_training] = True
output = sess.run([train_op, train_loss] + monitor_function_list, feed_dict=feed_dict)
l, monitor_output_batch = output[1], output[2:]
for name, out in zip(monitor_function_names, monitor_output_batch):
monitor_output_epoch[name] += out
if batch_counter % 100 == 0:
summary_str = sess.run(summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, batch_counter)
# Save the model checkpoint periodically.
if batch_counter % 1000 == 0 or epoch == max_epochs:
checkpoint_path = os.path.join(results_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=global_step)
l_t += l
l_t /= n_train_batches
for name in monitor_function_names:
monitor_output_train[name].append(monitor_output_epoch[name] / n_train_batches)
training_losses.append(l_t)
# Validation loop
l_v = 0
monitor_output_epoch = {name: 0 for name in monitor_function_names}
for _ in range(n_valid_batches):
feed_dict[x], feed_dict[x_w] = validation_data.next_batch(batch_size, whitened=False)
feed_dict[e] = np.random.normal(0, 1, (batch_size, dim_z))
feed_dict[is_training] = False
output = sess.run([valid_loss] + monitor_function_list, feed_dict=feed_dict)
l_v_batched, monitor_output_batch = output[0], output[1:]
for name, out in zip(monitor_function_names, monitor_output_batch):
monitor_output_epoch[name] += out
l_v += l_v_batched
l_v /= n_valid_batches
for name in monitor_function_names:
monitor_output_valid[name].append(monitor_output_epoch[name] / n_valid_batches)
validation_losses.append(l_v)
duration = time.time() - start_time
examples_per_sec = (n_valid_batches + n_train_batches) * batch_size * 1.0 / duration
print('Epoch: {:d}\t Weighted training loss: {:.2f}, Validation loss {:.2f} ({:.1f} examples/sec, {:.1f} sec/epoch)'.format(epoch, l, l_v, examples_per_sec, duration))
samples = sess.run([out_op], feed_dict={x: visualized, x_w: visualized, e: e_visualized, is_training: False})
samples = np.reshape(samples, (n_view, image_width, image_width))
samples_list.append(samples)
# show_samples(samples, image_width)
# Learning rate annealing
lr = lr / (1.0 + epoch * 1.0 / learning_rate_temperature) if learning_rate_temperature is not None else lr
if epoch % 100 == 0:
np.save(results_dir + '/validation_losses_{}.npy'.format(epoch), validation_losses)
np.save(results_dir + '/training_losses_{}.npy'.format(epoch), training_losses)
np.save(results_dir + '/sample_visualizations_{}.npy'.format(epoch), np.array(samples_list))
np.save(results_dir + '/real_visualizations_{}.npy'.format(epoch), np.reshape(visualized, (n_view,image_width, image_width)))
for name in monitor_function_names:
np.save(results_dir + '/{}_valid_{}.npy'.format(name, epoch), monitor_output_valid[name])
np.save(results_dir + '/{}_train_{}.npy'.format(name, epoch), monitor_output_train[name])
np.save(results_dir + '/validation_losses.npy', validation_losses)
np.save(results_dir + '/training_losses.npy', training_losses)
np.save(results_dir + '/sample_visualizations.npy', np.array(samples_list))
np.save(results_dir + '/real_visualizations.npy', np.reshape(visualized, (n_view,image_width, image_width)))
for name in monitor_function_names:
np.save(results_dir + '/{}_valid.npy'.format(name), monitor_output_valid[name])
np.save(results_dir + '/{}_train.npy'.format(name), monitor_output_train[name])
visualize = False
if visualize:
for samples in samples_list:
together = np.hstack((np.reshape(visualized, (n_view,image_width, image_width)), samples > 0.5))
plot_images_together(together)
sess.close()
tf.reverse(input_image, [3]),
collections=['train'])
tf.summary.image('input_image',
tf.reverse(input_image, [3]),
collections=['test'])
tf.summary.image(
'predictions',
tf.expand_dims(tf.cast(fcn_vgg.pred_up, tf.uint8), -1),
collections=[
'test'
]) # display the predictions for the test image at index 'test_iter-1'
# define loss
with tf.name_scope('Loss'):
weights = tf.cast(gt_label != ignore_label, dtype=tf.float32)
unnormalized_loss = loss(fcn_vgg.upscore, gt_label_onehot, num_classes,
weights)
batch_loss, update_op_batchloss, reset_op_batchloss = streamingLoss(
unnormalized_loss)
# scalar summary
tf.summary.scalar("loss", batch_loss, collections=['loss'])
# define accuracy
# Note: tf.metrics are inherently tricky due to their streaming properties, see https://github.com/tensorflow/tensorflow/issues/9498
with tf.variable_scope('Metrics'):
# confusion matrix
confusion_matrix, update_op_confmat = streamingConfusionMatrix(
gt_label, fcn_vgg.pred_up, num_classes)
tf.summary.image('confusion_matrix',
tf.reshape(tf.cast(confusion_matrix, tf.float32),
[1, num_classes, num_classes, 1]),
def read_config(filename, verbose=False):
"""
Takes filename of config file and runs
1) Training of network with data as requested (printing real time loss)
2) Testing
3) Print test metrics
"""
config = configparser.ConfigParser()
config.read(filename)
net = get_net(config['LAYERS'])
loss, reg, num_epochs, batch_size, validation, optimization_method, optimization_params = get_training_vars(
config['TRAIN'])
data_train, labels_train, data_test, labels_test, size, type = get_data(
config['DATA'])
print("Samples of the training data")
if type == "2D":
plot_data(data_test[:, 0:labels_test.shape[0]])
elif type == "1D":
plt.imshow(data_train)
plt.show()
print("Epoch nr | Loss")
train(net,
inputs=data_train,
targets=labels_train,
num_epochs=num_epochs,
batch_size=batch_size,
loss=loss(),
optimizer=Optimizer(optimization_method, optimization_params),
regularizer=reg,
validation=validation,
verbose=verbose)
if type == "2D":
compare_test(net, data_test, labels_test)
def plot_prediction(i):
labels = {
0: "cross",
1: "circle",
2: "triangle",
3: "square",
4: "pentagon"
}
pred = net.forward(data_test)
print(pred.shape)
pred_label = np.where(pred[:, i] == np.max(pred[:, i]))[0][0]
plot_title = labels[int(pred_label)] + " prob : " + str(
round(np.max(pred[:, i]), 2))
plt.title(plot_title)
plt.imshow(data_test[:, i].reshape(size, size), cmap="Greys")
plot_img_derivative(net, loss(), data_test[:, i], labels_test[:,
i])
plt.colorbar()
plt.show()
return plot_prediction, net
elif type == "1D":
plt.figure()
pred = net.forward(data_test)
x = np.arange(0, len(labels_test[0]))
plt.title("Error of 1D prediction")
plt.plot(x, labels_test[0] - pred[0], '.')
plt.show()
def train(opt):
# Init Model
generator = Generator().cuda()
# generator.load_state_dict(torch.load('checkpoint.pt'))
discriminator = Discriminator().cuda()
discriminator.train()
# Load Dataset
train_dataset = MNISTDataset('train')
train_data_loader = MNISTDataloader('train', opt, train_dataset)
# test_dataset = MNISTDataset('test')
# test_data_loader = MNISTDataloader('test', opt, test_dataset)
# Set Optimizer
optim_gen = torch.optim.Adam(generator.parameters(), lr=0.0002)
optim_dis = torch.optim.Adam(discriminator.parameters(), lr=0.0002)
# Set Loss
loss = Loss().cuda()
writer = SummaryWriter()
for epoch in range(opt.epoch):
for i in range(len(train_data_loader.data_loader)):
step = epoch * len(train_data_loader.data_loader) + i + 1
# load dataset only batch_size
image, label = train_data_loader.next_batch()
batch_size = image.shape[0] // 2
real_rand = np.random.random(batch_size)
batch_l = image[:batch_size, :, :, :]
batch_r = image[batch_size:, :, :, :]
image = torch.cat((batch_l, batch_r), dim=3).cuda()
label[:batch_size][real_rand < 0.5] = 10
label[batch_size:][real_rand >= 0.5] = 10
label = label.unsqueeze(1)
real_label = torch.cat(
(label[:batch_size, :], label[batch_size:, :]), dim=1).cuda()
# train generator
generator.train()
optim_gen.zero_grad()
noise = Variable(torch.randn(batch_size, 100).cuda())
fake_label = Variable(
torch.LongTensor(torch.randint(10, (batch_size, 2))).cuda())
fake_rand = np.random.random(batch_size)
fake_label[fake_rand < 0.5, 0] = 10
fake_label[fake_rand >= 0.5, 1] = 10
gen = generator(noise, fake_label)
validity = discriminator(gen, fake_label)
loss_gen_l = loss(validity[fake_rand < 0.5, 0],
torch.ones(batch_size)[fake_rand < 0.5].cuda())
loss_gen_r = loss(validity[fake_rand >= 0.5, 1],
torch.ones(batch_size)[fake_rand >= 0.5].cuda())
loss_gen_l.backward(retain_graph=True)
loss_gen_r.backward()
loss_gen = loss_gen_l + loss_gen_r
optim_gen.step()
# train discriminator
optim_dis.zero_grad()
validity_real = discriminator(image, real_label)
loss_dis_real_l = loss(
validity_real[real_rand < 0.5, 0],
torch.ones(batch_size)[real_rand < 0.5].cuda())
loss_dis_real_r = loss(
validity_real[real_rand >= 0.5, 1],
torch.ones(batch_size)[real_rand >= 0.5].cuda())
validity_fake = discriminator(gen.detach(), fake_label)
loss_dis_fake_l = loss(
validity_fake[fake_rand < 0.5, 0],
torch.zeros(batch_size)[fake_rand < 0.5].cuda())
loss_dis_fake_r = loss(
validity_fake[fake_rand >= 0.5, 1],
torch.zeros(batch_size)[fake_rand >= 0.5].cuda())
loss_dis_l = (loss_dis_real_l + loss_dis_fake_l) / 2
loss_dis_r = (loss_dis_real_r + loss_dis_fake_r) / 2
loss_dis_l.backward(retain_graph=True)
loss_dis_r.backward()
loss_dis = loss_dis_l + loss_dis_r
optim_dis.step()
writer.add_scalar('loss/gen', loss_gen, step)
writer.add_scalar('loss/dis', loss_dis, step)
if step % opt.display_step == 0:
writer.add_images('image', image[0][0], step, dataformats="HW")
writer.add_images('result', gen[0][0], step, dataformats="HW")
print('[Epoch {}] G_loss : {:.2} | D_loss : {:.2}'.format(
epoch + 1, loss_gen, loss_dis))
generator.eval()
noise = torch.randn(9, 100).cuda()
label = torch.LongTensor(torch.randint(10, (9, 2))).cuda()
sample_images = generator(noise, label)
grid = make_grid(sample_images, nrow=3, normalize=True)
writer.add_image('sample_image', grid, step)
torch.save(generator.state_dict(), 'checkpoint.pt')
def f(theta):
return loss(theta, self.x, self.y)
def train(opt):
# Init Modeil
generator = Generator().cuda()
discriminator = Discriminator().cuda()
discriminator.train()
# Load Dataset
train_dataset = MNISTDataset('train')
train_data_loader = MNISTDataloader('train', opt, train_dataset)
# Set Optimizer
optim_gen = torch.optim.Adam(generator.parameters(), lr=0.0001)
optim_dis = torch.optim.Adam(discriminator.parameters(), lr=0.0001)
# Set Loss
loss = Loss()
writer = SummaryWriter()
for epoch in range(opt.epoch):
for i in range(len(train_data_loader.data_loader)):
step = epoch * len(train_data_loader.data_loader) + i + 1
# load dataset only batch_size
image, label = train_data_loader.next_batch()
image = image.cuda()
# train generator
generator.train()
optim_gen.zero_grad()
noise = Variable(torch.randn(opt.batch_size, 100)).cuda()
gen = generator(noise)
validity = discriminator(gen)
loss_gen = loss(validity, torch.ones(opt.batch_size, 1).cuda())
loss_gen.backward()
optim_gen.step()
# train discriminator
optim_dis.zero_grad()
validity_real = discriminator(image)
loss_dis_real = loss(
validity_real,
Variable(torch.ones(opt.batch_size, 1)).cuda())
validity_fake = discriminator(gen.detach())
loss_dis_fake = loss(
validity_fake,
Variable(torch.zeros(opt.batch_size, 1)).cuda())
loss_dis = loss_dis_real + loss_dis_fake
loss_dis.backward()
optim_dis.step()
loss_tot = loss_gen + loss_dis
writer.add_scalar('loss/total', loss_tot, step)
writer.add_scalar('loss/gen', loss_gen, step)
writer.add_scalar('loss/dis', loss_dis, step)
writer.add_scalar('loss/dis_real', loss_dis_real, step)
writer.add_scalar('loss/dis_fake', loss_dis_fake, step)
if step % opt.display_step == 0:
writer.add_images('image', image[0][0], step, dataformats="HW")
writer.add_images('result', gen[0][0], step, dataformats="HW")
print(
'[Epoch {}] Total : {:.2} | G_loss : {:.2} | D_loss : {:.2}'
.format(epoch + 1, loss_gen + loss_dis, loss_gen,
loss_dis))
generator.eval()
z = Variable(torch.randn(10, 100)).cuda()
sample_images = generator(z)
grid = make_grid(sample_images, nrow=5, normalize=True)
writer.add_image('sample_image', grid, step)
torch.save(generator.state_dict(), 'checkpoint.pt')
def GeneticAlgorithm_f(self,model,parameters):
model=self.model
self.emin,self.emax=self.model.normalise_val()
self.kmax = parameters[0]
self.population_Size = parameters[1]
self.p_crossover=parameters[2]
self.p_mutation=parameters[3]
self.frontier_distribution=parameters[4]
#print self.emin,self.emax
population_frontier=[self.model.candidates(0,False,0) for _ in range(self.population_Size)]
population_frontier_old=self.calculate_best_frontier(population_frontier)
first_frontier=population_frontier_old
pop_front_best=population_frontier_old[:]
#print len(population_frontier_old)
#print population_frontier_old
k=0
life=0
while k<self.kmax :
i=0
children=[]
for i in xrange(self.population_Size):
parent1,parent2= self.selectParents(population_frontier_old,len(population_frontier_old))
child1,child2=self.crossover(parent1, parent2,self.p_crossover,model)
child1=self.mutate(child1,self.p_mutation,model)
child2=self.mutate(child2,self.p_mutation,model)
if(self.bin_dom(child1,child2)):
children.append(child1)
elif(self.bin_dom(child1,child2)):
children.append(child2)
else:
x=random.randint(0,100)
if(x%2==0):
children.append(child2)
else:
children.append(child1)
population_frontier_new=[]
for can1 in population_frontier:
flag=True
for can2 in population_frontier:
if(can1==can2):
continue
if(self.bin_dom(can1,can2)):
flag=False
break
if flag:
population_frontier_new.append(can1)
tmp=[]
new_frontier=False
for a in population_frontier_new:
for b in pop_front_best:
if self.bin_dom(a,b):
tmp.append(a)
pop_front_best.remove(b)
if tmp:
pop_front_best.extend(tmp)
new_frontier=True
if new_frontier:
life=0
else:
life=life+1
if(life==self.lifes):
break
population_frontier=children
population_frontier_old=population_frontier_new
k=k+1
# print population_frontier
#print first_frontier
x=loss(population_frontier[0],first_frontier[0])
global_variable.pop_front_best=pop_front_best
'''name='Pareto_Fronts/pareto.txt'
f=open(name,'w')
i=0
for i in xrange(len(pop_front_best)):
k=0
for j in (self.model.objectives(pop_front_best[i])):
j=(j - self.emin[k]) / (self.emax[k]- self.emin[k])
k+=1
s=str(j)
f.write(s)
f.write(" ")
f.write("\n")
f.close();
# hypervol=HyperVolume_wrapper()
os.remove('./Pareto_Fronts/pareto.txt')'''
#print hypervol
return x
