def main(data, cls_num, increase=True, n=None, num_epochs=500, strategy='momentum', model=None, lb_num=None):
X_label = data['X_label']
Y_label = data['Y_label']
X_train = data['X_train']
Y_train = data['Y_train']
X_val = data['X_val']
Y_val = data['Y_val']
X_test = data['X_test']
Y_test = data['Y_test']
X_unlabel = data['X_unlabel']
Y_unlabel = data['Y_unlabel']
# Prepare Theano variables for inputs and targets
input_var = T.tensor4('inputs')
target_var = T.ivector('targets')
print(Y_train.shape[0])
# Create neural network model
print("Building model and compiling functions...")
cls_network, kl_network = build_cnn(cls_num, input_var, n)
epsilon = 1e-35
margin = 2
batchsize = target_var.shape[0]
cls_prediction = lasagne.layers.get_output(cls_network)
kl_prediction = lasagne.layers.get_output(kl_network)
# add weight decay
all_layers = lasagne.layers.get_all_layers([cls_network, kl_network])
l2_penalty = lasagne.regularization.regularize_layer_params(all_layers, lasagne.regularization.l2) * 0.0005
cls_loss = lasagne.objectives.categorical_crossentropy(cls_prediction, target_var)
cls_loss = cls_loss.mean()
kl_prediction = kl_prediction + epsilon
kl_loss = get_klloss(batchsize, margin, kl_prediction, target_var)
loss = l2_penalty + kl_loss + 0*cls_loss
lr = 0.1
sh_lr = theano.shared(lasagne.utils.floatX(lr))
params = lasagne.layers.get_all_params([cls_network, kl_network], trainable=True)
if strategy=='momentum':
updates = lasagne.updates.momentum(loss, params, learning_rate=sh_lr, momentum=0.9)#nesterov_
else:
updates = lasagne.updates.adam(loss, params, learning_rate=1e-4)
train_fn = theano.function([input_var, target_var], [loss, cls_prediction, kl_prediction], updates=updates)
# Create a loss expression for validation/testing
test_cls_prediction = lasagne.layers.get_output(cls_network, deterministic=True)
test_cls_loss = lasagne.objectives.categorical_crossentropy(test_cls_prediction, target_var)
test_cls_loss = test_cls_loss.mean()
test_kl_prediction = lasagne.layers.get_output(kl_network, deterministic=True)
test_kl_prediction = test_kl_prediction + epsilon
test_kl_loss = get_klloss(batchsize, margin, test_kl_prediction, target_var)
test_loss = l2_penalty + test_kl_loss + test_cls_loss
# Compile a second function computing the validation loss and accuracy:
val_fn = theano.function([input_var, target_var], [test_loss, test_cls_prediction, test_kl_prediction])
print("number of parameters in model: %d" % lasagne.layers.count_params([cls_network, kl_network], trainable=True))
#print(architecture_string([cls_network, kl_network]))
if model is not None:
# load network weights from model file
with np.load(model) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
with np.load('mt_%d_indexes_%d.npz' % (lb_num,n)) as f:
indexes = f['arr_0']
lasagne.layers.set_all_param_values([cls_network, kl_network], param_values)
else:
# launch the training loop
print("Starting training...")
for epoch in np.arange(num_epochs):
print("Epoch {} of {}:".format(epoch + 1, num_epochs))
cls_preds, kl_preds, my_hat, indexes, cm = get_mtresults(X_train, Y_train, 'train', train_fn, True, True)
cls_preds, kl_preds, my_hat, indexes, cm = get_mtresults(X_val, Y_val, 'val', val_fn, indexes = indexes)
if (epoch+1) == 120 or (epoch+1) == 130 or (epoch+1)==140:
new_lr = sh_lr.get_value() * 0.1
print("New LR:"+str(new_lr))
sh_lr.set_value(lasagne.utils.floatX(new_lr))
# dump the network weights to a file :
model_path='model_mt_%d_%d.npz' % (lb_num,n)
if not os.path.exists(model_path):
np.savez(model_path, *lasagne.layers.get_all_param_values([cls_network, kl_network]))
np.savez('mt_%d_indexes_%d.npz' % (lb_num,n), indexes)
get_mtresults(X_test, Y_test, 'test', val_fn, indexes = indexes)
return input_var, cls_network, kl_network, val_fn, indexes
def main(data,
cls_num,
dim,
pos_ratio,
w=None,
increase=True,
n=2,
num_epochs=500,
strategy='momentum',
model=None,
thres=None,
lb_num=None):
x_train = data['training']
x_val = data['val']
# Prepare Theano variables for inputs and targets
input_var = T.tensor4('inputs')
target_var = T.ivector('targets')
w_var = T.vector('w')
num_boxes_train = len(x_train['image_ids'])
pos_idx = (x_train['boxes'][:, -1] == 1).nonzero()[0]
print(num_boxes_train, 'pos:{}'.format(pos_idx.shape[0]))
# Create neural network model
print("Building model and compiling functions...")
cls_network, kl_network = build_cnn(cls_num, input_var, n, dim)
epsilon = 1e-35
margin = 0.5
batchsize = target_var.shape[0]
cls_prediction = lasagne.layers.get_output(cls_network)
# cls_prediction = T.clip(cls_prediction, 1e-7, 1-(1e-7))
kl_prediction = lasagne.layers.get_output(kl_network)
# add weight decay
all_layers = lasagne.layers.get_all_layers([cls_network, kl_network])
l2_penalty = lasagne.regularization.regularize_layer_params(
all_layers, lasagne.regularization.l2) * 0.0005
cls_loss = lasagne.objectives.categorical_crossentropy(
cls_prediction, target_var)
cls_loss = aggregate(cls_loss, w_var)
kl_prediction = kl_prediction + epsilon
kl_loss = get_klloss(batchsize, margin, kl_prediction, target_var)
loss = l2_penalty + 0 * kl_loss + cls_loss
lr = 0.1
sh_lr = theano.shared(lasagne.utils.floatX(lr))
params = lasagne.layers.get_all_params([cls_network, kl_network],
trainable=True)
if strategy == 'momentum':
updates = lasagne.updates.momentum(loss,
params,
learning_rate=sh_lr,
momentum=0.9) #nesterov_
else:
updates = lasagne.updates.adam(loss, params, learning_rate=1e-4)
train_fn = theano.function([input_var, target_var, w_var],
[loss, cls_prediction, kl_prediction],
updates=updates,
allow_input_downcast=True) #, w_var
# Create a loss expression for validation/testing
test_cls_prediction = lasagne.layers.get_output(cls_network,
deterministic=True)
test_cls_prediction = T.clip(test_cls_prediction, 1e-7, 1 - 1e-7)
test_cls_loss = lasagne.objectives.categorical_crossentropy(
test_cls_prediction, target_var)
test_cls_loss = test_cls_loss.mean()
test_kl_prediction = lasagne.layers.get_output(kl_network,
deterministic=True)
test_kl_prediction = test_kl_prediction + epsilon
test_kl_loss = get_klloss(batchsize, margin, test_kl_prediction,
target_var)
test_loss = l2_penalty + test_kl_loss + test_cls_loss
# Compile a second function computing the validation loss and accuracy:
val_fn = theano.function(
[input_var, target_var],
[test_loss, test_cls_prediction, test_kl_prediction],
allow_input_downcast=True)
print(
"number of parameters in model: %d" %
lasagne.layers.count_params([cls_network, kl_network], trainable=True))
print(architecture_string([cls_network, kl_network]))
if model is not None and os.path.exists(model):
# load network weights from model file
with np.load(model) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
with np.load('weighted_indexes_{}'.format(model)) as f:
indexes = f['arr_0']
lasagne.layers.set_all_param_values([cls_network, kl_network],
param_values)
else:
# launch the training loop
print("Starting training...")
for epoch in np.arange(num_epochs):
print("Epoch {} of {}:".format(epoch + 1, num_epochs))
cls_preds, kl_preds, my_hat, indexes, cm = get_mtresults(x_train,\
'train','train', train_fn, cls_num, dim, pos_ratio, True, w=w, batchsize=128)
cls_preds, kl_preds, my_hat, indexes, cm = get_mtresults(x_val,\
'val', 'train',val_fn, cls_num, dim, pos_ratio, indexes = indexes)
if (epoch + 1) == 30 or (epoch + 1) == 50 or (epoch + 1) == 70:
new_lr = sh_lr.get_value() * 0.1
print("New LR:" + str(new_lr))
sh_lr.set_value(lasagne.utils.floatX(new_lr))
# dump the network weights to a file :
if model is not None:
np.savez(
model,
*lasagne.layers.get_all_param_values([cls_network,
kl_network]))
np.savez('weighted_indexes_{}'.format(model), indexes)
return input_var, cls_network, kl_network, val_fn, indexes
if os.path.exists(model):
kwargs['model'] = model
else:
kwargs['model'] = None
for i in thres:
print('thres = ', i)
input_var, cls_network, kl_network, val_fn, indexes = main(data, **kwargs)
###########################################################################
# net=lasagne.layers.get_all_layers(cls_network)
# cls_gp = net[-2]
# net=lasagne.layers.get_all_layers(kl_network)
# kl_gp = net[-2]
#
cls_preds, kl_preds, kl_yhat, indexes, cm = get_mtresults(X_unlabel, Y_unlabel, 'unlabel', val_fn, indexes = indexes)
# #prediction acc on unlabeled data using the trained network
#
comp_preds(cls_preds, kl_preds, kl_yhat, Y_unlabel)
#
unlabel_num = Y_unlabel.shape[0]
def getidx(preds):
plabel = np.argmax(preds, axis=1)
tpreds = preds[np.arange(unlabel_num), plabel]
pidx = np.argsort(tpreds)
return plabel, pidx, tpreds
cls_plabel, cls_pidx, cls_preds = getidx(cls_preds)
# cst_idx, lb1, lb2, i1, i2 = comp_preds(cls_preds, kl_preds, kl_yhat, Y_unlabel)
# print('len of cst_idx: ', cst_idx.shape[0] )
# def getFeats(X, Y, cls, kl, input_var):
def calculate_det(data, dataset, fn, cls_num, dim, pos_ratio, nms_thres,
iou_thres, indexes):
cls_preds, kl_preds, kl_yhat, indexes, cm = get_mtresults(data, dataset, \
'test', val_fn, cls_num, dim, pos_ratio, indexes = indexes)
# cls_plabel = np.argmax(cls_preds, axis=1)
cls_preds = kl_preds
pos_idx = 1
for pred, gt in indexes:
if pred == 1:
pos_idx = gt
break
pos_preds = cls_preds[:, pos_idx]
cls_pidx = np.argsort(pos_preds)
cls_pidx = cls_pidx[::-1]
boxes = data['boxes'][cls_pidx]
boxes = np.column_stack((boxes, pos_preds[cls_pidx]))
image_ids = np.array(data['image_ids'])
image_ids = image_ids[cls_pidx]
# boxes = boxes[np.arange(boxes.shape[0]-1,-1,-1)]
# image_ids = image_ids[np.arange(boxes.shape[0]-1,-1,-1)]
gt = {}
gt_idx = (boxes[:, 0] == 1).nonzero()[0]
gt['image_ids'] = image_ids[gt_idx]
gt['boxes'] = boxes[gt_idx]
gt['match'] = np.zeros(gt['boxes'].shape[0], dtype=np.int8)
print('total gts number:{}'.format(gt['boxes'].shape[0]))
candidate = {}
candidate_idx = (boxes[:, 0] == 0).nonzero()[0]
candidate['image_ids'] = image_ids[candidate_idx]
candidate['boxes'] = boxes[candidate_idx]
print('total candidates number:{}'.format(candidate['boxes'].shape[0]))
candidate = nms(candidate, nms_thres)
print('nms candidates number:{}'.format(candidate['boxes'].shape[0]))
num_images = (np.unique(candidate['image_ids'])).shape[0]
num_detects = candidate['image_ids'].shape[0]
total_num_gts = gt['image_ids'].shape[0]
fp = np.zeros(num_detects, dtype=np.float)
tp = np.zeros(num_detects, dtype=np.float)
for did in np.arange(0, num_detects):
dbox = candidate['boxes'][did, 2:6]
iid = candidate['image_ids'][did]
gidx = (gt['image_ids'] == iid).nonzero()[0]
max_iou = 0
max_id = -1
for gid in gidx:
gbox = gt['boxes'][gid, 2:6]
tiou = iou(dbox, gbox)
if tiou > max_iou:
max_iou = tiou
max_id = gid
# print(max_iou)
if max_iou >= iou_thres and gt['match'][max_id] == 0:
tp[did] = 1
gt['match'][max_id] = 1
else:
fp[did] = 1
fppi = np.cumsum(fp) / num_images
drate = np.cumsum(tp) / total_num_gts
idx = (fppi <= 10).nonzero()[0]
plt.plot(fppi[idx], drate[idx])
plt.show()
idx = (fppi <= 1).nonzero()[0]
print('when fppi=1, recall={}'.format(drate[idx[-1]]))
ret = {}
ret['images'] = data['images']
ret['gt'] = gt
ret['candidate'] = candidate
return ret, drate, fppi
input_var, cls_network, kl_network, val_fn, indexes = main(
data, **kwargs)
net = lasagne.layers.get_all_layers(cls_network)
cls_gp = net[-2]
net = lasagne.layers.get_all_layers(kl_network)
kl_gp = net[-2]
# cls_preds, kl_preds, kl_yhat, indexes, cm = get_mtresults(training_data, 'train', \
# 'test', val_fn, kwargs['cls_num'], dim, pos_ratio, indexes = indexes)
#
cls_preds, kl_preds, kl_yhat, indexes, cm = get_mtresults(
x_unlabel,
'unlabel',
'test',
val_fn,
kwargs['cls_num'],
dim,
pos_ratio,
indexes=indexes)
#prediction acc on unlabeled data using the trained network
#kwargs['model'] = None
boxes_unlabel = x_unlabel['boxes']
img_ids_unlabel = x_unlabel['image_ids']
img_ids_unlabel = np.array(img_ids_unlabel)
y_unlabel = boxes_unlabel[:, -1]
gt_unlabel = boxes_unlabel[:, 0]
cls_plabel, cls_pidx, cls_preds2 = getidx(cls_preds)
pos_index = (cls_plabel == 1).nonzero()[0]
def main(data, cls_num, dim, pos_ratio, w=None, increase=True, n=2, num_epochs=500, strategy='momentum', last_model=None, model=None, thres=None,lb_num=None):
x_train = data['training']
x_val = data['val']
# Prepare Theano variables for inputs and targets
input_var = T.tensor4('inputs')
target_var = T.ivector('targets')
w_var = T.vector('w')
neg_idx = (x_train['boxes'][:,-1]==0).nonzero()[0]
pos_idx = (x_train['boxes'][:,-1]==1).nonzero()[0]
print('neg:{}'.format(neg_idx.shape[0]), 'pos:{}'.format(pos_idx.shape[0]))
# Create neural network model
print("Building model and compiling functions...")
network = build_alexnet(cls_num, input_var, dim)
fc8 = network['fc8']
print(architecture_string(fc8))
batchsize = target_var.shape[0]
prediction = lasagne.layers.get_output(fc8)
# add weight decay
all_layers = lasagne.layers.get_all_layers(fc8)
l2_penalty = lasagne.regularization.regularize_layer_params(
all_layers, lasagne.regularization.l2) * 0.0005
loss = lasagne.objectives.categorical_crossentropy(prediction, target_var)
#loss = loss.mean()
loss = aggregate(loss, w_var)
loss = loss + l2_penalty
acc = T.mean(T.eq(T.argmax(prediction, axis=1), target_var), dtype=theano.config.floatX)
lr = 0.001
sh_lr = theano.shared(lasagne.utils.floatX(lr))
params = lasagne.layers.get_all_params(fc8, trainable=True)
if strategy=='momentum':
updates = lasagne.updates.momentum(loss, params, learning_rate=sh_lr, momentum=0.9)#nesterov_
else:
updates = lasagne.updates.adam(loss, params, learning_rate=1e-4)
train_fn = theano.function([input_var, target_var, w_var], [loss, acc, prediction],
updates=updates, allow_input_downcast=True)
# Create a loss expression for validation/testing
test_prediction = lasagne.layers.get_output(fc8, deterministic=True)
test_loss = lasagne.objectives.categorical_crossentropy(test_prediction, target_var)
test_loss = test_loss.mean()
test_acc = T.mean(T.eq(T.argmax(test_prediction, axis=1), target_var),
dtype=theano.config.floatX)
# Compile a second function computing the validation loss and accuracy:
val_fn = theano.function([input_var, target_var], [test_loss, test_acc,
test_prediction], allow_input_downcast=True)
print("number of parameters in model: %d" % lasagne.layers.count_params(fc8, trainable=True))
# print(architecture_string(fc8))
if model is not None and os.path.exists(model):
# load network weights from model file
with np.load(model) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
#with np.load('weighted_indexes_{}'.format(model)) as f:
# indexes = f['arr_0']
lasagne.layers.set_all_param_values(fc8, param_values)
else:
if last_model is None:
# first loop of training, copy model from caffe
caffe_root = '/home/dl/caffe/'
sys.path.insert(0, caffe_root + 'python')
import caffe
caffe.set_mode_gpu()
caffe_net = caffe.Net(caffe_root + 'models/bvlc_reference_caffenet/deploy.prototxt',
caffe_root + 'models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel',
caffe.TEST)
layers_caffe = dict(zip(list(caffe_net._layer_names), caffe_net.layers))
for name, layer in network.items():
try:
if name == 'conv2':
W = layers_caffe[name].blobs[0].data[:,:,::-1,::-1]
b = layers_caffe[name].blobs[1].data
network['conv2_part1'].W.set_value(W[0:128,:,:,:])
network['conv2_part1'].b.set_value(b[0:128])
network['conv2_part2'].W.set_value(W[128:,:,:,:])
network['conv2_part2'].b.set_value(b[128:])
elif name == 'conv4':
W = layers_caffe[name].blobs[0].data[:,:,::-1,::-1]
b = layers_caffe[name].blobs[1].data
network['conv4_part1'].W.set_value(W[0:192,:,:,:])
network['conv4_part1'].b.set_value(b[0:192])
network['conv4_part2'].W.set_value(W[192:,:,:,:])
network['conv4_part2'].b.set_value(b[192:])
elif name == 'conv5':
W = layers_caffe[name].blobs[0].data[:,:,::-1,::-1]
b = layers_caffe[name].blobs[1].data
network['conv5_part1'].W.set_value(W[0:128,:,:,:])
network['conv5_part1'].b.set_value(b[0:128])
network['conv5_part2'].W.set_value(W[128:,:,:,:])
network['conv5_part2'].b.set_value(b[128:])
elif name == 'fc6' or name == 'fc7':
layer.W.set_value(np.transpose(layers_caffe[name].blobs[0].data))
layer.b.set_value(layers_caffe[name].blobs[1].data)
elif name != 'fc8':
layer.W.set_value(layers_caffe[name].blobs[0].data[:,:,::-1,::-1])
layer.b.set_value(layers_caffe[name].blobs[1].data)
except AttributeError:
continue
except KeyError:
continue
print(architecture_string(fc8))
elif os.path.exists(last_model):
with np.load(last_model) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
#with np.load('weighted_indexes_{}'.format(last_model)) as f:
# indexes = f['arr_0']
lasagne.layers.set_all_param_values(fc8, param_values)
print("Starting training...")
for epoch in np.arange(num_epochs):
print("Epoch {} of {}:".format(epoch + 1, num_epochs))
preds = get_mtresults(x_train,\
'train','train', train_fn, cls_num, dim, pos_ratio, True, w=w, batchsize=128)
preds = get_mtresults(x_val,\
'val', 'train', val_fn, cls_num, dim, pos_ratio)
if (epoch+1) == 10 or (epoch+1) == 20 or (epoch+1)==30:
new_lr = sh_lr.get_value() * 0.1
print("New LR:"+str(new_lr))
sh_lr.set_value(lasagne.utils.floatX(new_lr))
# dump the network weights to a file :
if model is not None:
np.savez(model, *lasagne.layers.get_all_param_values(fc8))
#np.savez('weighted_indexes_{}'.format(model), indexes)
return input_var, fc8, val_fn
def calculate_det(data, dataset, fn, cls_num, dim, pos_ratio, nms_thres,iou_thres):
preds = get_mtresults(data, dataset, \
'test', val_fn, cls_num, dim, pos_ratio)
pos_preds = preds[:,1]
sorted_pidx = np.argsort(pos_preds)
sorted_pidx = sorted_pidx[::-1]
boxes = data['boxes'][sorted_pidx]
boxes = np.column_stack((boxes,pos_preds[sorted_pidx]))
image_ids = np.array(data['image_ids'])
image_ids = image_ids[sorted_pidx]
gt = {}
gt_idx = (boxes[:,0]==1).nonzero()[0]
gt['image_ids'] = image_ids[gt_idx]
gt['boxes'] = boxes[gt_idx]
gt['match'] = np.zeros(gt['boxes'].shape[0], dtype=np.int8)
print('total gts number:{}'.format(gt['boxes'].shape[0]))
candidate = {}
candidate_idx = (boxes[:,0]==0).nonzero()[0]
candidate['image_ids'] = image_ids[candidate_idx]
candidate['boxes'] = boxes[candidate_idx]
print('total candidates number:{}'.format(candidate['boxes'].shape[0]))
candidate = nms(candidate, nms_thres)
print('nms candidates number:{}'.format(candidate['boxes'].shape[0]))
num_images = (np.unique(candidate['image_ids'])).shape[0]
num_detects = candidate['image_ids'].shape[0]
total_num_gts = gt['image_ids'].shape[0]
fp = np.zeros(num_detects,dtype=np.float)
tp = np.zeros(num_detects,dtype=np.float)
for did in np.arange(0,num_detects):
dbox = candidate['boxes'][did,2:6]
iid = candidate['image_ids'][did]
gidx = (gt['image_ids']==iid).nonzero()[0]
max_iou = 0
max_id = -1
for gid in gidx:
gbox = gt['boxes'][gid,2:6]
tiou = iou(dbox, gbox)
if tiou>max_iou:
max_iou = tiou
max_id = gid
# print(max_iou)
if max_iou>=iou_thres and gt['match'][max_id]==0:
tp[did] = 1
gt['match'][max_id] = 1
else:
fp[did] = 1
fppi = np.cumsum(fp)/num_images
drate = np.cumsum(tp)/total_num_gts
idx = (fppi<=10).nonzero()[0]
fout = open('det_curve.txt', 'w')
for i in idx:
fout.write("%.3f %.3f\n" %(drate[i], fppi[i]))
fout.close()
#plt.plot(fppi[idx], drate[idx])
#plt.show()
idx = (fppi<=1).nonzero()[0]
print('when fppi=1, recall={}'.format(drate[idx[-1]]))
ret = {}
ret['images'] = data['images']
ret['gt'] = gt
ret['candidate'] = candidate
return ret, drate, fppi
input_var, network, val_fn = main(data, **kwargs)
net=lasagne.layers.get_all_layers(network)
if True:
# train_detects,train_drate,train_fppi = calculate_det(training_data,'train',\
# val_fn, kwargs['cls_num'],dim, pos_ratio,nms_thres,iou_thres)
test_detects,test_drate,test_fppi = calculate_det(x_test,'test',\
val_fn,kwargs['cls_num'],dim, pos_ratio,nms_thres,iou_thres)
idx = (test_fppi<=1).nonzero()[0]
drates[spl_iter] = test_drate[idx[-1]]
# visualize(test_detects, test_drate, test_fppi)
continue
#break
tra_preds = get_mtresults(x_train, 'train',
'test', val_fn, kwargs['cls_num'], dim, pos_ratio)
preds = get_mtresults(x_unlabel, 'unlabel',
'test', val_fn, kwargs['cls_num'], dim, pos_ratio)
tra_scores = tra_preds[:,1]
unlabel_scores = preds[:,1]
unlabel_w = np.array(unlabel_scores)
eps = 1e-10
max_w = 0
for ii in np.arange(unlabel_scores.shape[0]):
s = unlabel_scores[ii]
unlabel_w[ii] = max(1/(s-tra_scores+eps))
max_w = max(max_w, unlabel_w[ii])
unlabel_w = unlabel_w/max_w