def proba(self, X, load_model=None):
assert load_model != None, "load_model should be True of False"
layers = self.model.layers
x = self.model.x
y = self.model.y
index = T.lscalar() # index to a [mini]batch
if load_model == True:
dir = self.model.snapshot
if not os.path.isdir(dir):
raise IOError('no such snapshot file: %s' %(dir))
snapshots = glob.glob(dir+'*.npy')
#e = os.path.basename(snapshots[0])[-5]
e = self.config['e_snapshot']
tools.load_weights(layers, dir, e)
prob_model = theano.function(
inputs = [index],
outputs = self.model.proba,
givens={
x: X[index:(index + 1)],
}
)
y_prob = []
n_test = X.get_value(borrow=True).shape[0]
DropoutLayer.SetDropoutOff()
print "getting probability on %d datas" %(int(n_test))
for i in xrange(n_test):
y_prob.append(prob_model(i))
return np.asarray(y_prob).reshape(n_test,y_prob[0].shape[1])
def load(self):
layers = self.model.layers
dir = self.model.snapshot
if not os.path.isdir(dir):
raise IOError('no such snapshot file: %s' %(dir))
snapshots = glob.glob(dir+'*.npy')
e = self.config['e_snapshot']
tools.load_weights(layers, dir, e)
def predict_by_sentence(self, test_set_x, index_test, load_model=None, dropout=False):
assert load_model != None, "load_model should be True of False"
#batch_size = self.model.batch_size
#n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size
#self.n_test_batches = n_test_batches
test_set_x = self.share_var(test_set_x,testing=True)
layers = self.model.layers
x = self.model.x
y = self.model.y
index = T.lscalar() # index to a [mini]batch
predict_times = len(index_test)-1
index_test= T.cast(theano.tensor._shared(np.asarray(index_test,dtype=theano.config.floatX),borrow=True),'int32')
if load_model == True:
dir = self.model.snapshot
if not os.path.isdir(dir):
raise IOError('no such snapshot file: %s' %(dir))
snapshots = glob.glob(dir+'*.npy')
#e = os.path.basename(snapshots[0])[-5]
e = self.config['e_snapshot']
if dropout == False:
tools.load_weights(layers, dir, e)
else:
tools.dropout_load_weights(layers, dir, e)
test_model = theano.function(
inputs = [index],
outputs = self.model.y_pred,
givens={
x: test_set_x[index_test[index]:index_test[(index + 1)]],
}
)
n_test = test_set_x.get_value(borrow=True).shape[0]
y_pred = np.array([])
DropoutLayer.SetDropoutOff()
print "predict on %d datas" %(int(n_test))
for i in xrange(predict_times):
y_pred = np.concatenate((y_pred,test_model(i)),axis=0)
return y_pred
def predict(self, test_set_x, load_model=None, dropout=False):
assert load_model != None, "load_model should be True of False"
#batch_size = self.model.batch_size
#n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size
#self.n_test_batches = n_test_batches
layers = self.model.layers
x = self.model.x
y = self.model.y
index = T.lscalar() # index to a [mini]batch
if load_model == True:
dir = self.model.snapshot
if not os.path.isdir(dir):
raise IOError('no such snapshot file: %s' %(dir))
snapshots = glob.glob(dir+'*.npy')
#e = os.path.basename(snapshots[0])[-5]
e = self.config['e_snapshot']
if dropout == False:
tools.load_weights(layers, dir, e)
else:
tools.dropout_load_weights(layers, dir, e)
test_model = theano.function(
inputs = [index],
outputs = self.model.y_pred,
givens={
x: test_set_x[index:(index + 1)],
}
)
n_test = test_set_x.get_value(borrow=True).shape[0]
y_pred = np.zeros(n_test)
DropoutLayer.SetDropoutOff()
print "predict on %d datas" %(int(n_test))
for i in xrange(n_test):
y_pred[i] = int(test_model(i))
return y_pred
def valid():
rng = np.random.RandomState(23455)
config = {'batch_size':256,
'use_data_layer':True,
'lib_conv':'cudnn',
'rand_crop':False,
'rng':rng
}
model = AlexNet(config)
DropoutLayer.SetDropoutOff()
load_weights(model.layers, model_folder, model_epoch)
img_mean = np.load(img_mean_path)[:, :, :, np.newaxis]
rep = np.zeros((195 * 256, 4096))
batch_size = 256
x = theano.shared(np.random.normal(0, 1 , (3, 227, 227, 256)).astype(theano.config.floatX))
f = theano.function([], [model.proba, model.layers[-2].output], givens={model.x:x})
accuracy = []
tic = time.time()
for i, filename in enumerate(sorted(os.listdir(data_folder))):
val_img = hkl.load(os.path.join(data_folder, filename)) - img_mean
x.set_value(val_img)
probas, last_layer = f()
pred = np.argmax(probas, axis=1)
accuracy += [(pred == y[i * batch_size: (i + 1) * batch_size]).mean() * 100.]
toc = time.time()
print "filename %s in %.2f sec \taccuracy :: %.2f\r" % (filename, toc - tic, np.mean(accuracy)),
sys.stdout.flush()
tic = toc
rep[i * batch_size:(i + 1) * batch_size, :] = last_layer
print "\naccuracy :: ", np.array(accuracy).mean()
pdb.set_trace()
cPickle.dump(rep, open("representation.pkl", "w"))
def test_speed_cpu():
rng = np.random.RandomState(23455)
config = {'batch_size':1,
'use_data_layer':False,
'lib_conv':'cpu',
'rand_crop':False,
'rng':rng
}
model = AlexNet(config)
DropoutLayer.SetDropoutOff()
load_weights(model.layers, model_folder, model_epoch)
img_mean = np.load(img_mean_path)[:, :, :, np.newaxis]
# testing time to predict per image
tic = time.time()
niters = 50
for i in range(niters):
x = np.random.normal(0, 1 , (3, 227, 227, 1)).astype(theano.config.floatX)
f = theano.function([model.x], model.outputs[-1])
#print f(x).shape
print "%.2f secs per images (averaged over %i iterations)" % ((time.time() - tic) / niters, niters)
def validate_performance(config):
# UNPACK CONFIGS
(flag_para_load, train_filenames, val_filenames,
train_labels, val_labels, img_mean) = unpack_configs(config)
if flag_para_load:
# pycuda and zmq set up
drv.init()
dev = drv.Device(int(config['gpu'][-1]))
ctx = dev.make_context()
sock = zmq.Context().socket(zmq.PAIR)
sock.connect('tcp://localhost:{0}'.format(config['sock_data']))
load_send_queue = config['queue_t2l']
load_recv_queue = config['queue_l2t']
else:
load_send_queue = None
load_recv_queue = None
import theano.sandbox.cuda
theano.sandbox.cuda.use(config['gpu'])
import theano
theano.config.on_unused_input = 'warn'
from layers import DropoutLayer
from alex_net import AlexNet, compile_models
import theano.misc.pycuda_init
import theano.misc.pycuda_utils
# # BUILD NETWORK ##
model = AlexNet(config)
layers = model.layers
batch_size = model.batch_size
# # COMPILE FUNCTIONS ##
(train_model, validate_model, train_error, learning_rate,
shared_x, shared_y, rand_arr, vels) = compile_models(model, config,
flag_top_5=True)
print '... training'
if flag_para_load:
# pass ipc handle and related information
gpuarray_batch = theano.misc.pycuda_utils.to_gpuarray(
shared_x.container.value)
h = drv.mem_get_ipc_handle(gpuarray_batch.ptr)
sock.send_pyobj((gpuarray_batch.shape, gpuarray_batch.dtype, h))
load_send_queue.put(img_mean)
load_epoch = config['load_epoch']
load_weights(layers, config['weights_dir'], load_epoch)
DropoutLayer.SetDropoutOff()
this_validation_error, this_validation_error_top_5, this_validation_loss = \
get_val_error_loss(rand_arr, shared_x, shared_y,
val_filenames, val_labels,
flag_para_load,img_mean,
batch_size, validate_model,
send_queue=load_send_queue,
recv_queue=load_recv_queue,
flag_top_5=True)
print('validation error %f %%' %
(this_validation_error * 100.))
print('top 5 validation error %f %%' %
(this_validation_error_top_5 * 100.))
print('validation loss %f ' %
(this_validation_loss))
return this_validation_error, this_validation_loss
def train_net(config, private_config):
# UNPACK CONFIGS
(train_videos_spatial_jhmdb,val_videos_spatial_jhmdb,train_videos_temporal_jhmdb,val_videos_temporal_jhmdb,
train_targets,val_targets,
train_labels_jhmdb,val_labels_jhmdb) = unpack_configs_jhmdb(config,gpu_id=private_config['gpu_id'])
# print('val_len',len(val_videos_spatial_jhmdb),'train_len',len(train_videos_spatial_jhmdb))
if config['modal']=='rgb':
train_videos = list(train_videos_spatial_jhmdb)
test_videos = list(val_videos_spatial_jhmdb)
else:
train_videos = list(train_videos_temporal_jhmdb)
test_videos = list(val_videos_temporal_jhmdb)
print('jhmdb_len',len(train_videos),len(train_labels_jhmdb))#,len(tr_video_length_jhmdb))
flag_para_load =config['para_load']
gpu_send_queue = private_config['queue_gpu_send']
gpu_recv_queue = private_config['queue_gpu_recv']
# pycuda and zmq set up
drv.init()
dev = drv.Device(int(private_config['gpu'][-1]))
ctx = dev.make_context()
sock_gpu = zmq.Context().socket(zmq.PAIR)
if private_config['flag_client']:
sock_gpu.connect('tcp://*****:*****@ iter = ', num_iter
print 'training cost:', cost_ij,'cost_nll:',cost_nll,'cost_attention:',cost_att
if config['print_train_error']:
error_ij = train_error()
gpu_send_queue.put(error_ij)
that_error = gpu_recv_queue.get()
error_ij = (error_ij + that_error) / 2.
if private_config['flag_verbose']:
print 'training error rate:', error_ij
if flag_para_load and (count < len(minibatch_range)):
load_send_queue.put('calc_finished')
if count%20 == 0:
e = time.time()
print "time per 20 iter:", (e - s)
# ############### Test on Validation Set ##################
DropoutLayer.SetDropoutOff()
this_val_error, this_val_loss = get_test_error(config,
shared_x, shared_mask, shared_y,shared_target,shared_use_noise,
shared_conv,test_videos, val_labels_jhmdb,
flag_para_load,
batch_size,num_seq, validate_model_lstm,train_model,
send_queue=load_send_queue, recv_queue=load_recv_queue)
# report validation stats
gpu_send_queue.put(this_val_error)
that_val_error = gpu_recv_queue.get()
this_val_error = (this_val_error + that_val_error) / 2.
gpu_send_queue.put(this_val_loss)
that_val_loss = gpu_recv_queue.get()
this_val_loss = (this_val_loss + that_val_loss) / 2.
if private_config['flag_verbose']:
print('epoch %i: test loss of jhmdb %f ' %
(epoch, this_val_loss))
print('epoch %i: test error of jhmdb %f %%' %
(epoch, this_val_error * 100.))
val_record.append([this_val_error, this_val_loss])
if private_config['flag_save']:
np.save(config['weights_dir'] + 'test_record_jhmdb.npy', val_record)
DropoutLayer.SetDropoutOn()
###########################################
# Adapt Learning Rate
step_idx = adjust_learning_rate(config, epoch, step_idx,
val_record, learning_rate)
# Save Weights, only one of them will do
if private_config['flag_save'] :
if epoch % config['snapshot_freq'] == 0:
save_weights(layers, config['weights_dir'], epoch)
np.save(config['weights_dir'] + 'lr_' + str(epoch) + '.npy',
learning_rate.get_value())
save_momentums(vels, config['weights_dir'], epoch)
print('Optimization complete.')
def train(trn_data_generator, vld_data=None):
learning_rate = config['learning_rate']
experiment_dir = config['experiment_dir']
data_dims = config['data_dims']
batch_size = config['batch_size']
num_epochs = config['num_epochs']
num_samples_per_epoch = config["num_samples_per_epoch"]
steps_per_epoch = num_samples_per_epoch // batch_size
num_steps = steps_per_epoch * num_epochs
checkpoint_dir = pth.join(experiment_dir, 'checkpoints')
train_log_fpath = pth.join(experiment_dir, 'train.log')
vld_iter = config["vld_iter"]
checkpoint_iter = config["checkpoint_iter"]
pretrained_weights = config.get("pretrained_weights", None)
# ========================
# construct training graph
# ========================
G = tf.Graph()
with G.as_default():
input_data_tensor = tf.placeholder(tf.float32, [None] + data_dims)
input_label_tensor = tf.placeholder(tf.int32, [None])
model = build_model(input_data_tensor, input_label_tensor)
optimizer = tf.train.AdamOptimizer(learning_rate)
grads = optimizer.compute_gradients(model["loss"])
grad_step = optimizer.apply_gradients(grads)
init = tf.initialize_all_variables()
# ===================================
# initialize and run training session
# ===================================
log = tools.StatLogger(train_log_fpath)
config_proto = tf.ConfigProto(allow_soft_placement=True)
sess = tf.Session(graph=G, config=config_proto)
sess.run(init)
tf.train.start_queue_runners(sess=sess)
with sess.as_default():
if pretrained_weights:
print("-- loading weights from %s" % pretrained_weights)
tools.load_weights(G, pretrained_weights)
# Start training loop
for step in range(num_steps):
batch_train = list(trn_data_generator.__next__())
#print(np.array(list(batch_train)[1]).shape)
X_trn = np.array(batch_train[0])
Y_trn = np.array(batch_train[1])
ops = [grad_step] + [model[k] for k in sorted(model.keys())]
inputs = {input_data_tensor: X_trn, input_label_tensor: Y_trn}
results = sess.run(ops, feed_dict=inputs)
results = dict(zip(sorted(model.keys()), results[1:]))
print("TRN step:%-5d error_top1: %.4f, error_top5: %.4f, loss:%s" % (step,
results["error_top1"],
results["error_top5"],
results["loss"]))
with open("loss.log", "w+") as f:
f.write("{}\n".format(results["loss"]))
log.report(step=step,
split="TRN",
error_top5=float(results["error_top5"]),
error_top1=float(results["error_top5"]),
loss=float(results["loss"]))
# report evaluation metrics every 10 training steps
if (step % vld_iter == 0):
print("-- running evaluation on vld split")
X_vld = vld_data[0]
Y_vld = vld_data[1]
inputs = [input_data_tensor, input_label_tensor]
args = [X_vld, Y_vld]
ops = [model[k] for k in sorted(model.keys())]
results = tools.iterative_reduce(ops, inputs, args, batch_size=1, fn=lambda x: np.mean(x, axis=0))
results = dict(zip(sorted(model.keys()), results))
print("VLD step:%-5d error_top1: %.4f, error_top5: %.4f, loss:%s" % (step,
results["error_top1"],
results["error_top5"],
results["loss"]))
log.report(step=step,
split="VLD",
error_top5=float(results["error_top5"]),
error_top1=float(results["error_top1"]),
loss=float(results["loss"]))
if (step % checkpoint_iter == 0) or (step + 1 == num_steps):
print("-- saving check point")
tools.save_weights(G, pth.join(checkpoint_dir, "weights.%s" % step))
######################### TRAIN MODEL ################################
print '... training'
# Start Training Loop
epoch = 0
step_idx = 0
val_record = []
while epoch < config['n_epochs']:
epoch = epoch + 1
if config['resume_train'] and epoch == 1:
load_epoch = config['load_epoch']
load_weights(layers, config['weights_dir'], load_epoch)
lr_to_load = np.load(
config['weights_dir'] + 'lr_' + str(load_epoch) + '.npy')
val_record = list(
np.load(config['weights_dir'] + 'val_record.npy'))
learning_rate.set_value(lr_to_load)
load_momentums(vels, config['weights_dir'], load_epoch)
epoch = load_epoch + 1
n_batches = len(train_set) / batch_size
for it in range(n_batches):
num_iter = (epoch - 1) * len(train_set) + it
print 'epoch', epoch, 'num_iter', num_iter, '/', n_batches
with Timer('sample minibatch'):
(batch_x, batch_y) = sample_minibatch(train_set, batch_size)
def train(train_data_generator):
checkpoint_dir = config["checkpoint_dir"]
learning_rate = config['learning_rate']
data_dims = config['data_dims']
batch_size = config['batch_size']
num_gpus = config['num_gpus']
num_epochs = config['num_epochs']
num_samples_per_epoch = config["num_samples_per_epoch"]
pretrained_weights = config["pretrained_weights"]
steps_per_epoch = num_samples_per_epoch // (batch_size * num_gpus)
num_steps = steps_per_epoch * num_epochs
checkpoint_iter = config["checkpoint_iter"]
experiment_dir = config['experiment_dir']
train_log_fpath = pth.join(experiment_dir, 'train.log')
log = tools.MetricsLogger(train_log_fpath)
# =====================
# define training graph
# =====================
G = tf.Graph()
with G.as_default(), tf.device('/cpu:0'):
full_data_dims = [batch_size * num_gpus] + data_dims
data = tf.placeholder(dtype=tf.float32,
shape=full_data_dims,
name='data')
labels = tf.placeholder(dtype=tf.int32,
shape=[batch_size * num_gpus],
name='labels')
# we split the large batch into sub-batches to be distributed onto each gpu
split_data = tf.split(0, num_gpus, data)
split_labels = tf.split(0, num_gpus, labels)
# setup optimizer
optimizer = tf.train.AdamOptimizer(learning_rate)
# setup one model replica per gpu to compute loss and gradient
replica_grads = []
for i in range(num_gpus):
with tf.name_scope('tower_%d' % i), tf.device('/gpu:%d' % i):
model = build_model(split_data[i], split_labels[i])
loss = model["loss"]
grads = optimizer.compute_gradients(loss)
replica_grads.append(grads)
tf.get_variable_scope().reuse_variables()
# We must calculate the mean of each gradient. Note this is a
# synchronization point across all towers.
average_grad = L.average_gradients(replica_grads)
grad_step = optimizer.apply_gradients(average_grad)
train_step = tf.group(grad_step)
init = tf.initialize_all_variables()
# ==================
# run training graph
# ==================
config_proto = tf.ConfigProto(allow_soft_placement=True)
sess = tf.Session(graph=G, config=config_proto)
sess.run(init)
tf.train.start_queue_runners(sess=sess)
with sess.as_default():
if pretrained_weights:
print("-- loading weights from %s" % pretrained_weights)
tools.load_weights(G, pretrained_weights)
for step in range(num_steps):
data_batch, label_batch = train_data_generator.next()
inputs = {data: data_batch, labels: label_batch}
results = sess.run([train_step, loss], inputs)
print("step:%s loss:%s" % (step, results[1]))
log.report(step=step, split="TRN", loss=float(results[1]))
if (step % checkpoint_iter == 0) or (step + 1 == num_steps):
print("-- saving check point")
tools.save_weights(G, pth.join(checkpoint_dir, "weights.%s" % step))
def train_net(config, private_config):
# UNPACK CONFIGS
(flag_para_load, train_filenames, val_filenames,
train_labels, val_labels, img_mean) = \
unpack_configs(config, ext_data=private_config['ext_data'],
ext_label=private_config['ext_label'])
gpu_send_queue = private_config['queue_gpu_send']
gpu_recv_queue = private_config['queue_gpu_recv']
# pycuda and zmq set up
drv.init()
dev = drv.Device(int(private_config['gpu'][-1]))
ctx = dev.make_context()
sock_gpu = zmq.Context().socket(zmq.PAIR)
if private_config['flag_client']:
sock_gpu.connect('tcp://*****:*****@ iter = ', num_iter
log_iter.write("%d\n" % num_iter)
log_iter.flush()
print 'training cost:', cost_ij
log_err_cost.write("%f\n" % cost_ij)
log_err_cost.flush()
if config['print_train_error']:
error_ij = train_error()
gpu_send_queue.put(error_ij)
that_error = gpu_recv_queue.get()
error_ij = (error_ij + that_error) / 2.
if private_config['flag_verbose']:
print 'training error rate:', error_ij
log_err_rate.write("%f\n" % error_ij)
log_err_rate.flush()
if flag_para_load and (count < len(minibatch_range)):
load_send_queue.put('calc_finished')
if count%20 == 0:
e = time.time()
print "time per 20 iter:", (e - s)
############### Test on Validation Set ##################
DropoutLayer.SetDropoutOff()
this_val_error, this_val_loss = get_val_error_loss(
rand_arr, shared_x, shared_y,
val_filenames, val_labels,
flag_para_load, img_mean,
batch_size, validate_model,
send_queue=load_send_queue, recv_queue=load_recv_queue)
# report validation stats
gpu_send_queue.put(this_val_error)
that_val_error = gpu_recv_queue.get()
this_val_error = (this_val_error + that_val_error) / 2.
gpu_send_queue.put(this_val_loss)
that_val_loss = gpu_recv_queue.get()
this_val_loss = (this_val_loss + that_val_loss) / 2.
if private_config['flag_verbose']:
print('epoch %i: validation loss %f ' %
(epoch, this_val_loss))
print('epoch %i: validation error %f %%' %
(epoch, this_val_error * 100.))
val_record.append([this_val_error, this_val_loss])
if private_config['flag_save']:
np.save(config['weights_dir'] + 'val_record.npy', val_record)
np.savetxt(config['weights_dir'] + 'val_record_txt.txt', val_record)
DropoutLayer.SetDropoutOn()
############################################
# Adapt Learning Rate
step_idx = adjust_learning_rate(config, epoch, step_idx,
val_record, learning_rate)
# Save Weights, only one of them will do
if private_config['flag_save']:
if epoch % config['snapshot_freq'] == 0:
save_weights(layers, config['weights_dir'], epoch)
np.save(config['weights_dir'] + 'lr_' + str(epoch) + '.npy',
learning_rate.get_value())
save_momentums(vels, config['weights_dir'], epoch)
print('Optimization complete.')
def train_net(config):
# UNPACK CONFIGS
(flag_para_load, train_filenames, val_filenames,
train_labels, val_labels, img_mean) = unpack_configs(config)
if flag_para_load:
# zmq set up
sock = zmq.Context().socket(zmq.PAIR)
sock.connect('tcp://*****:*****@iter " + str(count)
if count == 1:
s = time.time()
if count == 20:
e = time.time()
print "time per 20 iter:", (e - s)
logger.info("time per 20 iter: %f" % (e - s))
cost_ij = train_model_wrap(train_model, shared_x,
shared_y, rand_arr, img_mean,
count, minibatch_index,
minibatch_range, batch_size,
train_filenames, train_labels,
flag_para_load,
config['batch_crop_mirror'],
send_queue=load_send_queue,
recv_queue=load_recv_queue)
if num_iter % config['print_freq'] == 0:
#print 'training @ iter = ', num_iter
#print 'training cost:', cost_ij
logger.info("training @ iter = %i" % (num_iter))
logger.info("training cost: %lf" % (cost_ij))
if config['print_train_error']:
logger.info('training error rate: %lf' % train_error())
#print 'training error rate:', train_error()
if flag_para_load and (count < len(minibatch_range)):
load_send_queue.put('calc_finished')
############### Test on Validation Set ##################
#"""
DropoutLayer.SetDropoutOff()
# result_list = [ this_validation_error, this_validation_error_top5, this_validation_loss ]
# or
# result_list = [ this_validation_error, this_validation_loss ]
result_list = get_val_error_loss(
#this_validation_error, this_validation_loss = get_val_error_loss(
rand_arr, shared_x, shared_y,
val_filenames, val_labels,
flag_para_load, img_mean,
batch_size, validate_model,
send_queue=load_send_queue,
recv_queue=load_recv_queue,
flag_top_5=flag_top5)
logger.info(('epoch %i: validation loss %f ' %
(epoch, result_list[-1])))
#print('epoch %i: validation loss %f ' %
# (epoch, this_validation_loss))
if flag_top5:
logger.info(('epoch %i: validation error (top 1) %f %%, (top5) %f %%' %
(epoch, result_list[0] * 100., result_list[1] * 100.)))
else:
logger.info(('epoch %i: validation error %f %%' %
(epoch, result_list[0] * 100.)))
#print('epoch %i: validation error %f %%' %
# (epoch, this_validation_error * 100.))
val_record.append(result_list)
#val_record.append([this_validation_error, this_validation_loss])
np.save(config['weights_dir'] + 'val_record.npy', val_record)
DropoutLayer.SetDropoutOn()
############################################
# Adapt Learning Rate
step_idx = adjust_learning_rate(config, epoch, step_idx,
val_record, learning_rate)
# Save weights
if epoch % config['snapshot_freq'] == 0:
save_weights(layers, config['weights_dir'], epoch)
np.save(config['weights_dir'] + 'lr_' + str(epoch) + '.npy',
learning_rate.get_value())
save_momentums(vels, config['weights_dir'], epoch)
#"""
print('Optimization complete.')
def train_net(config, private_config):
# UNPACK CONFIGS
(flag_para_load, flag_datalayer, train_filenames, val_filenames,
train_labels, val_labels, img_mean) = \
unpack_configs(config, ext_data=private_config['ext_data'],
ext_label=private_config['ext_label'])
gpu_send_queue = private_config['queue_gpu_send']
gpu_recv_queue = private_config['queue_gpu_recv']
# pycuda and zmq set up
drv.init()
dev = drv.Device(int(private_config['gpu'][-1]))
ctx = dev.make_context()
sock_gpu = zmq.Context().socket(zmq.PAIR)
if private_config['flag_client']:
sock_gpu.connect('tcp://*****:*****@ iter = ', num_iter
print 'training cost:', cost_ij
if config['print_train_error']:
error_ij = train_error()
gpu_send_queue.put(error_ij)
that_error = gpu_recv_queue.get()
error_ij = (error_ij + that_error) / 2.
if private_config['flag_verbose']:
print 'training error rate:', error_ij
if flag_para_load and (count < len(minibatch_range)):
load_send_queue.put('calc_finished')
############### Test on Validation Set ##################
DropoutLayer.SetDropoutOff()
this_val_error, this_val_loss = get_val_error_loss(
rand_arr,
shared_x,
shared_y,
val_filenames,
val_labels,
flag_datalayer,
flag_para_load,
batch_size,
validate_model,
send_queue=load_send_queue,
recv_queue=load_recv_queue)
# report validation stats
gpu_send_queue.put(this_val_error)
that_val_error = gpu_recv_queue.get()
this_val_error = (this_val_error + that_val_error) / 2.
gpu_send_queue.put(this_val_loss)
that_val_loss = gpu_recv_queue.get()
this_val_loss = (this_val_loss + that_val_loss) / 2.
if private_config['flag_verbose']:
print('epoch %i: validation loss %f ' % (epoch, this_val_loss))
print('epoch %i: validation error %f %%' %
(epoch, this_val_error * 100.))
val_record.append([this_val_error, this_val_loss])
if private_config['flag_save']:
np.save(config['weights_dir'] + 'val_record.npy', val_record)
DropoutLayer.SetDropoutOn()
############################################
# Adapt Learning Rate
step_idx = adjust_learning_rate(config, epoch, step_idx, val_record,
learning_rate)
# Save Weights, only one of them will do
if private_config['flag_save']:
if epoch % config['snapshot_freq'] == 0:
save_weights(layers, config['weights_dir'], epoch)
np.save(config['weights_dir'] + 'lr_' + str(epoch) + '.npy',
learning_rate.get_value())
save_momentums(vels, config['weights_dir'], epoch)
print('Optimization complete.')
def train_net(config):
# UNPACK CONFIGS
(flag_para_load, train_filenames, val_filenames, train_labels, val_labels,
img_mean) = unpack_configs(config)
# pycuda set up
drv.init()
dev = drv.Device(int(config['gpu'][-1]))
ctx = dev.make_context()
if flag_para_load:
# zmq set up
sock = zmq.Context().socket(zmq.PAIR)
sock.connect('tcp://*****:*****@ iter = ', num_iter
print 'training cost:', cost_ij
if config['print_train_error']:
print 'training error rate:', train_error()
if flag_para_load and (count < len(minibatch_range)):
load_send_queue.put('calc_finished')
############### Test on Validation Set ##################
DropoutLayer.SetDropoutOff()
this_validation_error, this_validation_loss = get_val_error_loss(
rand_arr,
shared_x,
shared_y,
val_filenames,
val_labels,
flag_para_load,
img_mean,
batch_size,
validate_model,
send_queue=load_send_queue,
recv_queue=load_recv_queue)
print('epoch %i: validation loss %f ' % (epoch, this_validation_loss))
print('epoch %i: validation error %f %%' %
(epoch, this_validation_error * 100.))
val_record.append([this_validation_error, this_validation_loss])
np.save(config['weights_dir'] + 'val_record.npy', val_record)
DropoutLayer.SetDropoutOn()
############################################
# Adapt Learning Rate
step_idx = adjust_learning_rate(config, epoch, step_idx, val_record,
learning_rate)
# Save weights
if epoch % config['snapshot_freq'] == 0:
save_weights(layers, config['weights_dir'], epoch)
np.save(config['weights_dir'] + 'lr_' + str(epoch) + '.npy',
learning_rate.get_value())
save_momentums(vels, config['weights_dir'], epoch)
print('Optimization complete.')
def validate_performance(config):
# UNPACK CONFIGS
(flag_para_load, train_filenames, val_filenames, train_labels, val_labels,
img_mean) = unpack_configs(config)
if flag_para_load:
# pycuda and zmq set up
drv.init()
dev = drv.Device(int(config['gpu'][-1]))
ctx = dev.make_context()
sock = zmq.Context().socket(zmq.PAIR)
sock.connect('tcp://localhost:{0}'.format(config['sock_data']))
load_send_queue = config['queue_t2l']
load_recv_queue = config['queue_l2t']
else:
load_send_queue = None
load_recv_queue = None
import theano.sandbox.cuda
theano.sandbox.cuda.use(config['gpu'])
import theano
theano.config.on_unused_input = 'warn'
from layers import DropoutLayer
from alex_net import AlexNet, compile_models
import theano.misc.pycuda_init
import theano.misc.pycuda_utils
# # BUILD NETWORK ##
model = AlexNet(config)
layers = model.layers
batch_size = model.batch_size
# # COMPILE FUNCTIONS ##
(train_model, validate_model, train_error, learning_rate, shared_x,
shared_y, rand_arr, vels) = compile_models(model, config, flag_top_5=True)
print '... training'
if flag_para_load:
# pass ipc handle and related information
gpuarray_batch = theano.misc.pycuda_utils.to_gpuarray(
shared_x.container.value)
h = drv.mem_get_ipc_handle(gpuarray_batch.ptr)
sock.send_pyobj((gpuarray_batch.shape, gpuarray_batch.dtype, h))
load_send_queue.put(img_mean)
load_epoch = config['load_epoch']
load_weights(layers, config['weights_dir'], load_epoch)
DropoutLayer.SetDropoutOff()
this_validation_error, this_validation_error_top_5, this_validation_loss = \
get_val_error_loss(rand_arr, shared_x, shared_y,
val_filenames, val_labels,
flag_para_load,img_mean,
batch_size, validate_model,
send_queue=load_send_queue,
recv_queue=load_recv_queue,
flag_top_5=True)
print('validation error %f %%' % (this_validation_error * 100.))
print('top 5 validation error %f %%' %
(this_validation_error_top_5 * 100.))
print('validation loss %f ' % (this_validation_loss))
return this_validation_error, this_validation_loss
def train(trn_data_generator, vld_data=None):
learning_rate = config['learning_rate']
experiment_dir = config['experiment_dir']
data_dims = config['data_dims']
batch_size = config['batch_size']
num_epochs = config['num_epochs']
num_samples_per_epoch = config["num_samples_per_epoch"]
steps_per_epoch = num_samples_per_epoch // batch_size
num_steps = steps_per_epoch * num_epochs
checkpoint_dir = pth.join(experiment_dir, 'checkpoints')
train_log_fpath = pth.join(experiment_dir, 'train.log')
vld_iter = config["vld_iter"]
checkpoint_iter = config["checkpoint_iter"]
pretrained_weights = config.get("pretrained_weights", None)
# ========================
# construct training graph
# ========================
G = tf.Graph()
with G.as_default():
input_data_tensor = tf.placeholder(tf.float32, [None] + data_dims)
input_label_tensor = tf.placeholder(tf.int32, [None])
model = build_model(input_data_tensor, input_label_tensor)
optimizer = tf.train.AdamOptimizer(learning_rate)
grads = optimizer.compute_gradients(model["loss"])
grad_step = optimizer.apply_gradients(grads)
init = tf.initialize_all_variables()
operations_tensors = {}
operations_names = G.get_operations()
print(operations_names)
count1 = 0
count2 = 0
for operation in operations_names:
operation_name = operation.name
operations_info = G.get_operation_by_name(operation_name).values()
if len(operations_info) > 0:
if not (operations_info[0].shape.ndims is None):
operation_shape = operations_info[0].shape.as_list()
operation_dtype_size = operations_info[0].dtype.size
if not (operation_dtype_size is None):
operation_no_of_elements = 1
for dim in operation_shape:
if not (dim is None):
operation_no_of_elements = operation_no_of_elements * dim
total_size = operation_no_of_elements * operation_dtype_size
operations_tensors[operation_name] = total_size
else:
count1 = count1 + 1
else:
count1 = count1 + 1
operations_tensors[operation_name] = -1
# print('no shape_1: ' + operation_name)
# print('no shape_2: ' + str(operations_info))
# operation_namee = operation_name + ':0'
# tensor = tf.get_default_graph().get_tensor_by_name(operation_namee)
# print('no shape_3:' + str(tf.shape(tensor)))
# print('no shape:' + str(tensor.get_shape()))
else:
# print('no info :' + operation_name)
# operation_namee = operation.name + ':0'
count2 = count2 + 1
operations_tensors[operation_name] = -1
# try:
# tensor = tf.get_default_graph().get_tensor_by_name(operation_namee)
# print(tensor)
# print(tf.shape(tensor))
# except:
# print('no tensor: ' + operation_namee)
print(count1)
print(count2)
with open('tensors_sz.json', 'w') as f:
json.dump(operations_tensors, f)
# ===================================
# initialize and run training session
# ===================================
config_proto = tf.ConfigProto(log_device_placement=True,
allow_soft_placement=True)
sess = tf.Session(graph=G, config=config_proto)
run_metadata = tf.RunMetadata()
options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
def profile(run_metadata, epoch=0):
with open('profs/timeline_step' + str(epoch) + '.json', 'w') as f:
# Create the Timeline object, and write it to a json file
fetched_timeline = timeline.Timeline(run_metadata.step_stats)
chrome_trace = fetched_timeline.generate_chrome_trace_format()
f.write(chrome_trace)
sess.run(init, run_metadata=run_metadata, options=options)
profile(run_metadata, -1)
tf.train.start_queue_runners(sess=sess)
with sess.as_default():
if pretrained_weights:
print("-- loading weights from %s" % pretrained_weights)
tools.load_weights(G, pretrained_weights)
# Start training loop
num_steps = 100
tot_samples_p_second = 0
for step in range(num_steps):
batch_train = trn_data_generator.next()
X_trn = np.array(batch_train[0])
Y_trn = np.array(batch_train[1])
options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
ops = [grad_step] + [model[k] for k in sorted(model.keys())]
inputs = {input_data_tensor: X_trn, input_label_tensor: Y_trn}
start_time = time.time()
results = sess.run(ops,
feed_dict=inputs,
run_metadata=run_metadata,
options=options)
elapsed = time.time() - start_time
samples_p_second = float(batch_size) / float(elapsed)
print("Ex/sec: %.1f" % samples_p_second)
profile(run_metadata, step)
tot_samples_p_second += samples_p_second
results = dict(zip(sorted(model.keys()), results[1:]))
print("TRN step:%-5d error_top1: %.4f, error_top5: %.4f, loss:%s" %
(step, results["error_top1"], results["error_top5"],
results["loss"]))
# report evaluation metrics every 10 training steps
'''
if (step % vld_iter == 0):
print("-- running evaluation on vld split")
X_vld = vld_data[0]
Y_vld = vld_data[1]
inputs = [input_data_tensor, input_label_tensor]
args = [X_vld, Y_vld]
ops = [model[k] for k in sorted(model.keys())]
results = tools.iterative_reduce(ops, inputs, args, batch_size=1, fn=lambda x: np.mean(x, axis=0))
results = dict(zip(sorted(model.keys()), results))
print("VLD step:%-5d error_top1: %.4f, error_top5: %.4f, loss:%s" % (step,
results["error_top1"],
results["error_top5"],
results["loss"]))
if (step % checkpoint_iter == 0) or (step + 1 == num_steps):
print("-- saving check point")
tools.save_weights(G, pth.join(checkpoint_dir, "weights.%s" % step))
'''
print("Average ex/sec: %.1f" % float(tot_samples_p_second / num_steps))
def train_net(config):
# UNPACK CONFIGS
(flag_para_load, train_filenames, val_filenames,
train_labels, val_labels, img_mean) = unpack_configs(config)
# pycuda set up
drv.init()
dev = drv.Device(int(config['gpu'][-1]))
ctx = dev.make_context()
if flag_para_load:
# zmq set up
sock = zmq.Context().socket(zmq.PAIR)
sock.connect('tcp://*****:*****@ iter = ', num_iter
print 'training cost:', cost_ij
if config['print_train_error']:
print 'training error rate:', train_error()
if flag_para_load and (count < len(minibatch_range)):
load_send_queue.put('calc_finished')
############### Test on Validation Set ##################
DropoutLayer.SetDropoutOff()
this_validation_error, this_validation_loss = get_val_error_loss(
rand_arr, shared_x, shared_y,
val_filenames, val_labels,
flag_para_load, img_mean,
batch_size, validate_model,
send_queue=load_send_queue, recv_queue=load_recv_queue)
print('epoch %i: validation loss %f ' %
(epoch, this_validation_loss))
print('epoch %i: validation error %f %%' %
(epoch, this_validation_error * 100.))
val_record.append([this_validation_error, this_validation_loss])
np.save(config['weights_dir'] + 'val_record.npy', val_record)
DropoutLayer.SetDropoutOn()
############################################
# Adapt Learning Rate
step_idx = adjust_learning_rate(config, epoch, step_idx,
val_record, learning_rate)
# Save weights
if epoch % config['snapshot_freq'] == 0:
save_weights(layers, config['weights_dir'], epoch)
np.save(config['weights_dir'] + 'lr_' + str(epoch) + '.npy',
learning_rate.get_value())
save_momentums(vels, config['weights_dir'], epoch)
print('Optimization complete.')
def train_net(config):
# UNPACK CONFIGS
(flag_para_load, train_filenames, val_filenames,
train_labels, val_labels, img_mean) = unpack_configs(config)
if flag_para_load:
# zmq set up
sock = zmq.Context().socket(zmq.PAIR)
sock.connect('tcp://*****:*****@ iter = %i" % (num_iter))
logger.info("training cost: %lf" % (cost_ij))
if config['print_train_error']:
logger.info('training error rate: %lf' % train_error())
if flag_para_load and (count < len(minibatch_range)):
load_send_queue.put('calc_finished')
############### Test on Validation Set ##################
#"""
DropoutLayer.SetDropoutOff()
result_list = get_val_error_loss(
rand_arr, shared_x, shared_y,
val_filenames, val_labels,
flag_para_load, img_mean,
batch_size, validate_model,
send_queue=load_send_queue,
recv_queue=load_recv_queue,
flag_top_5=flag_top5)
logger.info(('epoch %i: validation loss %f ' %
(epoch, result_list[-1])))
if flag_top5:
logger.info(('epoch %i: validation error (top 1) %f %%, (top5) %f %%' %
(epoch, result_list[0] * 100., result_list[1] * 100.)))
else:
logger.info(('epoch %i: validation error %f %%' %
(epoch, result_list[0] * 100.)))
val_record.append(result_list)
np.save(config['weights_dir'] + 'val_record.npy', val_record)
DropoutLayer.SetDropoutOn()
############################################
# Adapt Learning Rate
step_idx = adjust_learning_rate(config, epoch, step_idx,
val_record, learning_rate)
# Save weights
if epoch % config['snapshot_freq'] == 0:
save_weights(layers, config['weights_dir'], epoch)
np.save(config['weights_dir'] + 'lr_' + str(epoch) + '.npy',
learning_rate.get_value())
save_momentums(vels, config['weights_dir'], epoch)
#"""
print('Optimization complete.')
def code_extraction(config):
# UNPACK CONFIGS
(train_filenames, val_filenames, img_mean) = unpack_configs(config)
import theano.sandbox.cuda
theano.sandbox.cuda.use(config['gpu'])
import theano
theano.config.on_unused_input = 'warn'
import theano.tensor as T
from multilabel_layers import DropoutLayer
from multilabel_net import CNN_model, compile_models
import theano.misc.pycuda_init
import theano.misc.pycuda_utils
model = CNN_model(config)
batch_size = model.batch_size
layers = model.layers
n_train_batches = len(train_filenames)
n_val_batches = len(val_filenames)
## COMPILE FUNCTIONS ##
(predict_model, shared_x) = compile_models(model, config)
load_weights_epoch = 8
train_predicted_code = None
val_predicted_code = None
load_weights_dir = config['weights_dir']
load_weights(layers, load_weights_dir, load_weights_epoch)
code_save_dir = config['code_save_dir']
DropoutLayer.SetDropoutOff()
for minibatch_index in range(n_train_batches):
label = get_prediction_labels(predict_model, shared_x, train_filenames,
minibatch_index, img_mean)
if train_predicted_code is None:
train_predicted_code = label[0]
else:
train_predicted_code = np.vstack((train_predicted_code, label[0]))
database_code = {'database_code': train_predicted_code}
sio.savemat(code_save_dir + 'database_code.mat', database_code)
for minibatch_index in range(n_val_batches):
label = get_prediction_labels(predict_model, shared_x, val_filenames,
minibatch_index, img_mean)
if val_predicted_code is None:
val_predicted_code = label[0]
else:
val_predicted_code = np.vstack((val_predicted_code, label[0]))
test_code = {'test_code': val_predicted_code}
sio.savemat(code_save_dir + 'test_code.mat', test_code)
print('code extraction complete.')
import argparse
from skimage.transform import resize
from skimage.io import imread
G = tf.Graph()
with G.as_default():
images = tf.placeholder("float", [1, 224, 224, 3])
logits = vgg.build(images, n_classes=10, training=False)
probs = tf.nn.softmax(logits)
def predict(im):
labels = ['airplane', 'automobile', 'bird', 'cat', 'deer',
'dog', 'frog', 'horse', 'ship', 'truck']
if im.shape != (224, 224, 3):
im = resize(im, (224, 224))
im = np.expand_dims(im, 0)
sess = tf.get_default_session()
results = sess.run(probs, {images: im})
return labels[np.argmax(results)]
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("-w", "--weights", required=True, help="path to weights.npz file")
parser.add_argument("image", help="path to jpg image")
args = parser.parse_args()
im = imread(args.image)
sess = tf.Session(graph=G)
with sess.as_default():
tools.load_weights(G, args.weights)
print predict(im)
rand_arr, vels) = compile_models(model, config)
######################### TRAIN MODEL ################################
print '... training'
# Start Training Loop
epoch = 0
step_idx = 0
val_record = []
while epoch < config['n_epochs']:
epoch = epoch + 1
if config['resume_train'] and epoch == 1:
load_epoch = config['load_epoch']
load_weights(layers, config['weights_dir'], load_epoch)
lr_to_load = np.load(config['weights_dir'] + 'lr_' + str(load_epoch) +
'.npy')
val_record = list(np.load(config['weights_dir'] + 'val_record.npy'))
learning_rate.set_value(lr_to_load)
load_momentums(vels, config['weights_dir'], load_epoch)
epoch = load_epoch + 1
n_batches = len(train_set) / batch_size
for it in range(n_batches):
num_iter = (epoch - 1) * len(train_set) + it
print 'epoch', epoch, 'num_iter', num_iter, '/', n_batches
with Timer('sample minibatch'):
(batch_x, batch_y) = sample_minibatch(train_set, batch_size)
shared_x.set_value(batch_x)