def gen_batch(batch_size, num_workers=0):
sample_list = copy(sample_list_)
random.shuffle(sample_list)
# Set up the parallel generator
if num_workers > 0:
# Set up the queues
img_template = np.zeros((batch_size, self.preset.image_size.h,
self.preset.image_size.w, 3),
dtype=np.float32)
label_template = np.zeros((batch_size, self.preset.num_anchors,
self.num_classes + 5),
dtype=np.float32)
max_size = num_workers * 10
n_batches = int(math.ceil(len(sample_list_) / batch_size))
sample_queue = mp.Queue(n_batches)
batch_queue = DataQueue(img_template, label_template, max_size)
# Set up the workers. Make sure we can fork safely even if
# OpenCV has been compiled with CUDA and multi-threading
# support.
workers = []
os.environ['CUDA_VISIBLE_DEVICES'] = ""
cv2_num_threads = cv2.getNumThreads()
cv2.setNumThreads(1)
for i in range(num_workers):
args = (sample_queue, batch_queue)
w = mp.Process(target=batch_producer, args=args)
workers.append(w)
w.start()
del os.environ['CUDA_VISIBLE_DEVICES']
cv2.setNumThreads(cv2_num_threads)
# Fill the sample queue with data
for offset in range(0, len(sample_list), batch_size):
samples = sample_list[offset:offset + batch_size]
sample_queue.put(samples)
# Return the data
for offset in range(0, len(sample_list), batch_size):
images, labels, gt_boxes = batch_queue.get()
num_items = len(gt_boxes)
yield images[:num_items], labels[:num_items], gt_boxes
# Join the workers
for w in workers:
w.join()
# Return a serial generator
else:
for offset in range(0, len(sample_list), batch_size):
samples = sample_list[offset:offset + batch_size]
images, labels, gt_boxes = process_samples(samples)
yield images, labels, gt_boxes
def make_data_queue(self):
# add script name to domains
for domain in self.domains:
domain.script_name = self.script_name
self.data_queue = DataQueue(self.config, self.domains, self._network.train_shape_converter())
class Trainer(object):
# default network name
#network_name = 'advanced_network'
def __init__(self, config):
# in and out tensors
self.config = config # TODO remove this when config is corrected
self.DxQy = lattice.TYPES[config.DxQy]()
self.network_dir = config.latnet_network_dir
self.seq_length = config.seq_length
self.gan = config.gan
self.train_iters = config.train_iters
gpus = config.gpus.split(',')
self.gpus = map(int, gpus)
self.loss_stats = {}
self.time_stats = {}
self.start_time = time.time()
self.tic = time.time()
self.toc = time.time()
self.stats_history_length = 300
@classmethod
def add_options(cls, group, network_name):
pass
def init_network(self):
self._network = self.network(self.config)
self._network.train_unroll()
def make_data_queue(self):
# add script name to domains
for domain in self.domains:
domain.script_name = self.script_name
self.data_queue = DataQueue(self.config, self.domains, self._network.train_shape_converter())
def train(self):
# steps per print (hard set for now untill done debugging)
steps_per_print = 20
while True:
# get batch of data
feed_dict = self.data_queue.minibatch()
feed_dict['phase'] = 1
# perform optimization step for gen
gen_names = ['gen_train_op', 'loss_gen']
if not self.gan:
gen_names += ['loss_l2']
if self.gan:
gen_names += ['loss_l1', 'loss_gen_un_class', 'loss_layer_l2', 'loss_gen_con_class']
gen_output = self._network.run(gen_names, feed_dict=feed_dict, return_dict=True)
if self.gan:
disc_names = ['disc_train_op', 'loss_disc', 'loss_disc_un_class', 'loss_disc_con_class']
disc_output = self._network.run(disc_names, feed_dict=feed_dict, return_dict=True)
gen_output.update(disc_output)
# update loss summary
self.update_loss_stats(gen_output)
# update time summary
self.update_time_stats()
# print required data and save
step = self._network.run('gen_global_step')
if step % steps_per_print == 0:
self.print_stats(self.loss_stats, self.time_stats, self.data_queue.queue_stats(), step)
# TODO integrat this into self.saver
tf_feed_dict = {}
for name in feed_dict.keys():
if type(feed_dict[name]) is tuple:
tf_feed_dict[self._network.in_tensors[name]] = feed_dict[name][0]
tf_feed_dict[self._network.in_pad_tensors[name]] = feed_dict[name][1]
else:
tf_feed_dict[self._network.in_tensors[name]] = feed_dict[name]
###
self._network.saver.save_summary(self._network.sess, tf_feed_dict, int(self._network.run('gen_global_step')))
if step % self.config.save_network_freq == 0:
self._network.saver.save_checkpoint(self._network.sess, int(self._network.run('gen_global_step')))
if step % 2000 == 0:
print("getting new data")
self.active_data_add()
# end simulation
if step > self.train_iters:
break
def active_data_add(self):
# TODO this should be cleaned up
loss_data_point_pair = []
for i in tqdm(xrange(1000)):
sim_index, data_point, feed_dict = self.data_queue.rand_data_point()
loss_names = ['loss_l2']
loss_output = self._network.run(loss_names, feed_dict=feed_dict, return_dict=True)
loss_data_point_pair.append((loss_output['loss_l2'], sim_index, data_point))
loss_data_point_pair.sort()
for i in xrange(100):
self.data_queue.add_data_point(loss_data_point_pair[-i][1], loss_data_point_pair[-i][2])
def update_loss_stats(self, output):
names = output.keys()
names.sort()
for name in names:
if 'loss' in name:
# update loss history
if name + '_history' not in self.loss_stats.keys():
self.loss_stats[name + '_history'] = []
self.loss_stats[name + '_history'].append(output[name])
if len(self.loss_stats[name + '_history']) > self.stats_history_length:
self.loss_stats[name + '_history'].pop(0)
# update loss
self.loss_stats[name] = float(output[name])
# update ave loss
self.loss_stats[name + '_ave'] = float(np.sum(np.array(self.loss_stats[name + '_history']))
/ len(self.loss_stats[name + '_history']))
# update var loss
self.loss_stats[name + '_std'] = np.sqrt(np.var(np.array(self.loss_stats[name + '_history'])))
def update_time_stats(self):
# stop timer
self.toc = time.time()
# update total run time
self.time_stats['run_time'] = int(time.time() - self.start_time)
# update total step time
self.time_stats['step_time'] = ((self.toc - self.tic) /
(self.config.batch_size * len(self.config.gpus)))
# update time history
if 'step_time_history' not in self.time_stats.keys():
self.time_stats['step_time_history'] = []
self.time_stats['step_time_history'].append(self.time_stats['step_time'])
if len(self.time_stats['step_time_history']) > self.stats_history_length:
self.time_stats['step_time_history'].pop(0)
# update time ave
self.time_stats['step_time_ave'] = float(np.sum(np.array(self.time_stats['step_time_history']))
/ len(self.time_stats['step_time_history']))
# start timer
self.tic = time.time()
def print_stats(self, loss_stats, time_stats, queue_stats, step):
loss_string = print_dict('LOSS STATS', loss_stats, 'blue')
time_string = print_dict('TIME STATS', time_stats, 'magenta')
queue_string = print_dict('QUEUE STATS', queue_stats, 'yellow')
print_string = loss_string + time_string + queue_string
os.system('clear')
print("TRAIN INFO - step " + str(step))
print(print_string)
import time
from data_loader import CifarDataLoader
from data_queue import DataQueue
dl = CifarDataLoader(augmentation=True)
data, labels, _, _ = dl.get_data()
print('making data q')
dq = DataQueue(data, labels, 128, capacity=100, threads=16)
print('done making q')
start = time.time()
dq.start()
for i in range(6000):
print('popping' + str(i))
x, y = dq.pop()
assert (x.shape[0] == 128), x.shape[0]
assert (y.shape[0] == 128), y.shape[0]
print(time.time() - start)
dq.stop()
Args:
Returns:
A created Keras model for sequence prediction.
"""
inputs = keras.layers.Input(batch_shape=(batch_size, None, 1))
lstm = keras.layers.LSTM(cell_size, return_sequences=True,
stateful=True)(inputs)
dense = keras.layers.Dense(1)(lstm)
return keras.models.Model(inputs=inputs, outputs=dense)
sequence_predictor = CreateSequencePredictorModel(batch_size=500, cell_size=10)
train_dataset = DataQueue(5000, 500).dataset
sequence_predictor.compile(loss="mean_squared_error",
optimizer=keras.optimizers.Adam(lr=0.05),
metrics=['mse'])
sequence_predictor.fit(train_dataset, epochs=5)
sequence_predictor_single_batch = CreateSequencePredictorModel(batch_size=1,
cell_size=10)
sequence_predictor_single_batch.set_weights(sequence_predictor.get_weights())
sequence_predictor_single_batch.compile(
loss="mean_squared_error",
optimizer=keras.optimizers.Adam(lr=0.05),
metrics=['mse'])
print(
def __init__(self, data_loader, layers=(16, 32, 64), residual_layers=(5, 5, 5), data_augmentation=True,
non_core_layers=(1, 1, 1), learning_rate=0.01, batch_size=128, zero_init=False):
"""
:param layers: tuple that has the depth dimension vector
"""
assert (len(layers) == len(residual_layers))
m_data, labels, self.valid_data, self.valid_labels = data_loader.get_data()
self.q = DataQueue(m_data, labels, batch_size, capacity=200, threads=32, data_aug=data_augmentation)
self.q.start()
n, y_dim, x_dim, channel = m_data.shape
y_dim = x_dim = 32
self.batch_size = batch_size
self.batch_len_in_epoch = int(math.ceil(n / self.batch_size)) - 1
self.layers = layers
self.residual_layers_between = residual_layers
self.x = tf.placeholder(tf.float32, shape=(batch_size, y_dim, x_dim, channel))
self.y = tf.placeholder(tf.int32, shape=(batch_size,))
self.phase = tf.placeholder(tf.bool, name='phase')
self.global_step = tf.Variable(0, trainable=False, name='global_step')
self.learning_rate = tf.Variable(learning_rate, trainable=False, dtype=tf.float32, name='learning_rate')
self.decrease_learning_rate = tf.assign(self.learning_rate, tf.multiply(self.learning_rate, 0.1))
# Layers
conv_layer = None
for i, depth in enumerate(layers):
if conv_layer is None:
conv_layer = residual_block(self.x, depth, block_num=str(depth), first_block=True, core=True,
is_training=self.phase)
else:
conv_layer = residual_block(conv_layer, depth, block_num=str(depth), first_block=False, core=True,
is_training=self.phase)
assert (conv_layer.get_shape()[-1] == depth)
for k in range(residual_layers[i]):
if k > residual_layers[i] - non_core_layers[i]:
# non-core layer
conv_layer = residual_block(conv_layer, depth, block_num=str(depth) + '_' + str(k),
zero_init=zero_init, is_training=self.phase)
else:
conv_layer = residual_block(conv_layer, depth, block_num=str(depth) + '_' + str(k), core=True,
zero_init=False, is_training=self.phase)
assert (conv_layer.get_shape()[-1] == depth)
self.update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
pool_layer = tf.reduce_mean(conv_layer, [1, 2])
self.logits = tf.layers.dense(tf.reshape(pool_layer, [batch_size, -1]), 10)
self.prediction = tf.argmax(self.logits, 1)
trainable_vars = tf.trainable_variables()
core_var_list = [v for v in trainable_vars if 'core' in v.name]
loss_l2 = tf.add_n([tf.nn.l2_loss(v) for v in trainable_vars if 'bn' not in v.name]) / 2 * 0.0001
self.loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=self.logits, labels=self.y)) + loss_l2
prediction = tf.equal(tf.argmax(self.logits, 1), tf.cast(self.y, tf.int64))
self.accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))
with tf.control_dependencies(self.update_ops):
with tf.name_scope('train'):
self.optimizer = tf.train.AdagradOptimizer(learning_rate=self.learning_rate)
self.train_op = self.optimizer.minimize(loss=self.loss, global_step=self.global_step)
# train core only
self.train_core_op = self.optimizer.minimize(loss=self.loss, var_list=core_var_list,
global_step=self.global_step)
self.saver = tf.train.Saver(max_to_keep=10)
class ResNetClassifier(object):
def __init__(self, data_loader, layers=(16, 32, 64), residual_layers=(5, 5, 5), data_augmentation=True,
non_core_layers=(1, 1, 1), learning_rate=0.01, batch_size=128, zero_init=False):
"""
:param layers: tuple that has the depth dimension vector
"""
assert (len(layers) == len(residual_layers))
m_data, labels, self.valid_data, self.valid_labels = data_loader.get_data()
self.q = DataQueue(m_data, labels, batch_size, capacity=200, threads=32, data_aug=data_augmentation)
self.q.start()
n, y_dim, x_dim, channel = m_data.shape
y_dim = x_dim = 32
self.batch_size = batch_size
self.batch_len_in_epoch = int(math.ceil(n / self.batch_size)) - 1
self.layers = layers
self.residual_layers_between = residual_layers
self.x = tf.placeholder(tf.float32, shape=(batch_size, y_dim, x_dim, channel))
self.y = tf.placeholder(tf.int32, shape=(batch_size,))
self.phase = tf.placeholder(tf.bool, name='phase')
self.global_step = tf.Variable(0, trainable=False, name='global_step')
self.learning_rate = tf.Variable(learning_rate, trainable=False, dtype=tf.float32, name='learning_rate')
self.decrease_learning_rate = tf.assign(self.learning_rate, tf.multiply(self.learning_rate, 0.1))
# Layers
conv_layer = None
for i, depth in enumerate(layers):
if conv_layer is None:
conv_layer = residual_block(self.x, depth, block_num=str(depth), first_block=True, core=True,
is_training=self.phase)
else:
conv_layer = residual_block(conv_layer, depth, block_num=str(depth), first_block=False, core=True,
is_training=self.phase)
assert (conv_layer.get_shape()[-1] == depth)
for k in range(residual_layers[i]):
if k > residual_layers[i] - non_core_layers[i]:
# non-core layer
conv_layer = residual_block(conv_layer, depth, block_num=str(depth) + '_' + str(k),
zero_init=zero_init, is_training=self.phase)
else:
conv_layer = residual_block(conv_layer, depth, block_num=str(depth) + '_' + str(k), core=True,
zero_init=False, is_training=self.phase)
assert (conv_layer.get_shape()[-1] == depth)
self.update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
pool_layer = tf.reduce_mean(conv_layer, [1, 2])
self.logits = tf.layers.dense(tf.reshape(pool_layer, [batch_size, -1]), 10)
self.prediction = tf.argmax(self.logits, 1)
trainable_vars = tf.trainable_variables()
core_var_list = [v for v in trainable_vars if 'core' in v.name]
loss_l2 = tf.add_n([tf.nn.l2_loss(v) for v in trainable_vars if 'bn' not in v.name]) / 2 * 0.0001
self.loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=self.logits, labels=self.y)) + loss_l2
prediction = tf.equal(tf.argmax(self.logits, 1), tf.cast(self.y, tf.int64))
self.accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))
with tf.control_dependencies(self.update_ops):
with tf.name_scope('train'):
self.optimizer = tf.train.AdagradOptimizer(learning_rate=self.learning_rate)
self.train_op = self.optimizer.minimize(loss=self.loss, global_step=self.global_step)
# train core only
self.train_core_op = self.optimizer.minimize(loss=self.loss, var_list=core_var_list,
global_step=self.global_step)
self.saver = tf.train.Saver(max_to_keep=10)
def get_batch(self):
return self.q.pop()
def replace_data_queue(self, new_queue):
"""
Replaces the old data queue
:param new_queue: a new data queue
:return: None
"""
self.q.stop()
self.q = None
self.q = new_queue
self.q.start()
# def get_training_batch(self, index):
# data = self.m_data
# index = index % self.batch_len_in_epoch
# if (index + 1) * self.batch_size <= len(data):
# res = data[index * self.batch_size:(index + 1) * self.batch_size]
# labels = self.labels[index * self.batch_size:(index + 1) * self.batch_size]
# else:
# res = data[index * self.batch_size:] + data[:len(data) - (index * self.batch_size)]
# labels = self.labels[index * self.batch_size:] + self.labels[:len(data) - (index * self.batch_size)]
#
# res = np.array(res).astype(np.float32)
# return res, np.array(labels)
def get_validation_accuracy_op(self, sess):
return self.get_data_accuracy(sess, self.valid_data, self.valid_labels)
def get_data_accuracy(self, sess, data, labels):
batch_num = int(math.ceil(len(data) / self.batch_size))
length, y_dim, x_dim, channel = data.shape
prediction_holder = None
for index in range(batch_num):
if (index + 1) * self.batch_size <= len(data):
x = data[index * self.batch_size:(index + 1) * self.batch_size]
else:
data_last = data[index * self.batch_size:]
x = np.vstack((data_last, np.zeros((self.batch_size - len(data_last), y_dim, x_dim, channel))))
prediction = sess.run(self.prediction, feed_dict={self.x: x, self.phase: 0})
if prediction_holder is None:
prediction_holder = prediction
else:
prediction_holder = np.concatenate((prediction_holder, prediction))
return get_accuracy(prediction_holder[:len(data)], labels)
feed_dict={
model.x: x,
model.y: y,
model.phase: 1
})
# train_writer.add_summary(summary, step)
step = tf.train.global_step(sess, model.global_step)
train_accs.append(t_acc)
if step == FLAGS.change_data_set_step and FLAGS.partition > 1:
# change dataset
new_loader = CifarDataLoader(augmentation=FLAGS.data_augmentation)
m_data, labels, _, _ = new_loader.get_data()
new_q = DataQueue(m_data,
labels,
FLAGS.batch_size,
capacity=200,
threads=32,
data_aug=FLAGS.data_augmentation)
model.replace_data_queue(new_q)
if step == FLAGS.progressive_step and FLAGS.progressive:
print("Now training core")
train_op = model.train_op
acc_op = model.accuracy
if step in [int(float(e)) for e in FLAGS.steps[:-1]]:
print("decreasing learning rate")
learning_rate = sess.run(model.learning_rate)
sess.run(tf.assign(model.learning_rate, learning_rate / 10))
learning_rate = sess.run(model.learning_rate)
print('learning rate is %f' % learning_rate)