def __init__(self, args, sess):
self.args = args
self.sess = sess
'''
tf.Graph is a collection of operation
This method should be used if you want to create multiple graphs in the sampe process
The default graph is a property of the current thread. If you create a new thread, and wish to use the default graph
in that thread, you must explicitly add a 'with as_default() in that thread function
'''
# with tf.Graph().as_default():
self.global_step = tf.Variable(0, trainable=False)
# Get waves and labels
import multiprocessing
num_threads = multiprocessing.cpu_count() // self.args.num_gpu
print('Load data with %d threads' % num_threads)
with tf.device('/cpu:0'):
print('\tLoading training data')
with tf.variable_scope('train_data'):
train_wave, train_label, train_seq_len = data_loader.get_batches(data_category='train', shuffle=self.args.shuffle, batch_size=self.args.batch_size, num_gpu=self.args.num_gpu, num_threads=num_threads)
print('\tLoading valid data')
with tf.variable_scope('valid_data'):
test_wave, test_label, test_seq_len = data_loader.get_batches(data_category='valid', shuffle=self.args.shuffle, batch_size=self.args.batch_size, num_gpu=self.args.num_gpu, num_threads=num_threads)
# Build model
self.train_net = Wavenet_Model(self.args, train_wave, train_label, train_seq_len, self.global_step, name='train')
self.train_net.build_model()
self.train_net.train_optimizer()
self.train_summary_op = tf.summary.merge_all()
self.valid_net = Wavenet_Model(self.args, test_wave, test_label, test_seq_len, self.global_step, name='valid', reuse=True)
self.valid_net.build_model()
# Checkpoint with maximum checkpoints to keep 5
self.saver = tf.train.Saver(max_to_keep=5)
def __init__(self, args, sess):
self.args = args
self.sess = sess
self.global_step = tf.Variable(0, trainable=False)
# Get test data
with tf.device('/cpu:0'):
print('\tLoading test data')
self.args.num_gpu = 1
test_wave, test_label, test_seq_len = data_loader.get_batches(
data_category='test',
shuffle=self.args.shuffle,
batch_size=self.args.batch_size,
num_gpu=self.args.num_gpu,
num_threads=1)
self.test_net = Wavenet_Model(self.args,
test_wave,
test_label,
test_seq_len,
self.global_step,
name='test')
self.test_net.build_model()
# To load checkpoint
self.saver = tf.train.Saver()
self.decode()
def evaluate(self, sess, X_eval):
# Create input for batch creation
X_files, y_labels = list(), list()
for genre, files in X_eval.items():
X_files.extend(files)
y_labels.extend([get_class_from_genre[genre]] * len(files))
# For each iteration, get batches
batches = DataLoader.get_batches(paths=X_files,
labels=y_labels,
is_train=False,
num_classes=self.num_classes)
eval_accuracy = 0.0
eval_iter = 0.0
for batch in batches:
X_ = batch['X']
X_ = np.reshape(X_, [X_.shape[0], X_.shape[1], X_.shape[2], 1])
y_ = batch['y']
feed_dict = {self.X: X_, self.y: y_, self.is_train: False}
accuracy = sess.run(self.accuracy_op, feed_dict=feed_dict)
eval_accuracy += accuracy
eval_iter += 1
return eval_accuracy / eval_iter
def test_rgb():
with tf.Graph().as_default() as g:
rgb_image = tf.placeholder(
tf.float32, [None, IMG_HEIGHT, IMG_WIDTH, IMG_RGB_CHANNEL],
name='rgb_image')
label = tf.placeholder(tf.int32, [None, args.class_number],
name='label')
is_training = tf.placeholder(tf.bool)
rgb_logits = two_stream_model(rgb_image, 'None', args.network,
args.class_number, args.keep_prob,
args.batch_size, FRAMES_PER_VIDEO,
is_training, 'rgb')
#Loss
with tf.name_scope('loss'):
rgb_loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=rgb_logits,
labels=label))
with tf.name_scope('accuracy'):
rgb_accuracy = tf.reduce_mean(
tf.cast(
tf.equal(tf.argmax(rgb_logits, 1), tf.argmax(label, 1)),
tf.float32))
restorer = tf.train.Saver()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config, graph=g) as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
restorer.restore(sess, MODEL_DIR)
g.finalize()
ls_epoch = 0
acc_epoch = 0
rgb_acc_epoch = 0
flow_acc_epoch = 0
batch_index = 0
v_step = 0
for i in range(len(test_video_indices) // args.batch_size):
v_step += 1
if i % 20 == 0:
print('{} / {}'.format(
i,
len(test_video_indices) // args.batch_size))
rgb_batch_data, batch_index = get_batches(
TEST_RGB_PATH, 'None', args.batch_size, test_video_indices,
batch_index, 'rgb')
ls, acc = sess.run(
[rgb_loss, rgb_accuracy],
feed_dict={
rgb_image: rgb_batch_data['images'],
label: rgb_batch_data['labels'],
is_training: False
})
ls_epoch += ls
acc_epoch += acc
print('Loss {}, acc {}'.format(ls_epoch / v_step,
acc_epoch / v_step))
def train(self, sess, X_train, X_val):
sess.run(tf.global_variables_initializer())
step = 0
losses = []
accuracies = []
for epoch in range(self.num_epochs):
for iteration in range(self.num_iterations):
# Create input for batch creation
X_files, y_labels = list(), list()
for genre, files in X_train.items():
# Pop first element
file = files.pop(0)
X_files.append(file)
y_labels.append(get_class_from_genre[genre])
# Add back to queue
files.append(file)
# For each iteration, get batches
batches = DataLoader.get_batches(paths=X_files,
labels=y_labels,
is_train=True,
num_classes=self.num_classes)
# Run each step
for batch in batches:
X_ = batch['X']
X_ = np.reshape(X_,
[X_.shape[0], X_.shape[1], X_.shape[2], 1])
y_ = batch['y']
feed_dict = {
self.X: X_,
self.y: y_,
self.is_train: True,
self.keep_prob: 0.7
}
fetches = [self.train_op, self.loss_op, self.accuracy_op]
_, loss, accuracy = sess.run(fetches, feed_dict=feed_dict)
losses.append(loss)
accuracies.append(accuracy)
if step % self.log_step == 0:
print(
'step (%d): loss = %.3f, accuracy = %.3f, time elapsed = %.3f minutes'
% (step, loss, accuracy,
((time.time() - self.start_time) / 60)))
step += 1
# Evaluate validation set for each epoch
print('Validation for epoch %d' % epoch)
val_accuracy = self.evaluate(sess, X_val)
print('>> EPOCH %d: Validation Accuracy = %.4f' %
(epoch, val_accuracy))
def test_get_batches(self):
np.random.seed(0)
batch_size = 2
base_seq_len = 10
data = th.randn(5000, batch_size) # 5000 is arbitrary
batches = dl.get_batches(data, base_seq_len, vary_seq_len=True)
# Assert that batches are ordered by decreasing sequence length
for i in range(len(batches)-1):
self.assertTrue(batches[i][0].size(0) >= batches[i+1][0].size(0))
# Assert that inputs / targets within a batch are of matching dimensions
for i in range(len(batches)):
self.assertTrue(batches[i][0].size() == batches[i][1].size())
# Assert that targets offset input by one
for inputs, targets in batches:
self.assertTrue(inputs[1:].sum() == targets[:-1].sum())
# assert that sequence lengths vary, a crude test here is to compare
# first and second from last batch
self.assertTrue(batches[0][0].size(0) > batches[-2][0].size(0))
def train_fusion():
with tf.Graph().as_default() as g:
flow_image = tf.placeholder(
tf.float32, [None, IMG_HEIGHT, IMG_WIDTH, IMG_FLOW_CHANNEL],
name='flow_image')
label = tf.placeholder(tf.int32, [None, args.class_number],
name='label')
is_training = tf.placeholder(tf.bool)
flow_logits = two_stream_model('None', flow_image, args.network,
args.class_number, args.keep_prob,
args.batch_size, FRAMES_PER_VIDEO,
is_training, 'flow')
flow_variables = restore_variables()
flow_restorer = tf.train.Saver(flow_variables)
#vgg_16 first layer fo flow model
if args.network == 'vgg_16':
fiw_variables = slim.get_variables_to_restore(
include=['flow_model/vgg_16/conv1/conv1_1'])
elif args.network == 'resnet_v1_50':
fiw_variables = slim.get_variables_to_restore(
include=['flow_model/resnet_v1_50/conv1/weights'])
flow_input_weights_restorer = tf.train.Saver(fiw_variables)
#Loss
with tf.name_scope('loss'):
flow_loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=flow_logits,
labels=label))
tf.summary.scalar('flow_loss', flow_loss)
#Accuracy
with tf.name_scope('accuracy'):
flow_accuracy = tf.reduce_mean(
tf.cast(
tf.equal(tf.argmax(flow_logits, 1), tf.argmax(label, 1)),
tf.float32))
tf.summary.scalar('flow_accuracy', flow_accuracy)
opt = tf.train.AdamOptimizer(args.lr)
optimizer = slim.learning.create_train_op(flow_loss, opt)
saver = tf.train.Saver()
summary_op = tf.summary.merge_all()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config, graph=g) as sess:
summary_writer = tf.summary.FileWriter(TRAIN_LOG_DIR, sess.graph)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
g.finalize()
if args.network == 'vgg_16':
flow_restorer.restore(sess, VGG_16_MODEL_DIR)
flow_input_weights_restorer.restore(sess,
FLOW_INPUT_WEIGHTS_VGG_16)
elif args.network == 'resnet_v1_50':
flow_restorer.restore(sess, RES_v1_50_MODEL_DIR)
flow_input_weights_restorer.restore(
sess, FLOW_INPUT_WEIGHTS_RES_v1_50)
step = 0
best_acc = 0.0
best_ls = 10000.0
best_val_acc = 0.0
best_val_ls = 10000.0
best_epoch = 0
for epoch in range(args.epoches):
acc_epoch = 0
ls_epoch = 0
batch_index = 0
for i in range(len(train_video_indices) // args.batch_size):
step += 1
start_time = time.time()
flow_batch_data, batch_index = get_batches(
'None', TRAIN_FLOW_PATH, args.batch_size,
train_video_indices, batch_index, 'flow')
_, ls, acc, summary = sess.run(
[optimizer, flow_loss, flow_accuracy, summary_op],
feed_dict={
flow_image: flow_batch_data['images'],
label: flow_batch_data['labels'],
is_training: True
})
ls_epoch += ls
acc_epoch += acc
if i % 10 == 0:
end_time = time.time()
print('runing time {} :'.format(end_time - start_time))
print('Epoch {}, step {}, loss {}, acc {}'.format(
epoch + 1, step, ls, acc))
summary_writer.add_summary(summary, step)
num = len(train_video_indices) // args.batch_size
if best_acc < acc_epoch / num:
best_acc = acc_epoch / num
if best_ls > ls_epoch / num:
best_ls = ls_epoch / num
print(
'=========\n Epoch {}, best acc {}, best ls {}, loss {}, acc {}======'
.format(epoch + 1, best_acc, best_ls, ls_epoch / num,
acc_epoch / num))
#validation
ls_epoch = 0
acc_epoch = 0
batch_index = 0
v_step = 0
for i in range(
len(validation_video_indices) // args.batch_size):
v_step += 1
flow_batch_data, batch_index = get_batches(
'None', VALIDATION_FLOW_PATH, args.batch_size,
validation_video_indices, batch_index, 'flow')
ls, acc = sess.run(
[flow_loss, flow_accuracy],
feed_dict={
flow_image: flow_batch_data['images'],
label: flow_batch_data['labels'],
is_training: False
})
ls_epoch += ls
acc_epoch += acc
if best_val_acc < acc_epoch / v_step:
best_val_acc = acc_epoch / v_step
best_epoch = epoch
saver.save(sess, TRAIN_CHECK_POINT + 'flow_train.ckpt')
if best_val_ls > ls_epoch / v_step:
best_val_ls = ls_epoch / v_step
print(
'Validation best epoch {}, best acc {}, best ls {}, acc {}, ls {}'
.format(best_epoch + 1, best_val_acc, best_val_ls,
ls_epoch / v_step, acc_epoch / v_step))