def predict_batch(self, val_list, mean, weight_file, result_file):
in_img_size = (227, 227) #(height, width)
dropout_rate = 0.5
num_classes = 2
train_layers = []
x = tf.placeholder(tf.float32, [1, in_img_size[0], in_img_size[1], 3])
y = tf.placeholder(tf.float32, [None, num_classes])
model = alexnet(x,
1.,
num_classes,
train_layers,
in_size=in_img_size,
weights_path=weight_file)
score = model.fc8
softmax = tf.nn.softmax(score)
val_generator = ImageDataGenerator(val_list,
horizontal_flip=False,
shuffle=False,
mean=mean,
scale_size=in_img_size,
nb_classes=num_classes)
precision = np.zeros((num_classes + 1, num_classes), dtype=np.float)
total_presion = 0.
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
tf.train.Saver().restore(sess, weight_file)
self._start_end_time[0] = time.clock()
for index in range(val_generator.data_size):
print 'handing %d / %d ...\r' % (index + 1,
val_generator.data_size),
img_ = val_generator.images[index]
label = val_generator.labels[index]
img = cv2.imread(img_)
img = cv2.resize(
img,
(val_generator.scale_size[1], val_generator.scale_size[0]))
img = img.reshape(1, val_generator.scale_size[0],
val_generator.scale_size[1], 3)
img = img.astype(np.float32)
probs = sess.run(softmax, feed_dict={x: img})
guess = np.argmax(probs)
if guess == label:
precision[guess][guess] += 1
total_presion += 1
else:
precision[guess][int(val_generator.labels[index])] += 1
self._start_end_time[1] = time.clock()
for i in range(num_classes):
for j in range(num_classes):
precision[num_classes][i] += precision[j][i]
for i in range(num_classes):
for j in range(num_classes):
precision[i][j] /= precision[num_classes][j]
total_presion /= val_generator.data_size
slaped = (self._start_end_time[1] -
self._start_end_time[0]) / val_generator.data_size
file = open(result_file, 'w')
file.write('model: ' + weight_file + '\n')
print '\n#####################################################################'
file.writelines([
'################################################################\n'
])
text_ = ''
for i in range(num_classes):
print ' %d' % i,
text_ += ' %d' % i
print '\n'
file.write(text_ + '\n')
for i in range(num_classes):
print ' %d' % i,
file.write(' ' + str(i))
for j in range(num_classes):
str_preci = ' %.2f' % precision[i][j]
print ' %.2f ' % precision[i][j],
file.write(str_preci)
print '\n'
file.write('\n')
print '\ntotal precision: %.2f' % total_presion
print 'average speed: %.4f / image' % slaped
str_preci = 'total precision: %.2f' % total_presion
file.writelines(['\n' + str_preci + '\n'])
str_slaped = 'average speed: %.4f s / image' % slaped
file.write(str_slaped + '\n')
file.close()
# Network params
dropout_rate = 0.5
num_classes = 50
train_layers = ["fc7", "fc8"]
# How often we want to write the tf.summary data to disk
display_step = 20
# Path for tf.summary.FileWriter and to store model checkpoints
filewriter_path = "../record/tensorboard"
# Place data loading and preprocessing on the cpu
with tf.device("/cpu:0"):
tr_data = ImageDataGenerator(train_file,
mode="training",
batch_size=batch_size,
num_classes=num_classes,
shuffle=True)
val_data = ImageDataGenerator(val_file,
mode="inference",
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the two different iterators
training_init_op = iterator.make_initializer(tr_data.data)
def main(unused_argv):
if FLAGS.job_name is None or FLAGS.job_name == '':
raise ValueError('Must specify an expilict job_name')
else:
print('job_name: %s' % FLAGS.job_name)
if FLAGS.task_index is None or FLAGS.task_index == '':
raise ValueError('Must specify an explicit task_index')
else:
print('task_index:%s' % FLAGS.task_index)
ps_spec = FLAGS.ps_hosts.split(',')
worker_spec = FLAGS.worker_hosts.split(',')
num_worker = len(worker_spec)
cluster = tf.train.ClusterSpec({'ps': ps_spec, 'worker': worker_spec})
kill_ps_queue = create_done_queue(num_worker)
server = tf.train.Server(cluster,
job_name=FLAGS.job_name,
task_index=FLAGS.task_index)
if FLAGS.job_name == 'ps':
# server.join()
with tf.Session(server.target) as sess:
for i in range(num_worker):
sess.run(kill_ps_queue.dequeue())
return
is_chief = (FLAGS.task_index == 0)
if FLAGS.use_gpu:
worker_device = '/job:worker/task:%d/gpu:%d' % (FLAGS.task_index,
FLAGS.gpu_id)
else:
worker_device = '/job:worker/task:%d/cpu:0' % FLAGS.task_index
with tf.device(
tf.train.replica_device_setter(worker_device=worker_device,
ps_device='/job:ps/cpu:0',
cluster=cluster)):
global_step = tf.Variable(0, name='global_step', trainable=False)
x = tf.placeholder(tf.float32, [None, 227, 227, 3], name='x')
y = tf.placeholder(tf.float32, [None, FLAGS.n_classes], name='y')
keep_prob = tf.placeholder(tf.float32, name='kp')
model = AlexNet(x, keep_prob, FLAGS.n_classes)
score = model.fc3
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=score))
tf.summary.scalar('loss', cross_entropy)
opt = get_optimizer('Adam', FLAGS.learning_rate)
if FLAGS.sync_replicas:
replicas_to_aggregate = num_worker
opt = tf.train.SyncReplicasOptimizer(
opt,
replicas_to_aggregate=replicas_to_aggregate,
total_num_replicas=num_worker,
use_locking=False,
name='sync_replicas')
train_op = opt.minimize(cross_entropy, global_step=global_step)
correct_prediction = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuary', accuracy)
if FLAGS.sync_replicas:
local_init_op = opt.local_step_init_op
if is_chief:
local_init_op = opt.chief_init_op
ready_for_local_init_op = opt.ready_for_local_init_op
chief_queue_runner = opt.get_chief_queue_runner()
init_token_op = opt.get_init_tokens_op()
init_op = tf.global_variables_initializer()
kill_ps_enqueue_op = kill_ps_queue.enqueue(1)
summary_op = tf.summary.merge_all()
writer = tf.summary.FileWriter(FLAGS.logdir)
saver = tf.train.Saver()
# train_dir = tempfile.mkdtemp()
if FLAGS.sync_replicas:
sv = tf.train.Supervisor(
is_chief=is_chief,
logdir=FLAGS.checkpoint,
init_op=init_op,
local_init_op=local_init_op,
ready_for_local_init_op=ready_for_local_init_op,
summary_op=summary_op,
saver=saver,
summary_writer=writer,
recovery_wait_secs=1,
global_step=global_step)
else:
sv = tf.train.Supervisor(is_chief=is_chief,
logdir=FLAGS.checkpoint,
init_op=init_op,
recovery_wait_secs=1,
summary_op=summary_op,
saver=saver,
summary_writer=writer,
global_step=global_step)
sess_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False,
device_filters=[
'/job:ps',
'/job:worker/task:%d' %
FLAGS.task_index
])
if is_chief:
print('Worker %d: Initailizing session...' % FLAGS.task_index)
else:
print('Worker %d: Waiting for session to be initaialized...' %
FLAGS.task_index)
sess = sv.prepare_or_wait_for_session(server.target,
config=sess_config)
print('Worker %d: Session initialization complete.' % FLAGS.task_index)
if FLAGS.sync_replicas and is_chief:
sess.run(init_token_op)
sv.start_queue_runners(sess, [chief_queue_runner])
train_generator = ImageDataGenerator(FLAGS.train_file,
horizontal_flip=True,
shuffle=True)
val_generator = ImageDataGenerator(FLAGS.val_file, shuffle=False)
# Get the number of training/validation steps per epoch
train_batches_per_epoch = np.floor(train_generator.data_size /
FLAGS.batch_size).astype(np.int16)
val_batches_per_epoch = np.floor(val_generator.data_size /
FLAGS.batch_size).astype(np.int16)
print("{} Start training...".format(datetime.now()))
print("{} Open Tensorboard at --logdir {}".format(
datetime.now(), FLAGS.logdir))
for epoch in range(FLAGS.num_epoches):
print("{} Epoch number: {}".format(datetime.now(), epoch + 1))
step = 1
while step < train_batches_per_epoch:
start_time = time.time()
# Get a batch of images and labels
batch_xs, batch_ys = train_generator.next_batch(
FLAGS.batch_size)
# And run the training op
_, loss, gstep = sess.run(
[train_op, cross_entropy, global_step],
feed_dict={
x: batch_xs,
y: batch_ys,
keep_prob: FLAGS.dropout
})
print('total step: %d, loss: %f' % (gstep, loss))
duration = time.time() - start_time
# Generate summary with the current batch of data and write to file
if step % FLAGS.display_step == 0:
s = sess.run(sv.summary_op,
feed_dict={
x: batch_xs,
y: batch_ys,
keep_prob: 1.
})
writer.add_summary(s,
epoch * train_batches_per_epoch + step)
# print
if step % 10 == 0:
print("[INFO] {} pics has trained. time using {}".format(
step * FLAGS.batch_size, duration))
step += 1
# Validate the model on the entire validation set
print("{} Start validation".format(datetime.now()))
test_acc = 0.
test_count = 0
for _ in range(val_batches_per_epoch):
batch_tx, batch_ty = val_generator.next_batch(FLAGS.batch_size)
acc = sess.run(accuracy,
feed_dict={
x: batch_tx,
y: batch_ty,
keep_prob: 1.
})
test_acc += acc
test_count += 1
test_acc /= test_count
print("Validation Accuracy = {} {}".format(datetime.now(),
test_acc))
# Reset the file pointer of t
# he image data generator
val_generator.reset_pointer()
train_generator.reset_pointer()
print("{} Saving checkpoint of model...".format(datetime.now()))
# save checkpoint of the model
checkpoint_name = os.path.join(
FLAGS.checkpoint, 'model_epoch' + str(epoch + 1) + '.ckpt')
save_path = sv.saver.save(sess, checkpoint_name)
print("{} Model checkpoint saved at {}".format(
datetime.now(), checkpoint_name))
display_step = 20 #check acc per epoach
train_layers = ['fc8', 'fc7', 'fc6']
#read all image path config
train_img_paths, train_labels = read_train_detail(train_file)
validate_img_paths, validate_labels = read_train_detail(Validate_file)
print("Total Dataset {}".format(int(len(train_labels) + len(validate_labels))))
print("Split to Training {} and to Validation {}".format(
len(train_labels), len(validate_labels)))
with tf.device('/cpu:0'):
tr_data = ImageDataGenerator(mode='training',
batch_size=batch_size,
num_classes=num_classes,
class_file=class_file,
shuffle=True,
img_paths=train_img_paths,
labels=train_labels)
val_data = ImageDataGenerator(mode='validation',
batch_size=batch_size,
num_classes=num_classes,
class_file=class_file,
shuffle=False,
img_paths=validate_img_paths,
labels=validate_labels)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
from datetime import datetime
from tensorflow.contrib.data import Iterator
# Path to the textfiles for the trainings and validation set
dataroot = '/scratch0/dataset/domain_generalization/kfold/'
test_file = '../data/sourceonly/art_painting/test.txt'
checkpoint_path = 'results/checkpoints'
num_classes = 7
scratch_layers = ['fc8']
batch_size = 1
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
test_data = ImageDataGenerator(test_file,
dataroot=dataroot,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(test_data.data.output_types,
test_data.data.output_shapes)
next_batch_test = iterator.get_next()
# Ops for initializing the two different iterators
test_init_op = iterator.make_initializer(test_data.data)
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3])
y = tf.placeholder(tf.float32, [batch_size, num_classes])
keep_prob = tf.placeholder(tf.float32)
def main(argv = None):
if (argv is None):
argv = sys.argv
try:
try:
opts = argv
first_time_load = True
parent_dir = './'
keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3']
prune_thresholds = {}
WITH_BIASES = False
save_for_next_iter = False
TEST = False
for key in keys:
prune_thresholds[key] = 0.
for item in opts:
print (item)
opt = item[0]
val = item[1]
if (opt == '-cRates'):
cRates = val
if (opt == '-first_time'):
first_time_load = val
if (opt == '-file_name'):
file_name = val
if (opt == '-train'):
TRAIN = val
if (opt == '-prune'):
PRUNE = val
if (opt == '-test'):
TEST = val
if (opt == '-parent_dir'):
parent_dir = val
if (opt == '-lr'):
lr = val
if (opt == '-with_biases'):
WITH_BIASES = val
if (opt == '-lambda1'):
lambda_1 = val
if (opt == '-lambda2'):
lambda_2 = val
if (opt == '-save'):
save_for_next_iter = val
if (opt == '-org_file_name'):
org_file_name = val
print('pruning thresholds are {}'.format(prune_thresholds))
except getopt.error, msg:
raise Usage(msg)
epochs = 200
dropout = 0.5
batch_size = 128
num_classes = 1000
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000
INITIAL_LEARNING_RATE = 0.001
INITIAL_LEARNING_RATE = lr
LEARNING_RATE_DECAY_FACTOR = 0.1
NUM_EPOCHS_PER_DECAY = 350.0
DISPLAY_FREQ = 10
TEST = 1
TRAIN_OR_TEST = 0
NUM_CHANNELS = 3
mask_dir = parent_dir
weights_dir = parent_dir
LOCAL_TEST = 0
file_name_part = compute_file_name(cRates)
if (save_for_next_iter):
(weights_mask, biases_mask)= initialize_weights_mask(first_time_load, mask_dir, 'mask'+org_file_name + '.pkl')
else:
(weights_mask, biases_mask)= initialize_weights_mask(True, mask_dir, 'mask'+file_name_part + '.pkl')
if (LOCAL_TEST):
index_file_dir = 'cpu_test_data/'
else:
meta_data_dir = '/local/scratch/share/ImageNet/ILSVRC/Data/CLS-LOC/'
index_file_dir = '/local/scratch/share/ImageNet/ILSVRC/ImageSets/CLS-LOC/'
if (TRAIN):
train_file_txt = index_file_dir + 'train.txt'
val_file_txt = index_file_dir + 'val.txt'
test_file_txt = index_file_dir + 'test.txt'
else:
test_file_txt = index_file_dir + 'test.txt'
test_file_txt = index_file_dir + 'val.txt'
# if (first_time_load):
# PREV_MODEL_EXIST = 0
# weights, biases = initialize_variables(PREV_MODEL_EXIST, '')
# else:
# PREV_MODEL_EXIST = 1
# if (save_for_next_iter):
# file_name_part = org_file_name
# else:
# file_name_part = compute_file_name(cRates)
# weights, biases = initialize_variables( PREV_MODEL_EXIST,
# weights_dir + 'weights' + file_name_part + '.pkl')
#
x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3])
y = tf.placeholder(tf.float32, [None, num_classes])
keep_prob = tf.placeholder(tf.float32)
# initilize the model from the class constructer
model = AlexNet(x, keep_prob, num_classes, weights_mask, new_model = first_time_load, weights_path = 'base.npy')
# model = AlexNet(x, keep_prob, num_classes, weights_mask, new_model = first_time_load)
score = model.fc8
softmax = tf.nn.softmax(score)
var_list = tf.trainable_variables()
var_name_list = [v.name for v in var_list]
with tf.name_scope("cross_ent"):
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = score, labels = y))
with tf.name_scope("train"):
# l1_norm = lambda_1 * l1
# l2_norm = lambda_2 * l2
# regulization_loss = l1_norm + l2_norm
# opt = tf.train.AdamOptimizer(lr)
# grads = opt.compute_gradients(loss)
# org_grads = [(ClipIfNotNone(grad), var) for grad, var in grads]
# train_step = opt.apply_gradients(org_grads)
print('check var list :{}'.format(var_name_list))
gradients = tf.gradients(loss, var_list)
gradients = list(zip(gradients, var_list))
opt = tf.train.AdamOptimizer(learning_rate=lr)
train_step = opt.apply_gradients(grads_and_vars=gradients)
with tf.name_scope("accuracy"):
correct_prediction = tf.equal(tf.argmax(score,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3']
# weights_new[key] = weights[key] * tf.constant(weights_mask[key], dtype=tf.float32)
#
# l1, l2, pred = cov_network(images, weights_new, biases, keep_prob)
# _, _, test_pred = cov_network(test_images, weights_new, biases, keep_prob)
# cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits = pred, labels = y)
#
# new_grads = mask_gradients(weights, org_grads, weights_mask, biases, biases_mask, WITH_BIASES)
# Apply gradients.
init = tf.global_variables_initializer()
accuracy_list = np.zeros(20)
train_acc_list = []
# Launch the graph
print('Graph launching ..')
if (TRAIN):
train_generator = ImageDataGenerator(train_file_txt,
horizontal_flip = False, shuffle = True)
val_generator = ImageDataGenerator(val_file_txt, shuffle = True)
# Get the number of training/validation steps per epoch
train_batches_per_epoch = train_generator.data_size / batch_size
val_batches_per_epoch = val_generator.data_size / batch_size
if (TEST):
test_generator = ImageDataGenerator(test_file_txt)
test_batches_per_epoch = test_generator.data_size / batch_size
print('data size is {}'.format(test_generator.data_size))
print('Number of test batches per epoch is {}'.format(test_batches_per_epoch))
with tf.Session() as sess:
sess.run(init)
epoch_acc = []
epoch_entropy = []
weights_mask = model.load_initial_weights(sess)
if (TRAIN):
print("{} Start training...".format(datetime.now()))
for i in range(0,epochs):
print("{} Epoch number: {}".format(datetime.now(), i+1))
for step in range(train_batches_per_epoch):
(batch_x, batch_y) = train_generator.next_batch(batch_size, meta_data_dir+'train/')
train_acc, cross_en = sess.run([accuracy, loss], feed_dict = {
x: batch_x,
y: batch_y,
keep_prob: 1.0})
if (step % DISPLAY_FREQ == 0):
if (PRUNE):
print('This is the {}th of {}pruning, time is {}'.format(
i,
cRates,
datetime.now()
))
# print("accuracy is {} and cross entropy is {}".format(
# train_acc,
# cross_en
# ))
accuracy_list = np.concatenate((np.array([train_acc]),accuracy_list[0:19]))
epoch_acc.append(train_acc)
epoch_entropy.append(cross_en)
if (step%(DISPLAY_FREQ*50) == 0 and step != 0):
train_acc_list.append(train_acc)
model.save_weights()
# file_name_part = compute_file_name(cRates)
# save_pkl_model(weights, biases, weights_dir, 'weights' + file_name_part + '.pkl')
# print("saved the network")
with open ('acc_hist.txt', 'wb') as f:
for item in epoch_acc:
f.write("{}\n".format(item))
with open ('entropy_hist.txt', 'wb') as f:
for item in epoch_entropy:
f.write("{}\n".format(item))
if (np.mean(accuracy_list) > 0.8):
accuracy_list = np.zeros(20)
test_acc = sess.run(accuracy, feed_dict = {
x: images_test,
y: labels_test,
keep_prob: 1.0})
print('test accuracy is {}'.format(test_acc))
if (test_acc > 0.823):
print("training accuracy is large, show the list: {}".format(accuracy_list))
_ = sess.run(train_step, feed_dict = {
x: batch_x,
y: batch_y,
keep_prob: dropout})
test_acc_list = []
for _ in range(val_batches_per_epoch):
# Taverse one epoch
(batch_tx, batch_ty) = val_generator.next_batch(batch_size, meta_data_dir + 'val/')
tmp_acc, c_pred, c_softmax = sess.run([accuracy, score, softmax], feed_dict = {
x: batch_tx,
y: batch_ty,
keep_prob: 1.0})
test_acc_list.append(tmp_acc)
test_acc_list = np.array(test_acc_list)
test_acc = np.mean(test_acc_list)
print("Time {}, Validation Accuracy = {}".format(datetime.now(), test_acc))
if (TEST):
test_acc_list = []
if (LOCAL_TEST):
image_dir = "cpu_test_data/tmp_images/"
imagenet_mean = np.array([104., 117., 124.], dtype = np.float32)
test_batches_per_epoch = 3
img_files = [os.path.join(image_dir, f) for f in os.listdir(image_dir) if f.endswith('.jpeg')]
imgs_test = []
for i,f in enumerate(img_files):
tmp = cv2.imread(f)
tmp = cv2.resize(tmp.astype(np.float32), (227,227))
tmp -= imagenet_mean[i]
tmp = tmp.reshape((1,227,227,3))
imgs_test.append(tmp)
names = []
probs = []
for step in range(test_batches_per_epoch):
prob = sess.run(softmax, feed_dict = {
x: imgs_test[step],
# y: labels_test,
keep_prob: 1.0})
name = class_names[np.argmax(prob)]
probs.append(np.max(prob))
names.append(name)
print("names are {}".format(names))
print("probs are {}".format(probs))
sys.exit()
else:
test_acc_list = []
# Taverse one epoch
for step in range(test_batches_per_epoch):
(batch_x, batch_y) = test_generator.next_batch(batch_size, meta_data_dir + 'val/')
tmp_acc, c_pred, c_softmax = sess.run([accuracy, score, softmax], feed_dict = {
x: batch_x,
y: batch_y,
keep_prob: 1.0})
test_acc_list.append(tmp_acc)
test_acc_list = np.array(test_acc_list)
test_acc = np.mean(test_acc_list)
print("test accuracy of AlexNet is {}".format(test_acc))
sys.exit()
def run_net(args):
print(args)
np.random.seed(1234)
# data
tr_data = ImageDataGenerator("train.txt",
img_size=args.img_size,
batch_size=args.batch_size,
num_classes=args.num_classes,
shuffle=True,
scale=args.scale)
val_data = ImageDataGenerator("val.txt",
img_size=args.img_size,
batch_size=args.batch_size,
num_classes=args.num_classes,
shuffle=False,
scale=args.scale)
# the input image from the
img = tf.keras.layers.Input([args.img_size, args.img_size, 3], name="img")
# Initialize model
cornet = CORNetZV(args.num_classes, args.dropout_rate)
cls = cornet.forward(img)
model_class = keras.models.Model(inputs=img, outputs=cls)
opti = keras.optimizers.SGD(learning_rate=args.learning_rate)
def logit_cost(y, x):
return tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=x))
model_class.compile(optimizer=opti, loss=logit_cost, metrics=["accuracy"])
print(model_class.summary())
# generate the output path
output_path = getOutputPath(args)
# callbacks
if args.no_log is False:
callbacks = [
keras.callbacks.ModelCheckpoint(str(output_path / "weights.hdf5"),
save_best_only=True,
save_weights_only=True),
TrainingHistory(output_path),
]
else:
callbacks = []
if args.v != 'None':
# get the data generator for the images from the neuronal data
v_data = ImageDataGeneratorV(args.v,
batch_size=args.v_batch_size,
length=len(tr_data),
crop=args.crop,
center=args.v_center,
scale=args.v_scale)
# take two images as input
img1 = tf.keras.layers.Input([args.img_size, args.img_size, 3],
name="img1")
img2 = tf.keras.layers.Input([args.img_size, args.img_size, 3],
name="img2")
# process then with the V1 part of the model
img1_v1 = cornet.forward_V1(img1)
img2_v2 = cornet.forward_V1(img2)
# calculate the cosine distance
rsa = CosineDistance(name="rsa")([
tf.keras.layers.Flatten()(img1_v1),
tf.keras.layers.Flatten()(img2_v2)
])
# multiply by weighting factor
lambd = tf.keras.layers.Input([1], name="lambd")
lambd_rsa = tf.keras.layers.Lambda(lambda x: x[0] * x[1],
name="rsa_lambda")([rsa, lambd])
model_both = keras.models.Model(inputs=[img, img1, img2, lambd],
outputs=[cls, lambd_rsa])
print(model_both.summary())
#tf.keras.utils.plot_model(model_both, to_file='model.png', show_shapes=True, expand_nested=True)
model_both.compile(optimizer=opti,
loss={
"class": logit_cost,
"rsa_lambda": "mean_absolute_error"
},
metrics=["accuracy"])
print(model_both.summary())
model_rsa = keras.models.Model(inputs=[img1, img2], outputs=[rsa])
model_rsa.compile(optimizer="adam",
loss={"rsa": "mean_absolute_error"},
metrics=["accuracy"])
class CustomCallback(keras.callbacks.Callback):
def __init__(self, data_gen):
self.data_gen = data_gen
def on_train_batch_begin(self, batch, logs=None):
if args.r_per_epoch is False:
i, o = self.data_gen[batch]
loss, class_loss, rsa_loss, class_acc, rsa_acc = model_both.evaluate(
i, o, verbose=False)
lambd = args.r * class_loss / rsa_loss
self.data_gen.lambd = lambd
def on_epoch_begin(self, epoch, logs=None):
if args.r_per_epoch is True:
import numpy as np
i, o = self.data_gen[np.random.randint(len(self.data_gen))]
loss, class_loss, rsa_loss, class_acc, rsa_acc = model_both.evaluate(
i, o, verbose=False)
lambd = args.r * class_loss / rsa_loss
self.data_gen.lambd = lambd
training_input = MergedGenerators(tr_data, v_data)
training_input2 = MergedGenerators(val_data,
v_data,
use_min_length=True)
# the initial epochs with teacher
model_both.fit(training_input,
validation_data=training_input2,
shuffle=False,
epochs=10,
callbacks=[CustomCallback(training_input)] + callbacks)
# the rest of the epochs without a teacher
model_class.fit(tr_data,
validation_data=val_data,
epochs=args.num_epochs,
callbacks=callbacks,
initial_epoch=10)
else:
# train without teacher signal
model_class.fit(tr_data,
validation_data=val_data,
epochs=args.num_epochs,
callbacks=callbacks)
def train_session(train_file, val_file, alexnet_file):
"""
Main Part of the finetuning Script.
"""
# Create parent path if it doesn't exist
if not os.path.isdir(checkpoint_path):
os.makedirs(checkpoint_path)
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
tr_data = ImageDataGenerator(train_file,
mode='training',
batch_size=batch_size,
num_classes=num_classes,
shuffle=True)
val_data = ImageDataGenerator(val_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
# create an reinitializable iterator given the dataset structure
iterator = tf.data.Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the two different iterators
training_init_op = iterator.make_initializer(tr_data.data)
validation_init_op = iterator.make_initializer(val_data.data)
# TF placeholder for graph input and output
#x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3])
#y = tf.placeholder(tf.float32, [batch_size, num_classes])
# Link variable to model output
score = model.fc8
# List of trainable variables of the layers we want to train
var_list = [
v for v in tf.trainable_variables()
if v.name.split('/')[0] in train_layers
]
# Op for calculating the loss
with tf.name_scope("cross_ent"):
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=score, labels=y))
# Train op
with tf.name_scope("train"):
# Get gradients of all trainable variables
gradients = tf.gradients(loss, var_list)
gradients = list(zip(gradients, var_list))
# Create optimizer and apply gradient descent to the trainable variables
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
train_op = optimizer.apply_gradients(grads_and_vars=gradients)
with tf.name_scope("test"):
prob = tf.nn.softmax(score, name='prob')
# Add gradients to summary
for gradient, var in gradients:
tf.summary.histogram(var.name + '/gradient', gradient)
# Add the variables we train to the summary
for var in var_list:
tf.summary.histogram(var.name, var)
# Add the loss to summary
tf.summary.scalar('cross_entropy', loss)
# Evaluation op: Accuracy of the model
with tf.name_scope("accuracy"):
correct_pred = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Add the accuracy to the summary
tf.summary.scalar('accuracy', accuracy)
# Merge all summaries together
merged_summary = tf.summary.merge_all()
# Initialize the FileWriter
writer = tf.summary.FileWriter(filewriter_path)
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
# Get the number of training/validation steps per epoch
train_batches_per_epoch = int(np.floor(tr_data.data_size / batch_size))
val_batches_per_epoch = int(np.floor(val_data.data_size / batch_size))
# Start Tensorflow session
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.5
with tf.Session(config=config) as sess:
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Add the model graph to TensorBoard
writer.add_graph(sess.graph)
# Load the pretrained weights into the non-trainable layer
model.load_initial_weights(sess)
print("{} Start training...".format(datetime.now()))
print("{} Open Tensorboard at --logdir {}".format(
datetime.now(), filewriter_path))
# Loop over number of epochs
for epoch in range(num_epochs):
# print("{} Epoch number: {}".format(datetime.now(), epoch+1))
# Initialize iterator with the training dataset
sess.run(training_init_op)
train_acc = 0.
train_count = 0
train_loss = 0
for step in range(train_batches_per_epoch):
# get next batch of data
img_batch, label_batch = sess.run(next_batch)
# And run the training op
train_op_return, train_acc_value, train_loss_value = sess.run(
(train_op, accuracy, loss),
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: dropout_rate
})
# Generate summary with the current batch of data and write to file
if step % display_step == 0:
s = sess.run(merged_summary,
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: 1.
})
writer.add_summary(s,
epoch * train_batches_per_epoch + step)
train_loss += train_loss_value
train_acc += train_acc_value
train_count += 1
train_acc /= train_count
train_loss /= train_count
# print("{} Training Loss = {:.4f}".format(datetime.now(),train_loss))
# print("{} Saving checkpoint of model...".format(datetime.now()))
# Validate the model on the entire validation set
# print("{} Start validation".format(datetime.now()))
sess.run(validation_init_op)
test_acc = 0.
test_count = 0
test_loss = 0
for _ in range(val_batches_per_epoch):
img_batch, label_batch = sess.run(next_batch)
acc, loss_value = sess.run((accuracy, loss),
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: 1.
})
test_loss += loss_value
test_acc += acc
test_count += 1
test_acc /= test_count
test_loss /= test_count
#print("{} test loss = {:.4f} acc = {:.4f}".format(datetime.now(),test_loss,test_acc))
print(
"speaker{} {}Epoch:{} Training loss= {:.4f} acc= {:.4f} test acc= {:.4f}"
.format(
alexnet_file.split('/')[-2], datetime.now(), epoch + 1,
train_loss, train_acc, test_acc))
# print("{} Validation Loss = {:.4f}".format(datetime.now(),test_loss))
# print("{} Saving checkpoint of model...".format(datetime.now()))
# save checkpoint of the model
checkpoint_name = os.path.join(
checkpoint_path, 'model_epoch' + str(epoch + 1) + '.ckpt')
save_path = saver.save(sess, checkpoint_name)
# print("{} Model checkpoint saved at {}".format(datetime.now(),checkpoint_name))
graph = tf.graph_util.convert_variables_to_constants(
sess, sess.graph_def, ['test/prob'])
tf.train.write_graph(graph, '.', alexnet_file, as_text=False)
def main():
args = parser.parse_args()
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
######################
# directory preparation
filewriter_path = args.tensorboard_dir
checkpoint_path = args.checkpoint_dir
test_mkdir(filewriter_path)
test_mkdir(checkpoint_path)
######################
# data preparation
train_file = os.path.join(args.list_dir, "train.txt")
val_file = os.path.join(args.list_dir, "val.txt")
train_generator = ImageDataGenerator(train_file, shuffle=True)
val_generator = ImageDataGenerator(val_file, shuffle=False)
batch_size = args.batch_size
train_batches_per_epoch = train_generator.data_size
val_batches_per_epoch = val_generator.data_size
######################
# model graph preparation
patch_height = args.patch_size
patch_width = args.patch_size
batch_size = args.batch_size
# TF placeholder for graph input
leftx = tf.placeholder(tf.float32,
shape=[batch_size, patch_height, patch_width, 1])
rightx_pos = tf.placeholder(
tf.float32, shape=[batch_size, patch_height, patch_width, 1])
rightx_neg = tf.placeholder(
tf.float32, shape=[batch_size, patch_height, patch_width, 1])
# Initialize model
left_model = NET(leftx,
input_patch_size=patch_height,
batch_size=batch_size)
right_model_pos = NET(rightx_pos,
input_patch_size=patch_height,
batch_size=batch_size)
right_model_neg = NET(rightx_neg,
input_patch_size=patch_height,
batch_size=batch_size)
featuresl = tf.squeeze(left_model.features, [1, 2])
featuresr_pos = tf.squeeze(right_model_pos.features, [1, 2])
featuresr_neg = tf.squeeze(right_model_neg.features, [1, 2])
# Op for calculating cosine distance/dot product
with tf.name_scope("correlation"):
cosine_pos = tf.reduce_sum(tf.multiply(featuresl, featuresr_pos),
axis=-1)
cosine_neg = tf.reduce_sum(tf.multiply(featuresl, featuresr_neg),
axis=-1)
# Op for calculating the loss
with tf.name_scope("hinge_loss"):
margin = tf.ones(shape=[batch_size], dtype=tf.float32) * args.margin
loss = tf.maximum(0.0, margin - cosine_pos + cosine_neg)
loss = tf.reduce_mean(loss)
# Train op
with tf.name_scope("train"):
var_list = tf.trainable_variables()
for var in var_list:
print("{}: {}".format(var.name, var.shape))
# Get gradients of all trainable variables
gradients = tf.gradients(loss, var_list)
gradients = list(zip(gradients, var_list))
# Create optimizer and apply gradient descent with momentum to the trainable variables
optimizer = tf.train.MomentumOptimizer(args.learning_rate, args.beta)
train_op = optimizer.apply_gradients(grads_and_vars=gradients)
# summary Ops for tensorboard visualization
with tf.name_scope("training_metric"):
training_summary = []
# Add loss to summary
training_summary.append(tf.summary.scalar('hinge_loss', loss))
# Merge all summaries together
training_merged_summary = tf.summary.merge(training_summary)
# validation loss
with tf.name_scope("val_metric"):
val_summary = []
val_loss = tf.placeholder(tf.float32, [])
# Add val loss to summary
val_summary.append(tf.summary.scalar('val_hinge_loss', val_loss))
val_merged_summary = tf.summary.merge(val_summary)
# Initialize the FileWriter
writer = tf.summary.FileWriter(filewriter_path)
# Initialize an saver for store model checkpoints
saver = tf.train.Saver(max_to_keep=10)
######################
# DO training
# Start Tensorflow session
with tf.Session(config=tf.ConfigProto(
log_device_placement=False, \
allow_soft_placement=True, \
gpu_options=tf.GPUOptions(allow_growth=True))) as sess:
# Initialize all variables
sess.run(tf.global_variables_initializer())
# resume from checkpoint or not
if args.resume is None:
# Add the model graph to TensorBoard before initial training
writer.add_graph(sess.graph)
else:
#saver.restore(sess, args.resume)
checkpoint = args.resume
ckpt = tf.train.get_checkpoint_state(checkpoint)
print("{}: restoring from {}...".format(datetime.now(),
checkpoint))
print("{}: restoring from {}...".format(
datetime.now(), ckpt.model_checkpoint_path))
saver.restore(sess, ckpt.model_checkpoint_path)
print("training_batches_per_epoch: {}, val_batches_per_epoch: {}.".format(\
train_batches_per_epoch, val_batches_per_epoch))
print("{} Start training...".format(datetime.now()))
print("{} Open Tensorboard at --logdir {}".format(
datetime.now(), filewriter_path))
# Loop training
for epoch in range(args.start_epoch, args.end_epoch):
print("{} Epoch number: {}".format(datetime.now(), epoch + 1))
for batch in tqdm(range(train_batches_per_epoch)):
# Get a batch of data
batch_left, batch_right_pos, batch_right_neg = train_generator.next_batch(
batch_size)
# And run the training op
sess.run(train_op,
feed_dict={
leftx: batch_left,
rightx_pos: batch_right_pos,
rightx_neg: batch_right_neg
})
# Generate summary with the current batch of data and write to file
if (batch + 1) % args.print_freq == 0:
s = sess.run(training_merged_summary,
feed_dict={
leftx: batch_left,
rightx_pos: batch_right_pos,
rightx_neg: batch_right_neg
})
writer.add_summary(s,
epoch * train_batches_per_epoch + batch)
if (epoch + 1) % args.save_freq == 0:
print("{} Saving checkpoint of model...".format(
datetime.now()))
# save checkpoint of the model
checkpoint_name = os.path.join(
checkpoint_path, 'model_epoch' + str(epoch + 1) + '.ckpt')
save_path = saver.save(sess, checkpoint_name)
if (epoch + 1) % args.val_freq == 0:
# Validate the model on the entire validation set
print("{} Start validation".format(datetime.now()))
val_ls = 0.
for _ in tqdm(range(val_batches_per_epoch)):
batch_left, batch_right_pos, batch_right_neg = val_generator.next_batch(
batch_size)
result = sess.run(loss,
feed_dict={
leftx: batch_left,
rightx_pos: batch_right_pos,
rightx_neg: batch_right_neg
})
val_ls += result
val_ls = val_ls / (1. * val_batches_per_epoch)
print('validation loss: {}'.format(val_ls))
s = sess.run(val_merged_summary,
feed_dict={val_loss: np.float32(val_ls)})
writer.add_summary(s, train_batches_per_epoch * (epoch + 1))
# Reset the file pointer of the image data generator
val_generator.reset_pointer()
train_generator.reset_pointer()
train_text_i, train_text_j, train_text_k = datasets[0], datasets[1], datasets[
2]
valid_text_i, valid_text_j, valid_text_k = datasets[3], datasets[4], datasets[
5]
test_text_i, test_text_j, test_text_k = datasets[6], datasets[7], datasets[8]
"""
Main Part of the finetuning Script.
"""
# Create parent path if it doesn't exist
if not os.path.isdir(checkpoint_path):
os.makedirs(checkpoint_path)
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
tr_data = ImageDataGenerator(train_file,
mode='training',
batch_size=batch_size,
shuffle=False)
val_data = ImageDataGenerator(val_file,
mode='inference',
batch_size=batch_size,
shuffle=False)
test_data = ImageDataGenerator(test_file,
mode='inference',
batch_size=batch_size,
shuffle=False)
tra_text_i = TextualDataGenerator(train_text_i,
batch_size_text=int(batch_size / 3))
tra_text_j = TextualDataGenerator(train_text_j,
batch_size_text=int(batch_size / 3))
tra_text_k = TextualDataGenerator(train_text_k,
batch_size_text=int(batch_size / 3))
# Get gradients of all trainable variables
gradients = tf.gradients(loss, var_list)
gradients = list(zip(gradients, var_list))
# Create optimizer and apply gradient descent to the trainable variables
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
train_op = optimizer.apply_gradients(grads_and_vars=gradients)
# Evaluation op: Accuracy of the model
with tf.name_scope("accuracy"):
correct_pred = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
# Initalize the data generator seperately for the training and validation set
train_generator = ImageDataGenerator(train_file, horizontal_flip = True, shuffle = True)
val_generator = ImageDataGenerator(val_file, shuffle = False,testData=False)
val_batches_per_epoch=np.ceil(val_generator.data_size / float(batch_size)).astype(np.int32)
train_batches_per_epoch = np.ceil(train_generator.data_size / float(batch_size)).astype(np.int32)
# Start Tensorflow session
def trainModel():
labelList=np.zeros([val_generator.data_size,17])
acList=[]
with open("valLabels.txt","r") as f:
i=0
for line in f:
labels=map(int,line.strip().split())
for l in labels:
class_validNum = tf.reduce_sum(tf.cast(class_validNum, tf.float32))
cIoU = tf.reduce_sum(cIoU) / class_validNum
classIoU.append(cIoU)
meanIoU = meanIoU + cIoU
valid_classes = tf.greater_equal(tf.reduce_sum(y, axis=[0, 1, 2]),
tf.ones([num_classes], dtype=np.float32))
valid_classes = tf.reduce_sum(tf.cast(valid_classes, tf.float32))
meanIoU = meanIoU / valid_classes
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
if EVAL_TRAIN:
# Initalize the data generator seperately for the training and validation set
train_generator = ImageDataGenerator(train_file,
train_gt_file,
data_augment=False,
shuffle=False)
train_batches_per_epoch = np.floor(train_generator.data_size /
batch_size).astype(np.int32)
if EVAL_VAL:
val_generator = ImageDataGenerator(val_file,
val_gt_file,
data_augment=False,
shuffle=False)
val_batches_per_epoch = np.floor(val_generator.data_size /
batch_size).astype(np.int32)
if EVAL_TEST:
test_generator = ImageDataGenerator(test_file,
test_gt_file,
for i in xrange(10):
labelDictionary[cnt] = chr(48 + i)
cnt += 1
return labelDictionary
# Path to the textfiles for the test set
test_file = 'word_test.txt'
#tf.reset_default_graph()
batch_size = 1
# Place data loading and preprocessing on the cpu
num_classes = 999
with tf.device('/cpu:0'):
test_data = ImageDataGenerator(test_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=True)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(test_data.data.output_types,
test_data.data.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the two different iterators
testing_init_op = iterator.make_initializer(test_data.data)
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3])
y = tf.placeholder(tf.float32, [batch_size, num_classes])
keep_prob = tf.placeholder(tf.float32)
def fine_tuning(self, train_list, test_list, mean, snapshot,
filewriter_path):
# Learning params
learning_rate = 0.0005
num_epochs = 151
batch_size = 64
# Network params
in_img_size = (227, 227) #(height, width)
dropout_rate = 1
num_classes = 2
train_layers = ['fc7', 'fc8']
# How often we want to write the tf.summary data to disk
display_step = 30
x = tf.placeholder(tf.float32,
[batch_size, in_img_size[0], in_img_size[1], 3])
y = tf.placeholder(tf.float32, [None, num_classes])
keep_prob = tf.placeholder(tf.float32)
# Initialize model
model = alexnet(x,
keep_prob,
num_classes,
train_layers,
in_size=in_img_size)
#link variable to model output
score = model.fc8
# List of trainable variables of the layers we want to train
var_list = [
v for v in tf.trainable_variables()
if v.name.split('/')[0] in train_layers
]
# Op for calculating the loss
with tf.name_scope("cross_ent"):
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=score,
labels=y))
# Train op
# Get gradients of all trainable variables
gradients = tf.gradients(loss, var_list)
gradients = list(zip(gradients, var_list))
'''
# Create optimizer and apply gradient descent to the trainable variables
learning_rate = tf.train.exponential_decay(learning_rate,
global_step=tf.Variable(0, trainable=False),
decay_steps=10,decay_rate=0.9)
'''
optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9)
train_op = optimizer.minimize(loss)
# Add gradients to summary
for gradient, var in gradients:
tf.summary.histogram(var.name + '/gradient', gradient)
# Add the variables we train to the summary
for var in var_list:
tf.summary.histogram(var.name, var)
# Add the loss to summary
tf.summary.scalar('cross_entropy', loss)
# Evaluation op: Accuracy of the model
with tf.name_scope("accuracy"):
correct_pred = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Add the accuracy to the summary
tf.summary.scalar('accuracy', accuracy)
# Merge all summaries together
merged_summary = tf.summary.merge_all()
# Initialize the FileWriter
writer = tf.summary.FileWriter(filewriter_path)
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
# Initalize the data generator seperately for the training and validation set
train_generator = ImageDataGenerator(train_list,
horizontal_flip=True,
shuffle=False,
mean=mean,
scale_size=in_img_size,
nb_classes=num_classes)
val_generator = ImageDataGenerator(test_list,
shuffle=False,
mean=mean,
scale_size=in_img_size,
nb_classes=num_classes)
# Get the number of training/validation steps per epoch
train_batches_per_epoch = np.floor(train_generator.data_size /
batch_size).astype(np.int16)
val_batches_per_epoch = np.floor(val_generator.data_size /
batch_size).astype(np.int16)
# Start Tensorflow session
with tf.Session() as sess:
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Add the model graph to TensorBoard
writer.add_graph(sess.graph)
# Load the pretrained weights into the non-trainable layer
model.load_initial_weights(sess)
print("{} Start training...".format(datetime.now()))
print("{} Open Tensorboard at --logdir {}".format(
datetime.now(), filewriter_path))
# Loop over number of epochs
for epoch in range(num_epochs):
print("{} Epoch number: {}/{}".format(datetime.now(),
epoch + 1, num_epochs))
step = 1
while step < train_batches_per_epoch:
# Get a batch of images and labels
batch_xs, batch_ys = train_generator.next_batch(batch_size)
# And run the training op
sess.run(train_op,
feed_dict={
x: batch_xs,
y: batch_ys,
keep_prob: dropout_rate
})
# Generate summary with the current batch of data and write to file
if step % display_step == 0:
s = sess.run(merged_summary,
feed_dict={
x: batch_xs,
y: batch_ys,
keep_prob: 1.
})
writer.add_summary(
s, epoch * train_batches_per_epoch + step)
step += 1
# Validate the model on the entire validation set
print("{} Start validation".format(datetime.now()))
test_acc = 0.
test_count = 0
for _ in range(val_batches_per_epoch):
batch_tx, batch_ty = val_generator.next_batch(batch_size)
acc = sess.run(accuracy,
feed_dict={
x: batch_tx,
y: batch_ty,
keep_prob: 1.
})
test_acc += acc
test_count += 1
test_acc /= test_count
print("{} Validation Accuracy = {:.4f}".format(
datetime.now(), test_acc))
# Reset the file pointer of the image data generator
val_generator.reset_pointer()
train_generator.reset_pointer()
print("{} Saving checkpoint of model...".format(
datetime.now()))
#save checkpoint of the model
if epoch % display_step == 0:
checkpoint_name = os.path.join(
snapshot, 'model_epoch' + str(epoch) + '.ckpt')
save_path = saver.save(sess, checkpoint_name)
print("{} Model checkpoint saved at {}".format(
datetime.now(), checkpoint_name))
def main():
# 初始参数设置
ratio = 0.3 # 分割训练集和验证集的比例
learning_rate = 1e-3
num_epochs = 17 # 代的个数 之前是10
train_batch_size = 5 # 之前是1024
test_batch_size = 1
dropout_rate = 0.5
num_classes = 4 # 类别标签
display_step = 2 # display_step个train_batch_size训练完了就在tensorboard中写入loss和accuracy
# need: display_step <= train_dataset_size / train_batch_size
filewriter_path = "./tmp/tensorboard" # 存储tensorboard文件
checkpoint_path = "./tmp/checkpoints" # 训练好的模型和参数存放目录
ckpt_path = "./resnet_v1_50.ckpt"
image_format = 'jpg'
# ============================================================================
# -----------------生成图片路径和标签的List------------------------------------
file_dir = './dataset'
kitchenWaste = []
label_kitchenWaste = []
Recyclables = []
label_Recyclables = []
harmfulGarbage = []
label_harmfulGarbage = []
otherGarbage = []
label_otherGarbage = []
# step1：获取所有的图片路径名，存放到
# 对应的列表中，同时贴上标签，存放到label列表中。
for file in os.listdir(file_dir + '/kitchenWaste'):
kitchenWaste.append(file_dir + '/kitchenWaste' + '/' + file)
label_kitchenWaste.append(0)
for file in os.listdir(file_dir + '/Recyclables'):
Recyclables.append(file_dir + '/Recyclables' + '/' + file)
label_Recyclables.append(1)
for file in os.listdir(file_dir + '/harmfulGarbage'):
harmfulGarbage.append(file_dir + '/harmfulGarbage' + '/' + file)
label_harmfulGarbage.append(2)
for file in os.listdir(file_dir + '/otherGarbage'):
otherGarbage.append(file_dir + '/otherGarbage' + '/' + file)
label_otherGarbage.append(3)
# step2：对生成的图片路径和标签List做打乱处理
image_list = np.hstack(
(kitchenWaste, Recyclables, harmfulGarbage, otherGarbage))
label_list = np.hstack((label_kitchenWaste, label_Recyclables,
label_harmfulGarbage, label_otherGarbage))
# 利用shuffle打乱顺序
temp = np.array([image_list, label_list])
temp = temp.transpose()
np.random.shuffle(temp)
# 将所有的img和lab转换成list
all_image_list = list(temp[:, 0])
all_label_list = list(temp[:, 1])
# 将所得List分为两部分，一部分用来训练tra，一部分用来测试val
# ratio是测试集的比例
n_sample = len(all_label_list)
n_val = int(math.ceil(n_sample * ratio)) # 测试样本数
n_train = n_sample - n_val # 训练样本数
tra_images = all_image_list[0:n_train]
tra_labels = all_label_list[0:n_train]
tra_labels = [int(float(i)) for i in tra_labels]
val_images = all_image_list[n_train:-1]
val_labels = all_label_list[n_train:-1]
val_labels = [int(float(i)) for i in val_labels]
# get Datasets
# 调用图片生成器，把训练集图片转换成三维数组
train_data = ImageDataGenerator(images=tra_images,
labels=tra_labels,
batch_size=train_batch_size,
num_classes=num_classes,
image_format=image_format,
shuffle=True)
# 调用图片生成器，把测试集图片转换成三维数组
test_data = ImageDataGenerator(images=val_images,
labels=val_labels,
batch_size=test_batch_size,
num_classes=num_classes,
image_format=image_format,
shuffle=False)
# get Iterators
with tf.name_scope('input'):
# 定义迭代器
train_iterator = Iterator.from_structure(train_data.data.output_types,
train_data.data.output_shapes)
training_initalizer = train_iterator.make_initializer(train_data.data)
test_iterator = Iterator.from_structure(test_data.data.output_types,
test_data.data.output_shapes)
testing_initalizer = test_iterator.make_initializer(test_data.data)
# 定义每次迭代的数据
train_next_batch = train_iterator.get_next()
test_next_batch = test_iterator.get_next()
x = tf.placeholder(tf.float32, [None, 227, 227, 3])
y = tf.placeholder(tf.float32, [None, num_classes])
keep_prob = tf.placeholder(tf.float32)
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
fc8, end_points = resnet_v1.resnet_v1_50(x,
num_classes,
is_training=True)
fc8 = tf.squeeze(fc8, [1, 2])
# loss
with tf.name_scope('loss'):
loss_op = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(logits=fc8, labels=y))
# optimizer
with tf.name_scope('optimizer'):
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
train_op = optimizer.minimize(loss_op)
# accuracy
with tf.name_scope("accuracy"):
correct_pred = tf.equal(tf.argmax(fc8, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
init = tf.global_variables_initializer()
# Tensorboard
tf.summary.scalar('loss', loss_op)
tf.summary.scalar('accuracy', accuracy)
merged_summary = tf.summary.merge_all()
writer = tf.summary.FileWriter(filewriter_path)
# load vars
flag = False
if ckpt_path.split('.')[-2] == '/resnet_v1_50':
init = tf.global_variables_initializer()
variables = tf.contrib.framework.get_variables_to_restore()
variables_to_resotre = [
v for v in variables
if v.name.split('/')[1] != 'logits' and v.name.split(
'/')[0] != 'optimizer' and v.name.split('/')[0] != 'save'
and v.name.split('/')[-1] != 'Adam:0' and v.name.split(
'/')[-1] != 'Adam_1:0' and v.name.split('/')[-1] != 'gamma:0'
]
# saver = tf.train.Saver()
saver = tf.train.Saver(variables_to_resotre)
flag = True
else:
saver = tf.train.Saver()
# 定义一代的迭代次数
train_batches_per_epoch = int(
np.floor(train_data.data_size / train_batch_size))
test_batches_per_epoch = int(
np.floor(test_data.data_size / test_batch_size))
# Start training
with tf.Session() as sess:
if flag:
sess.run(init)
saver.restore(sess, ckpt_path)
# Tensorboard
writer.add_graph(sess.graph)
print("{}: Start training...".format(datetime.now()))
print("{}: Open Tensorboard at --logdir {}".format(
datetime.now(), filewriter_path))
for epoch in range(num_epochs):
sess.run(training_initalizer)
print("{}: Epoch number: {} start".format(datetime.now(),
epoch + 1))
# train
for step in range(train_batches_per_epoch):
img_batch, label_batch = sess.run(train_next_batch)
loss, _ = sess.run([loss_op, train_op],
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: dropout_rate
})
if step % display_step == 0:
# loss
print("{}: loss = {}".format(datetime.now(), loss))
# Tensorboard
s = sess.run(merged_summary,
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: 1.
})
writer.add_summary(s,
epoch * train_batches_per_epoch + step)
# accuracy
print("{}: Start validation".format(datetime.now()))
sess.run(testing_initalizer)
test_acc = 0.
test_count = 0
for _ in range(test_batches_per_epoch):
img_batch, label_batch = sess.run(test_next_batch)
acc = sess.run(accuracy,
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: 1.0
})
test_acc += acc
test_count += 1
try:
test_acc /= test_count
except:
print('ZeroDivisionError!')
print("{}: Validation Accuracy = {:.4f}".format(
datetime.now(), test_acc))
# save model
print("{}: Saving checkpoint of model...".format(datetime.now()))
checkpoint_name = os.path.join(
checkpoint_path, 'model_epoch' + str(epoch + 1) + '.ckpt')
save_path = saver.save(sess, checkpoint_name)
# this epoch is over
print("{}: Epoch number: {} end".format(datetime.now(), epoch + 1))
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--data_dir', help = 'Directory to the file lists.')
ap.add_argument('--image_dir', help = 'Directory to the raw images.')
ap.add_argument('--teacher_dir', help = 'Directory to the trained teachers.')
ap.add_argument('--student_dir', help = 'Directory the trained student and events will be saved to.')
ap.add_argument('--model_name', default = 'inception',
help = 'Network architecture to use. One of inception, resnet, densenet, or mobilenet.')
ap.add_argument('--batch_size', type = int, default = 32)
ap.add_argument('--num_epoch', type = int, default = 1)
ap.add_argument('--optimizer', default = 'SGD',
help = 'Optimizer to train the model. One of SGD, adam, or rmsprop.')
ap.add_argument('--initial_lr', type = float, default = 1e-2, help = 'Initial learning rate.')
ap.add_argument('--partition_id', type = int, default = 1,
help = 'Partition index (1-based).')
ap.add_argument('--partition_num', type = int, default = 1,
help = 'Number of partitions.')
ap.add_argument('--alpha', type = float, default = 0.25,
help = """Weight factor for losses between true label and aggregated prediction.
(1 for true label and 0 for aggregated prediction.""")
ap.add_argument('--weight_loss', action = 'store_true')
args = ap.parse_args()
if os.path.isdir(args.student_dir):
rmtree(args.student_dir)
os.mkdir(args.student_dir)
preprocess_input = {
'inception': inception_pre,
'resnet': resnet_pre,
'mobilenet': mobilenet_pre,
'densenet': densenet_pre
}
image_size = {
'inception': 299,
'resnet': 224,
'mobilenet': 224,
'densenet': 224
}
teacher_dir = [x for x in os.listdir(args.teacher_dir) if 'teacher' in x]
num_teachers = len(teacher_dir)
teachers = []
for directory in teacher_dir:
directory = os.path.join(args.teacher_dir, directory)
ckpts = [x for x in os.listdir(directory) if 'hdf5' in x]
ckpts.sort()
teacher, config = load_model(directory, ckpts[-1])
teacher_dict = {}
teacher_dict['model'] = teacher
model_name = config['model_name']
teacher_dict['model_name'] = model_name
teacher_dict['image_size'] = image_size[model_name]
teacher_dict['datagen'] = ImageDataGenerator(preprocessing_function = preprocess_input[model_name])
teachers.append(teacher_dict)
student_config = {'model_name': args.model_name,
'optimizer': args.optimizer,
'initial_lr': args.initial_lr}
with open(os.path.join(args.student_dir, 'model_config.json'), 'w') as f:
json.dump(student_config, f)
with open(os.path.join(args.data_dir, 'label_map.json'), 'r') as f:
label_map = json.load(f)
with open(os.path.join(args.student_dir, 'label_map.json'), 'w') as f:
json.dump(label_map, f)
num_class = len(list(label_map.keys()))
X_train, Y_train = load_filelist(args.data_dir, 'train', args.partition_id, args.partition_num)
X_val, Y_val = load_filelist(args.data_dir, 'val', args.partition_id, args.partition_num)
datagetn_aug = ImageDataGenerator(rotation_range = 10,
width_shift_range = 0.1,
height_shift_range = 0.1,
zoom_range = 0.1,
fill_mode = 'constant',
cval = 0,
horizontal_flip = True,
preprocessing_function = preprocess_input[args.model_name])
datagen = ImageDataGenerator(preprocessing_function = preprocess_input[args.model_name])
Y = np.empty((num_teachers, len(Y_train), num_class))
for i, teacher in enumerate(teachers):
model = teacher['model']
generator = teacher['datagen']
size = teacher['image_size']
Y[i] = model.predict_generator(generator.flow_from_list(x = X_train, directory = args.image_dir,
batch_size = args.batch_size, target_size = (size, size),
shuffle = False))
for teacher in teachers:
del teacher
del teachers
Y = aggregate_teachers(Y)
Y_train = np.concatenate((np.array(Y_train), Y), axis = -1)
loss = weighted_binary_crossentropy(args.alpha)
student = create_model(student_config, image_size[args.model_name], label_map, loss)
class_weight = None
if args.weight_loss:
class_weight = calc_weights(Y_train)
tensorbard = TensorBoard(args.student_dir)
reducelr = ReduceLROnPlateau(monitor = 'loss', factor = 0.9, patience = 5, mode = 'min')
earlystop = EarlyStopping(monitor = 'val_mauc', min_delta = 1e-3,
patience = max(5, args.num_epoch / 10), mode = 'max')
ckpt = ModelCheckpoint(os.path.join(args.student_dir, 'weights.{epoch:03d}-{val_mauc:.2f}.hdf5'),
monitor = 'val_mauc', save_best_only = True, mode = 'max')
size = image_size[args.model_name]
student.fit_generator(datagetn_aug.flow_from_list(x = X_train, y = Y_train, directory = args.image_dir,
batch_size = args.batch_size, target_size = (size, size)), epochs = args.num_epoch,
steps_per_epoch = math.ceil(len(X_train) / float(args.batch_size)),
validation_data = datagen.flow_from_list(x=X_val, y=Y_val, directory=args.image_dir,
batch_size = args.batch_size, target_size=(size, size)),
validation_steps = math.ceil(len(X_val) / float(args.batch_size)),
class_weight = class_weight, verbose = 2,
callbacks = [tensorbard, reducelr, earlystop, ckpt])
model = Model(inputs=[INPUT1_1, INPUT2_1], outputs=FC8)
return model
if __name__ == '__main__':
# import training data
argv = sys.argv
if len(argv) < 1:
raise Exception('please enter directory of dataset respectively')
tf = argv[1]
if tf[-1] != '/':
tf = tf + '/'
dg = ImageDataGenerator(tf, shuffle=True)
# build model
model = buildModel()
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=Adam(lr=1e-4),
metrics=['accuracy'])
ckpt = ModelCheckpoint('model4cost',
monitor='val_acc',
save_best_only=True,
save_weights_only=True,
verbose=1)
cb = [ckpt]
def run_net(args, test):
# Path for tf.summary.FileWriter and to store model checkpoints
print(args)
r = round(np.random.randn(), 4)
# Create parent path if it doesn't exist
if not os.path.isdir(args['checkpoint_path']):
os.makedirs(args['checkpoint_path'])
# data
tr_data = ImageDataGenerator(args['train_file'],
mode='training',
batch_size=args['batch_size'],
num_classes=args['num_classes'],
shuffle=True)
val_data = ImageDataGenerator(args['val_file'],
mode='inference',
batch_size=args['batch_size'],
num_classes=args['num_classes'],
shuffle=False)
if args['v'] != 'None':
v_data = ImageDataGeneratorV(args['v_file'],
mode='inference',
batch_size=args['v_batch_size'])
iterator_v1 = Iterator.from_structure(v_data.data.output_types,
v_data.data.output_shapes)
next_batch_v1 = iterator_v1.get_next()
v_init_op = iterator_v1.make_initializer(v_data.data)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the two different iterators
training_init_op = iterator.make_initializer(tr_data.data)
validation_init_op = iterator.make_initializer(val_data.data)
# TF placeholder for graph input and output
if args['v'] != 'None':
img1 = tf.placeholder(tf.float32, [args['v_batch_size'], 227, 227, 3])
img2 = tf.placeholder(tf.float32, [args['v_batch_size'], 227, 227, 3])
RSA = tf.placeholder(tf.float32, [args['v_batch_size']])
lam = tf.placeholder(tf.float32)
x = tf.placeholder(tf.float32, [args['batch_size'], 227, 227, 3])
y = tf.placeholder(tf.float32, [args['batch_size'], args['num_classes']])
keep_prob = tf.placeholder(tf.float32)
# Initialize model
model = CORNetZV(x, keep_prob, args['num_classes'], args['train_layers'])
# Link variable to model output
if (args['v'] == 'V1'):
pool_tf1 = model.forward_V1(img1)
pool_tf2 = model.forward_V1(img2)
elif (args['v'] == 'V4'):
pool_tf1 = model.forward_V4(img1)
pool_tf2 = model.forward_V4(img2)
elif (args['v'] == 'IT'):
pool_tf1 = model.forward_IT(img1)
pool_tf2 = model.forward_IT(img2)
score = model.forward()
# Op for calculating the loss
with tf.name_scope("cross_ent"):
cif_cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=score, labels=y))
if args['v'] != 'None':
v_cost = compute_cost_V(pool_tf1, pool_tf2, RSA)
total_cost = get_total_cost(v_cost, cif_cost, lam)
# Train op
# Create optimizer and apply gradient descent to the trainable variables
optimizer = tf.train.GradientDescentOptimizer(args['learning_rate'])
train_op = optimizer.minimize(loss=cif_cost,
global_step=tf.train.get_global_step())
if args['v'] != 'None':
train_op_2 = optimizer.minimize(loss=total_cost,
global_step=tf.train.get_global_step())
# Evaluation op: Accuracy of the model
model_pred = tf.argmax(score, 1)
act_pred = tf.argmax(y, 1)
correct_pred = tf.equal(model_pred, act_pred)
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
# Get the number of training/validation steps per epoch
train_batches_per_epoch = int(
np.floor(tr_data.data_size / args['batch_size']))
val_batches_per_epoch = int(
np.floor(val_data.data_size / args['batch_size']))
if (test):
print('Running test with 5 batches per epoch')
train_batches_per_epoch = 10
val_batches_per_epoch = 10
#config = tf.ConfigProto(log_device_placement=True)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
if (args['v'] != 'None'):
f_info = 'CORNetZ_' + str(args['v']) + 'initial_ratio=' + str(
args['v_ratio']) + 'num_epochs=' + str(
args['num_epochs']) + 'n_e_c=' + str(
args['n_e_c']) + 'n_e_v=' + str(
args['n_e_v']) + '_' + str(r)
df = pd.DataFrame(np.zeros((args['num_epochs'], 9)),
columns=[
'train_fine_acc', 'train_coarse_acc',
'test_fine_acc', 'test_coarse_acc',
'cif_cost_train', 'cif_cost_test', 'v_cost',
'ratio', 'time'
])
else:
f_info = 'CORNetZ_' + str(args['num_epochs']) + '_' + str(r)
df = pd.DataFrame(np.zeros((args['num_epochs'], 7)),
columns=[
'train_fine_acc', 'train_coarse_acc',
'test_fine_acc', 'test_coarse_acc',
'cif_cost_train', 'cif_cost_test', 'time'
])
results_f = args['results_path'] + f_info + '.csv'
t0 = time.time()
if args['v'] != 'None':
print('Running ' + str(args['v']) + ' ratio = ' + str(args['v_ratio']))
else:
print('Running CORNetZ')
#f.write(str(ratio) + '\n')
with tf.Session(config=config) as sess:
# Initialize all variables
sess.run(tf.global_variables_initializer())
#saver.restore(sess, 'log/checkpoints/CORNETZmodel_epoch5.ckpt')
print("{} Start training...".format(datetime.now()))
lam_cur = 0
# Loop over number of epochs
for epoch in range(args['num_epochs']):
print("{} Epoch number: {}".format(datetime.now(), epoch + 1))
# Initialize iterator with the training dataset
sess.run(training_init_op)
if (args['v'] != 'None'):
sess.run(v_init_op)
train_fine_acc = 0.
train_count = 0
train_coarse_acc = 0
cif_cost_train = 0
v_cost_ = 0
for step in range(train_batches_per_epoch):
# get next batch of data
t = which_train(step, args, epoch)
img_batch, label_batch = sess.run(next_batch)
cif_cost_cur = sess.run(cif_cost,
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: 1.
})
cif_cost_train += cif_cost_cur
acc, model_preds, act_preds = sess.run(
[accuracy, model_pred, act_pred],
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: 1.
})
c_acc = get_coarse_accuracy(model_preds, act_preds)
train_coarse_acc += c_acc
train_fine_acc += acc
train_count += 1
if (args['v'] != 'None'):
# get v1 batches
img1_batch, img2_batch, RSAs_batch = sess.run(
next_batch_v1)
## calculate costs for lambda vlaue
v_cost_cur = sess.run(v_cost,
feed_dict={
img1: img1_batch,
img2: img2_batch,
RSA: RSAs_batch
})
v_cost_ += v_cost_cur
if (t == 'CE'):
sess.run(train_op,
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: args['dropout_rate']
})
else:
if (lam_cur == 0):
lam_cur = (float(args['v_ratio']) *
cif_cost_cur) / (v_cost_cur)
## run v1 training op on total cost
sess.run(train_op_2,
feed_dict={
img1: img1_batch,
img2: img2_batch,
RSA: RSAs_batch,
lam: lam_cur,
x: img_batch,
y: label_batch,
keep_prob: args['dropout_rate']
})
train_fine_acc /= train_count
train_coarse_acc /= train_count
cif_cost_train /= train_count
if (v_cost_ != 0):
v_cost_ /= train_count
# Validate the model on the entire validation set
print("{} Start validation for epoch= " + str(epoch + 1) + ' ' +
format(datetime.now()))
sess.run(validation_init_op)
test_fine_acc = 0.
test_count = 0
test_coarse_acc = 0
cif_cost_test = 0
for _ in range(val_batches_per_epoch):
img_batch, label_batch = sess.run(next_batch)
acc, model_preds, act_preds = sess.run(
[accuracy, model_pred, act_pred],
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: 1.
})
cif_cost_cur = sess.run(cif_cost,
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: 1.
})
cif_cost_test += cif_cost_cur
c_acc = get_coarse_accuracy(model_preds, act_preds)
test_coarse_acc += c_acc
test_fine_acc += acc
test_count += 1
test_fine_acc /= test_count
test_coarse_acc /= test_count
cif_cost_test /= test_count
ti = time.time()
time_run = (ti - t0) / 60
df['train_fine_acc'].ix[epoch] = train_fine_acc
df['train_coarse_acc'].ix[epoch] = train_coarse_acc
df['test_fine_acc'].ix[epoch] = test_fine_acc
df['test_coarse_acc'].ix[epoch] = test_coarse_acc
df['cif_cost_train'].ix[epoch] = cif_cost_train
df['cif_cost_test'].ix[epoch] = cif_cost_test
if args['v'] != 'None':
cur_ratio = (lam_cur * v_cost_cur) / cif_cost_cur
df['v_cost'].ix[epoch] = v_cost_
df['ratio'].ix[epoch] = cur_ratio
df['time'].ix[epoch] = time_run
df.to_csv(results_f)
print("Time to run epoch " + str(epoch + 1) + ' : ' +
str(round(time_run, 2)) + ' minutes')
print("{} Validation Accuracy = {:.4f}".format(
datetime.now(), test_fine_acc))
print("{} Validation Coarse Accuracy = {:.4f}".format(
datetime.now(), test_coarse_acc))
print("{} Training Accuracy = {:.4f}".format(
datetime.now(), train_fine_acc))
print("{} Training Coarse Accuracy = {:.4f}".format(
datetime.now(), train_coarse_acc))
print("{} Validation Cost= {:.4f}".format(datetime.now(),
cif_cost_test))
print("{} Training Cost = {:.4f}".format(datetime.now(),
cif_cost_train))
# save checkpoint of the model
checkpoint_name = os.path.join(
args['checkpoint_path'],
'weights' + f_info + '_epoch' + str(epoch + 1) + '.ckpt')
#if not args['test']:
save_path = saver.save(sess, checkpoint_name)
print("{} Model checkpoint saved at {}".format(
datetime.now(), checkpoint_name))
tf.summary.scalar('cross_entropy', loss)
with tf.name_scope("accuracy"):
labels = tf.argmax(y, 1)
topFiver = tf.nn.in_top_k(score, labels, top_N)
accuracy = tf.reduce_mean(tf.cast(topFiver, tf.float32))
prediction = tf.argmax(score, 1)
tf.summary.scalar('accuracy', accuracy)
merged_summary = tf.summary.merge_all()
writer = tf.summary.FileWriter(filewriter_path)
saver = tf.train.Saver()
train_generator = ImageDataGenerator(train_image,
train_label,
horizontal_flip=False,
shuffle=False)
val_generator = ImageDataGenerator(val_image, val_truth, shuffle=False)
train_batches_per_epoch = np.floor(train_generator.data_size /
batch_size).astype(np.int16)
val_batches_per_epoch = np.floor(val_generator.data_size / batch_size).astype(
np.int16)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
writer.add_graph(sess.graph)
if restore_checkpoint == '':
model.load_initial_weights(sess)
else:
# Path for tf.summary.FileWriter and to store model checkpoints
filewriter_path = "../check/tensorboard"
checkpoint_path = "../check/checkpoints"
"""
Main Part of the finetuning Script.
"""
# Create parent path if it doesn't exist
if not os.path.isdir(checkpoint_path):
os.mkdir(checkpoint_path)
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
tr_data = ImageDataGenerator(train_file,
mode='training',
batch_size=batch_size,
num_classes=num_classes,
shuffle=True)
val_data = ImageDataGenerator(val_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
# create an reinitializable iterator given the dataset structure
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3], name="x")
y = tf.placeholder(tf.float32, [batch_size, num_classes])
keep_prob = tf.placeholder(tf.float32)
# Initialize model
def training(self,
source_file,
target_file,
init_lr=1e-3,
training_epochs=1000,
batch_size=100,
batch_test=100,
version=0):
sess = tf.Session()
"""
----------------
MODEL DEFINITION
----------------
"""
one_hot_labels = tf.placeholder(tf.float32, [None, self.n_class])
lr = tf.placeholder(tf.float32)
domain_label = tf.concat(
[tf.ones([self.n_source, 1]),
tf.zeros([self.n_target, 1])],
axis=0)
non_adversarial = [
v for v in tf.trainable_variables()
if not ('adversarial' in v.name)
]
adversarial_list = [
v for v in tf.trainable_variables() if 'adversarial' in v.name
]
scratch_list = [
v for v in tf.trainable_variables() if 'adapt' in v.name
]
finetuning_list = [
v for v in tf.trainable_variables()
if not (('adapt' in v.name) or ('adversarial' in v.name))
]
weight_decay_not_ad = 0.0005 / 2 * tf.add_n([
tf.nn.l2_loss(v) for v in non_adversarial
if ('weight' in v.name) or ('kernel' in v.name)
])
weight_decay_ad = 0.0005 / 2 * tf.add_n([
tf.nn.l2_loss(v) for v in adversarial_list
if ('weight' in v.name) or ('kernel' in v.name)
])
#prediction_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits = self.source_classify, labels = one_hot_labels))
prediction_loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=self.source_classify,
labels=tf.argmax(one_hot_labels, 1)))
discriminator_loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=self.discriminator,
labels=domain_label))
adver_loss = discriminator_loss + weight_decay_ad
"""
#block adaptation version 2
skip_source_w = [v for v in tf.trainable_variables() if ('adapt/skip_connection_source/weights' in v.name)][0]
skip_source_b = [v for v in tf.trainable_variables() if ('adapt/skip_connection_source/biases' in v.name)][0]
skip_target_w = [v for v in tf.trainable_variables() if ('adapt/skip_connection_target/weights' in v.name)][0]
skip_target_b = [v for v in tf.trainable_variables() if ('adapt/skip_connection_target/biases' in v.name)][0]
print(skip_source_w.name, skip_source_b.name, skip_target_w.name, skip_target_b.name)
difference_loss = (tf.reduce_sum(tf.square(skip_source_w - skip_target_w)) + tf.reduce_sum(tf.square(skip_source_b - skip_target_b)))
"""
#block adaptation version 3
difference_loss = tf.add_n(
[tf.nn.l2_loss(v) for v in scratch_list if 'projection' in v.name])
print([v for v in scratch_list if 'projection' in v.name])
loss = prediction_loss + weight_decay_not_ad - self.lamda * discriminator_loss + difference_loss
"""
--------------------
OPTIMIZER DEFINITION
--------------------
"""
print(adversarial_list)
print(scratch_list)
print(finetuning_list)
min_var_list = scratch_list + finetuning_list
adv_var_list = adversarial_list
min_gradients = tf.gradients(loss, min_var_list)
adv_gradients = tf.gradients(adver_loss, adv_var_list)
#min_gradients, _ = tf.clip_by_global_norm(min_gradients, 5.0)
#adv_gradients, _ = tf.clip_by_global_norm(adv_gradients, 5.0)
min_gradients = list(zip(min_gradients, min_var_list))
adv_gradients = list(zip(adv_gradients, adv_var_list))
optimizer1 = tf.train.MomentumOptimizer(lr * 10, momentum=0.9)
optimizer2 = tf.train.MomentumOptimizer(lr, momentum=0.9)
optimizer3 = tf.train.MomentumOptimizer(lr * 10, momentum=0.9)
train_op1 = optimizer1.apply_gradients(
grads_and_vars=min_gradients[:len(scratch_list)])
train_op2 = optimizer2.apply_gradients(
grads_and_vars=min_gradients[len(scratch_list):])
train_op3 = optimizer3.apply_gradients(grads_and_vars=adv_gradients)
train_op = tf.group(train_op1, train_op2, train_op3)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
optimize = tf.group([train_op, update_ops])
"""
-----------------------------------------------
DATA PREPARATION (REUSE FROM THE ALEXNET MODEL)
-----------------------------------------------
"""
size = len(open(source_file).readlines())
valmode = True
with tf.device('/cpu:0'):
train_source_data = ImageDataGenerator(source_file,
mode='training',
batch_size=batch_size,
num_classes=self.n_class,
shuffle=True,
size=size)
train_target_data = ImageDataGenerator(target_file,
mode='training',
batch_size=batch_size,
num_classes=self.n_class,
shuffle=True,
size=size)
val_data = []
for t in range(9):
i = t // 3
j = t % 3
val_data.append(
ImageDataGenerator(target_file,
mode='inference',
batch_size=batch_test,
num_classes=self.n_class,
shuffle=False,
fulval=valmode,
pos_x=i,
pos_y=j))
iterator_source_train = Iterator.from_structure(
train_source_data.data.output_types,
train_source_data.data.output_shapes)
iterator_target_train = Iterator.from_structure(
train_target_data.data.output_types,
train_target_data.data.output_shapes)
iterator_val = []
for t in range(9):
iterator_val.append(
Iterator.from_structure(val_data[t].data.output_types,
val_data[t].data.output_shapes))
next_batch_sr = iterator_source_train.get_next()
next_batch_tr = iterator_target_train.get_next()
next_batch_val = []
for t in range(9):
next_batch_val.append(iterator_val[t].get_next())
train_batches_per_epoch = int(
np.floor(train_source_data.data_size / batch_size))
val_batches_per_epoch = int(np.ceil(val_data[0].data_size /
batch_size))
training_source_init_op = iterator_source_train.make_initializer(
train_source_data.data)
training_target_init_op = iterator_target_train.make_initializer(
train_target_data.data)
validation_init_op = []
for t in range(9):
validation_init_op.append(iterator_val[t].make_initializer(
val_data[t].data))
"""
---------------------------
VALIDATION SCORE DEFINITION
---------------------------
"""
prob = tf.placeholder(tf.float32, [None, self.n_class])
correct_pred = tf.equal(tf.argmax(prob, 1),
tf.argmax(one_hot_labels, 1))
accuracy = tf.reduce_sum(tf.cast(correct_pred, tf.float32))
accunum = tf.shape(one_hot_labels)[0]
"""
------------------------------
ACTUAL TRAINING AND EVALUATION
------------------------------
"""
init = tf.global_variables_initializer()
sess.run(init)
self.load_initial_weight(sess)
print(config)
print(init_lr)
print(train_batches_per_epoch)
print(val_batches_per_epoch)
print(source_file, target_file)
best = 0
#print(sess.run([v for v in tf.trainable_variables() if 'conv1/weights' in v.name]))
for epochs in range(training_epochs):
sess.run(training_source_init_op)
sess.run(training_target_init_op)
times_pass = 0
#print(sess.run([tf.get_default_graph().get_tensor_by_name('conv1/weights:0')]))
L_pred = 0
L_total = 0
L_MMD = 0
L_diff = 0
for times in range(train_batches_per_epoch):
if times_pass == val_batches_per_epoch - 1:
sess.run(training_target_init_op)
times_pass = 0
else:
times_pass += 1
p = 0.5 * epochs / training_epochs # + times/train_batches_per_epoch/training_epochs
learning_rate = init_lr / (1. + 10.0 * p)**0.75
b = 1.0 * epochs / training_epochs
#print(learning_rate)
alpha = 2.0 / (1.0 +
np.exp(-10.0 * epochs / training_epochs)) - 1.0
#(1 - np.exp(-10.0 * epochs / training_epochs)) / (1 + np.exp(-10.0 * epochs / training_epochs))
#begin = time.time()
x1, y1 = sess.run(next_batch_sr)
x2, y2 = sess.run(next_batch_tr)
#end = time.time()
#print("Loading: ", end - begin)
#print(times, np.mean(x1), np.mean(x2))
#begin = time.time()
_, total_loss, pred_loss, M_loss, D_loss = sess.run(
[
optimize, loss, prediction_loss, adver_loss,
difference_loss
],
feed_dict={
self.source: x1,
self.target: x2,
one_hot_labels: y1,
lr: learning_rate,
self.keep_prob: self.KEEP_PROB_TRAINING,
self.lamda: alpha,
self.beta: b
})
#end = time.time()
#print("Running: ",end - begin)
L_pred = L_pred + pred_loss
L_total = L_total + total_loss
L_MMD = L_MMD + M_loss
L_diff = L_diff + D_loss
if not (epochs % 10 == 9):
continue
#print(sess.run([v for v in tf.trainable_variables() if 'conv1/weights' in v.name]))
print(L_pred * 1.0 / train_batches_per_epoch)
print(L_MMD * 1.0 / train_batches_per_epoch)
print(L_diff * 1.0 / train_batches_per_epoch)
print("TIMES: ", epochs, " LOSS: ",
L_total * 1.0 / train_batches_per_epoch)
test_acc = 0.
num = 0.
for t in range(9):
sess.run(validation_init_op[t])
for _ in range(val_batches_per_epoch):
val_input = 0
for t in range(9):
if True: #target_file == './testtxt/amazon.txt':
if t != 4:
continue
img_batch, label_batch = sess.run(next_batch_val[t])
#img_batch = np.reshape(img_batch, [-1 ,height, width, 3])
#print(img_batch.size)
logits = sess.run(
[tf.nn.softmax(self.logits, axis=-1)],
feed_dict={
self.source:
np.zeros(shape=(0, height, width, 3),
dtype=np.float32),
self.target:
img_batch,
self.keep_prob:
self.KEEP_PROB_VALIDATION,
self.beta:
1.0
})
val_input += logits[0]
acc, num_ = sess.run([accuracy, accunum],
feed_dict={
prob: val_input,
one_hot_labels: label_batch
})
test_acc += acc
num += num_
print("TIMES: ", epochs, " VALIDATION: ", test_acc / num)
best = max(best, test_acc / num)
print(best)
# test loss and error_bias
with tf.name_scope("testing_metric"):
testing_summary = []
test_loss = tf.placeholder(tf.float32, [])
# Add test loss and error_bias to summary
testing_summary.append(tf.summary.scalar('test_l2_loss', test_loss))
testing_merged_summary = tf.summary.merge(testing_summary)
# Initialize the FileWriter
writer = tf.summary.FileWriter(filewriter_path)
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
# Initalize the data generator seperately for the training and validation set
train_generator = ImageDataGenerator(train_file, shuffle=True)
val_generator = ImageDataGenerator(val_file, shuffle=False)
# Get the number of training/validation steps per epoch
train_batches_per_epoch = np.floor(train_generator.data_size /
batch_size).astype(np.int32)
val_batches_per_epoch = np.floor(val_generator.data_size / batch_size).astype(
np.int32)
# Start Tensorflow session
with tf.Session(config=tf.ConfigProto(log_device_placement=False, \
allow_soft_placement=True)) as sess:
# Initialize all variables
sess.run(tf.global_variables_initializer())
if FROM_SCRATCH:
def dataInputGen():
# File Path desc
train_file = "./TrainingFile_detail.txt"
Validate_file = "./ValidationFile_detail.txt"
class_file = "./Classmap.txt"
checkpoint_path = "./ckp/"
# Network params
learning_rate = 0.00001
dropout_rate = 0.5
num_classes = 1000
batch_size = 1
num_epochs = 100
display_step = 20 #check acc per epoach
train_layers = ['fc8', 'fc7', 'fc6']
#read all image path config
train_img_paths,train_labels = read_train_detail(train_file)
validate_img_paths,validate_labels = read_train_detail(Validate_file)
print("Total Dataset {}".format(int(len(train_labels)+len(validate_labels))))
print("Split to Training {} and to Validation {}".format(len(train_labels),len(validate_labels)))
with tf.device('/cpu:0'):
tr_data = ImageDataGenerator(mode='training',
batch_size=batch_size,
num_classes=num_classes,
class_file=class_file,
shuffle=False,
img_paths=train_img_paths,
labels=train_labels)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the two different iterators
training_init_op = iterator.make_initializer(tr_data.data)
validation_init_op = iterator.make_initializer(val_data.data)
# Get the number of training/validation steps per epoch
train_batches_per_epoch = int(np.floor(tr_data.data_size/batch_size))
val_batches_per_epoch = int(np.floor(val_data.data_size/batch_size))
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [None, 227, 227, 3])
y = tf.placeholder(tf.float32, [None, num_classes])
keep_prob = tf.placeholder(tf.float32)
# Initialize model
model = AlexNet(x, keep_prob, num_classes, train_layers,weights_path="./weights/bvlc_alexnet.npy")
# Link variable to model output
score = model.fc8
# List of trainable variables of the layers we want to train
var_list = [v for v in tf.trainable_variables() if v.name.split('/')[0] in train_layers]
# Op for calculating the loss
with tf.name_scope("cross_ent"):
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=score,labels=y))
# Train op
with tf.name_scope("train"):
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss)
# Evaluation op: Accuracy of the model
with tf.name_scope("accuracy"):
correct_pred = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32),name="ACC")
# Add the loss to summary
tf.summary.scalar('cross_entropy', loss)
tf.summary.scalar('Acc', accuracy)
# Merge all summaries together
merged_summary = tf.summary.merge_all()
# Initialize the FileWriter
writer = tf.summary.FileWriter("./Graph")
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
with tf.Session() as sess:
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Add the model graph to TensorBoard
writer.add_graph(sess.graph)
# Load the pretrained weights into the non-trainable layer
print("Weight Loading...")
model.load_initial_weights(sess)
print("{} Start training...".format(datetime.now()))
best_acc = 0
# Loop over number of epochs
for epoch in range(num_epochs):
print("{} Epoch number: {}".format(datetime.now(), epoch+1))
# Initialize iterator with the training dataset
sess.run(training_init_op)
for step in tqdm(range(train_batches_per_epoch)):
# get next batch of data
img_batch, label_batch = sess.run(next_batch)
# And run the training op
sess.run(train_op, feed_dict={x: img_batch,
y: label_batch,
keep_prob: dropout_rate})
# Generate summary with the current batch of data and write to file
if step % display_step == 0:
s,acc_data = sess.run([merged_summary,accuracy], feed_dict={x: img_batch,
y: label_batch,
keep_prob: 1.})
writer.add_summary(s, epoch*train_batches_per_epoch + step)
#print("Accuracy at steps {} is {:f}".format((epoch*train_batches_per_epoch + step),acc_data))
# Validate the model on the entire validation set
print("{} Start validation".format(datetime.now()))
sess.run(validation_init_op)
test_acc = 0.
test_count = 0
for _ in tqdm(range(val_batches_per_epoch)):
img_batch, label_batch = sess.run(next_batch)
acc = sess.run(accuracy, feed_dict={x: img_batch,
y: label_batch,
keep_prob: 1.})
test_acc += acc
test_count += 1
test_acc /= test_count
print("{} Validation Accuracy = {:.4f}".format(datetime.now(), test_acc))
print("Checking improving Accuracy...")
if test_acc > best_acc:
print("Accuracy improve from {:.4f} to {:.4f}".format(best_acc,test_acc))
print("Saving best weights ...")
save_path = saver.save(sess, "./BestWeight/weight")
best_acc = test_acc
else:
print("Accuracy not improve")
print("{} Saving checkpoint of model...".format(datetime.now()))
# save checkpoint of the model
checkpoint_name = os.path.join(checkpoint_path,
'model_epoch'+str(epoch+1)+'.ckpt')
save_path = saver.save(sess, checkpoint_name)
print("{} Model checkpoint saved at {}".format(datetime.now(),
checkpoint_name))
filewriter_path = "./tmp/DeViSE/tensorboard"
checkpoint_path = "./tmp/DeViSE/checkpoints"
checkpoint_path0 = "./tmp/finetune_alexnet/checkpoints"
"""
Main Part of the finetuning Script.
"""
# Create parent path if it doesn't exist
if not os.path.isdir(checkpoint_path):
os.mkdir(checkpoint_path)
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
tr_data = ImageDataGenerator(train_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=True)
val_data = ImageDataGenerator(val_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
tst_data = ImageDataGenerator(test_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=True)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
# Add the accuracy to the summary
tf.summary.scalar('accuracy', accuracy)
# Merge all summaries together
merged_summary = tf.summary.merge_all()
# Initialize the FileWriter
writer = tf.summary.FileWriter(filewriter_path)
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
# Initalize the data generator seperately for the training and validation set
train_generator = ImageDataGenerator(train_file,
horizontal_flip=True,
shuffle=True)
#print(train_generator.labels)
val_generator = ImageDataGenerator(val_file, shuffle=False)
# Get the number of training/validation steps per epoch
train_batches_per_epoch = np.floor(train_generator.data_size /
batch_size).astype(np.int16)
val_batches_per_epoch = np.floor(val_generator.data_size / batch_size).astype(
np.int16)
# Start Tensorflow session
with tf.Session() as sess:
#sess.run(tf.global_variables_initializer())
saver.restore(
def fine_tuning_training(self,
source_file,
target_file,
init_lr=1e-3,
training_epochs=200,
batch_size=100):
sess = tf.Session()
"""
----------------
MODEL DEFINITION
----------------
"""
one_hot_labels = tf.placeholder(tf.float32, [None, self.n_class])
lr = tf.placeholder(tf.float32)
if self.mode == 'fine_tuning' or self.mode == 'vgg_fine_tuning':
classifier = self.logits
prediction_loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(
logits=classifier, labels=one_hot_labels))
loss = prediction_loss + 0.0005 / 2 * tf.add_n([
tf.nn.l2_loss(v) for v in tf.trainable_variables() if
('weight' in v.name) or ('kernel' in v.name)
])
elif self.mode == 'adaptation_block':
source_logits, target_logits = tf.split(
self.logits, [self.n_source, self.n_target])
prediction_loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(
logits=source_logits, labels=one_hot_labels))
MMD_loss = self.DAN_MMD(self.source_feature, self.target_feature)
loss = prediction_loss + 0.0005 / 2 * tf.add_n([
tf.nn.l2_loss(v) for v in tf.trainable_variables() if
('weight' in v.name) or ('kernel' in v.name)
]) + self.lamda * MMD_loss
elif 'JAN' in self.mode:
source_logits, target_logits = tf.split(
self.logits, [self.n_source, self.n_target])
prediction_loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(
logits=source_logits, labels=one_hot_labels))
MMD_loss = self.JAN_MMD(
[self.source_feature,
tf.nn.softmax(source_logits, axis=-1)],
[self.target_feature,
tf.nn.softmax(target_logits, axis=-1)])
#MMD_loss = 2 * self.MJAN_MMD(self.source_feature, self.target_feature, tf.nn.softmax(source_logits, axis = -1), tf.nn.softmax(target_logits, axis = -1))
"""
skip_source_w = [v for v in tf.trainable_variables() if ('adapt/skip_connection_source/weights' in v.name)][0]
skip_source_b = [v for v in tf.trainable_variables() if ('adapt/skip_connection_source/biases' in v.name)][0]
skip_target_w = [v for v in tf.trainable_variables() if ('adapt/skip_connection_target/weights' in v.name)][0]
skip_target_b = [v for v in tf.trainable_variables() if ('adapt/skip_connection_target/biases' in v.name)][0]
difference_loss = (tf.reduce_sum(tf.square(skip_source_w - skip_target_w)) + tf.reduce_sum(tf.square(skip_source_b - skip_target_b)))
"""
difference_loss = tf.add_n([
tf.nn.l2_loss(v) for v in tf.trainable_variables()
if 'projection' in v.name
])
loss = prediction_loss + 0.0005 / 2 * tf.add_n([
tf.nn.l2_loss(v) for v in tf.trainable_variables() if
('weight' in v.name) or ('kernel' in v.name)
]) + self.lamda * MMD_loss + difference_loss
print("check:", [
v for v in tf.trainable_variables()
if ('weight' in v.name) or ('kernel' in v.name)
])
"""
--------------------
OPTIMIZER DEFINITION
--------------------
"""
if self.mode == 'fine_tuning':
scratch_list = [
v for v in tf.trainable_variables() if 'logits' in v.name
]
finetuning_list = [
v for v in tf.trainable_variables() if not ('fc8' in v.name)
]
elif self.mode == 'vgg_fine_tuning':
scratch_list = [
v for v in tf.trainable_variables() if 'fc8' in v.name
]
finetuning_list = [
v for v in tf.trainable_variables() if not ('fc8' in v.name)
]
elif 'adaptation_block' in self.mode:
scratch_list = [
v for v in tf.trainable_variables() if 'adapt' in v.name
]
finetuning_list = [
v for v in tf.trainable_variables() if not ('adapt' in v.name)
]
elif self.mode == 'JAN':
scratch_list = [
v for v in tf.trainable_variables() if 'adapt' in v.name
]
finetuning_list = [
v for v in tf.trainable_variables() if not ('adapt' in v.name)
]
print(scratch_list)
print(finetuning_list)
var_list = scratch_list + finetuning_list
gradients = tf.gradients(loss, var_list)
gradients = list(zip(gradients, var_list))
optimizer1 = tf.train.MomentumOptimizer(lr * 10, momentum=0.9)
optimizer2 = tf.train.MomentumOptimizer(lr, momentum=0.9)
train_op1 = optimizer1.apply_gradients(
grads_and_vars=gradients[:len(scratch_list)])
train_op2 = optimizer2.apply_gradients(
grads_and_vars=gradients[len(scratch_list):])
train_op = tf.group(train_op1, train_op2)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
optimize = tf.group([train_op, update_ops])
"""
-----------------------------------------------
DATA PREPARATION (REUSE FROM THE ALEXNET MODEL)
-----------------------------------------------
"""
size = len(open(source_file).readlines())
valmode = True
with tf.device('/cpu:0'):
train_source_data = ImageDataGenerator(source_file,
mode='training',
batch_size=batch_size,
num_classes=self.n_class,
shuffle=True,
size=size)
train_target_data = ImageDataGenerator(target_file,
mode='training',
batch_size=batch_size,
num_classes=self.n_class,
shuffle=True,
size=size)
val_data = ImageDataGenerator(target_file,
mode='inference',
batch_size=batch_size,
num_classes=self.n_class,
shuffle=False,
fulval=valmode)
iterator_source_train = Iterator.from_structure(
train_source_data.data.output_types,
train_source_data.data.output_shapes)
iterator_target_train = Iterator.from_structure(
train_target_data.data.output_types,
train_target_data.data.output_shapes)
iterator_val = Iterator.from_structure(val_data.data.output_types,
val_data.data.output_shapes)
next_batch_sr = iterator_source_train.get_next()
next_batch_tr = iterator_target_train.get_next()
next_batch_val = iterator_val.get_next()
train_batches_per_epoch = int(
np.floor(train_source_data.data_size / batch_size))
val_batches_per_epoch = int(np.ceil(val_data.data_size / batch_size))
training_source_init_op = iterator_source_train.make_initializer(
train_source_data.data)
training_target_init_op = iterator_target_train.make_initializer(
train_target_data.data)
validation_init_op = iterator_val.make_initializer(val_data.data)
"""
---------------------------
VALIDATION SCORE DEFINITION
---------------------------
"""
print(self.logits)
correct_pred = tf.equal(tf.argmax(self.logits, 1),
tf.argmax(one_hot_labels, 1))
accuracy = tf.reduce_sum(tf.cast(correct_pred, tf.float32))
accunum = tf.shape(one_hot_labels)[0]
"""
------------------------------
ACTUAL TRAINING AND EVALUATION
------------------------------
"""
init = tf.global_variables_initializer()
sess.run(init)
self.load_initial_weight(sess)
print(train_batches_per_epoch)
print(val_batches_per_epoch)
print(self.mode)
print(source_file, target_file)
best = 0
#print(sess.run([v for v in tf.trainable_variables() if 'conv1/weights' in v.name]))
for epochs in range(training_epochs):
sess.run(training_source_init_op)
sess.run(training_target_init_op)
times_pass = 0
#print(sess.run([tf.get_default_graph().get_tensor_by_name('conv1/weights:0')]))
L_pred = 0
L_total = 0
L_MMD = 0
L_diff = 0
for times in range(train_batches_per_epoch):
if times_pass == val_batches_per_epoch - 1:
sess.run(training_target_init_op)
times_pass = 0
else:
times_pass += 1
p = 0.5 * epochs / training_epochs # + times/train_batches_per_epoch/training_epochs
learning_rate = init_lr / (1. + 10.0 * p)**0.75
#print(learning_rate)
alpha = 0.5 #2.0 / (1 + np.exp(-10.0 * epochs / training_epochs)) - 1 #min(0.45, 2.0 / (1 + np.exp(-10.0 * epochs / training_epochs)) - 1)
b = 1.0 * epochs / training_epochs
x1, y1 = sess.run(next_batch_sr)
x2, y2 = sess.run(next_batch_tr)
#print(times, np.mean(x1), np.mean(x2))
if self.mode == 'fine_tuning':
_, total_loss, pred_loss = sess.run(
[optimize, loss, prediction_loss],
feed_dict={
self.source:
x1,
self.target:
np.zeros(shape=(0, height, width, 3),
dtype=np.float32),
one_hot_labels:
y1,
lr:
learning_rate
})
elif self.mode == 'vgg_fine_tuning':
_, total_loss, pred_loss = sess.run(
[optimize, loss, prediction_loss],
feed_dict={
self.source:
x1,
self.target:
np.zeros(shape=(0, height, width, 3),
dtype=np.float32),
one_hot_labels:
y1,
lr:
learning_rate,
self.keep_prob:
self.KEEP_PROB_TRAINING
})
elif 'adaptation_block' in self.mode:
_, total_loss, pred_loss = sess.run(
[optimize, loss, prediction_loss],
feed_dict={
self.source: x1,
self.target: x2,
one_hot_labels: y1,
lr: learning_rate,
self.keep_prob: self.KEEP_PROB_TRAINING,
self.lamda: alpha,
self.beta: b
})
elif self.mode == 'JAN':
_, total_loss, pred_loss, M_loss, diff_loss = sess.run(
[
optimize, loss, prediction_loss, MMD_loss,
difference_loss
],
feed_dict={
self.source: x1,
self.target: x2,
one_hot_labels: y1,
lr: learning_rate,
self.keep_prob: self.KEEP_PROB_TRAINING,
self.lamda: alpha
})
L_pred = L_pred + pred_loss
L_total = L_total + total_loss
L_MMD = L_MMD + M_loss
L_diff = L_diff + diff_loss
if not (epochs % 10 == 9):
continue
#print(sess.run([v for v in tf.trainable_variables() if 'conv1/weights' in v.name]))
print(L_pred * 1.0 / train_batches_per_epoch)
print(L_MMD / train_batches_per_epoch)
print(L_diff / train_batches_per_epoch)
print(alpha)
print("TIMES: ", epochs, " LOSS: ",
L_total * 1.0 / train_batches_per_epoch)
sess.run(validation_init_op)
test_acc = 0.
num = 0.
for _ in range(val_batches_per_epoch):
img_batch, label_batch = sess.run(next_batch_val)
#print(np.mean(img_batch))
if self.mode == 'fine_tuning':
acc, num_ = sess.run(
[accuracy, accunum],
feed_dict={
self.source:
img_batch,
self.target:
np.zeros(shape=(0, height, width, 3),
dtype=np.float32),
one_hot_labels:
label_batch
})
elif 'adaptation_block' in self.mode:
acc, num_ = sess.run(
[accuracy, accunum],
feed_dict={
self.source:
img_batch,
self.target:
np.zeros(shape=(0, height, width, 3),
dtype=np.float32),
one_hot_labels:
label_batch,
self.keep_prob:
self.KEEP_PROB_VALIDATION,
self.beta:
1.0
})
elif self.mode == 'JAN':
acc, num_ = sess.run(
[accuracy, accunum],
feed_dict={
self.target:
img_batch,
self.source:
np.zeros(shape=(0, height, width, 3),
dtype=np.float32),
one_hot_labels:
label_batch,
self.keep_prob:
self.KEEP_PROB_VALIDATION
})
test_acc += acc
num += num_
print("TIMES: ", epochs, " VALIDATION: ", test_acc / num)
best = max(best, test_acc / num)
print(best)
label_path.append(0)
# 打开测试数据集目录，读取全部图片，生成图片路径列表
test_image = np.array(glob.glob(test_image_cat_path + '*.jpg')).tolist()
test_image_path_dog = np.array(glob.glob(test_image_dog_path + '*.jpg')).tolist()
test_image[len(test_image):len(test_image)] = test_image_path_dog
for i in range(len(test_image)):
if i < 1500:
test_label.append(0)
else:
test_label.append(1)
# 调用图片生成器，把训练集图片转换成三维数组
tr_data = ImageDataGenerator(
images=image_path,
labels=label_path,
batch_size=batch_size,
num_classes=num_classes)
# 调用图片生成器，把测试集图片转换成三维数组
test_data = ImageDataGenerator(
images=test_image,
labels=test_label,
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
with tf.name_scope('input'):
# 定义迭代器
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
##############################################################
#initialize data and save path
# Create parent path if it doesn't exist
if not os.path.isdir(checkpoint_path):
os.mkdir(checkpoint_path)
if not os.path.isdir(filewriter_path):
os.mkdir(filewriter_path)
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
source_tr_data = ImageDataGenerator(source_train_file,
mode='training',
batch_size=batch_size,
num_classes=num_classes,
shuffle=True,
mean=meanfile_path,
need_c=True,
click_fature_1000=click1000_path,
click_fature_200=click200_path,
click_fature_10=click10_path)
target_tr_data = ImageDataGenerator(target_train_file,
mode='training',
batch_size=batch_size,
num_classes=num_classes,
shuffle=True,
mean=meanfile_path)
source_val_data = ImageDataGenerator(source_val_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False,
# Network params
dropout_rate = 0.5
num_classes = 8
train_layers = ['fc8', 'fc7', 'fc6']
# How often we want to write the tf.summary data to disk
display_step = 20
# Path for tf.summary.FileWriter and to store model checkpoints
filewriter_path = 'D:/tensorflow/bvlc_alexnet/finetune_alexnet_with_tensorflow-master/tensorboard'
checkpoint_path = 'D:/tensorflow/bvlc_alexnet/finetune_alexnet_with_tensorflow-master/checkpoints'
with tf.device('/cpu:0'):
tr_data = ImageDataGenerator(filed,
train_file,
mode='training',
batch_size=batch_size,
num_classes=num_classes,
shuffle=True)
val_data = ImageDataGenerator(filed,
train_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
# Network params
num_classes = 4
batch_size = 32
# Path to store model checkpoints
checkpoint_path = os.path.join(current_dir, 'log', 'checkpoints')
"""
Main Part of the finetuning Script.
"""
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
test_data = ImageDataGenerator(test_file,
mode='test',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(test_data.data.output_types,
test_data.data.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the iterators
testing_init_op = iterator.make_initializer(test_data.data)
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3])
y = tf.placeholder(tf.float32, [batch_size, num_classes])
keep_prob = tf.placeholder(tf.float32)
