def train_classifiers():
genre_mapper = helper.get_genre_mapper()
dataset_size = 100
test_size = 20
dataloader = Loader(['rock','classical','jazz', 'blues','disco','country','pop','metal'],
'30new.csv','/media/files/musicsamples/genres')
#dataloader = Loader(['rock','classical','jazz', 'blues','disco','country','pop','metal'],
# '_mfcc_scaled.csv','/media/files/musicsamples/genres')
datasets = dataloader.get_dataset()
dv_pairs = []
for v in datasets.values():
dv_pairs.append(helper.combine_to_data_value_pair(v[0], v[1]))
training_set, test_set = create_sets(dataset_size - test_size, dv_pairs)
training_set = map_to_numeric_dataset(training_set, genre_mapper)
test_set = map_to_numeric_dataset(test_set, genre_mapper)
N = len(genre_mapper)
support_vector_machine = svm.SVC(C = 10**5)
kmeans = KMeans(n_clusters=N)
bayes = GaussianNB()
classifiers = [support_vector_machine,kmeans,bayes]
training_data, training_values = helper.separate_input_and_check_values(training_set)
for c in classifiers:
fit_classifier(c,training_data, training_values)
plot_confusion_matrix(c,test_set)
save_classifiers(classifiers)
def test():
# dataloader for testing accuracy computation -dataloader for training data is embedded in model
loader = Loader(FLAGS.batch_size, FLAGS.dataset)
test_iterator = loader.get_dataset(train=False).get_next()
# load model
model = Model(FLAGS.batch_size)
logits = model.logits
labels = model.y_placeholder
## create testing op - add fake nodes to simulate quantization in inference
# NOTE: keep below commented until quantization support is not enabled for training
#print ('Quantization bits: %d delay: %d ' % (FLAGS.quant_bits, FLAGS.quant_delay))
#tf.contrib.quantize.experimental_create_eval_graph(weight_bits=FLAGS.quant_bits, activation_bits=FLAGS.quant_bits)
outputs = tf.nn.softmax(logits)
test_op = tf.nn.in_top_k(outputs, labels, 1)
acc_op = tf.reduce_mean(tf.cast(test_op, tf.float32))
# set a saver for checkpointing
saver = tf.train.Saver()
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,log_device_placement=False)) as sess:
# setup logfile for this testing session
test_writer = tf.summary.FileWriter(logdir="./"+FLAGS.logdir, graph=sess.graph)
assert (tf.gfile.Exists(FLAGS.chpk_dir)), 'Chpk file doesn\'t contain a trained model/checkpoint ...'
saver.restore(sess, tf.train.latest_checkpoint("./"+FLAGS.chpk_dir))
num_batch_per_epoch_test = math.ceil(loader.num_testing_examples / FLAGS.batch_size)
counter = 0
true_count = 0
while (counter < num_batch_per_epoch_test):
counter += 1
# a batch of testing
test_x,test_y = sess.run(test_iterator)
test_equality, batch_accuracy = sess.run([test_op, acc_op],feed_dict={model.x_placeholder:test_x,model.y_placeholder:test_y,model.training:False})
true_count += np.sum(test_equality)
# add batch accuracy to summary
batch_accuracy_summary = tf.Summary()
batch_accuracy_summary.value.add(tag='Batch Accuracy',simple_value=batch_accuracy)
test_writer.add_summary(batch_accuracy_summary, global_step=counter)
test_accuracy = true_count / (FLAGS.batch_size * num_batch_per_epoch_test)
print ('Testing accuracy %.4f' % test_accuracy)
# add test accuracy to summary
accuracy_summary = tf.Summary()
accuracy_summary.value.add(tag='Testing Accuracy',simple_value=test_accuracy)
test_writer.add_summary(accuracy_summary, global_step=counter)
def test(self, restore=True):
saver = tf.train.Saver()
with tf.Session() as sess:
if restore:
saver.restore(sess, tf.train.latest_checkpoint('./saves'))
else:
sess.run(tf.global_variables_initializer())
train_loader = Loader(batch_size=64)
val_x, val_y = sess.run(
train_loader.get_dataset(train=False).get_next())
with open('fine_label_names.txt', 'r') as fp:
lines = fp.readlines()
lines = [line.rstrip('\n') for line in lines]
idx = 61
acc, outputs = sess.run(
[self.accuracy, self.outputs],
feed_dict={
self.x_placeholder: val_x,
self.y_placeholder: val_y,
self.training: False
})
for img in [
0, 1, 2, 3, 4, 5, 12, 13, 21, 23, 24, 25, 26, 28, 29, 30,
31, 32, 34, 37, 39, 40, 41, 42, 44, 45, 48, 50, 51, 52, 53,
54, 55, 56, 57, 58, 61, 62
]:
choices = np.flip(np.argsort(outputs[img],
axis=-1,
kind='quicksort',
order=None),
axis=0)[0:5]
names = [lines[x] for x in choices]
fig = plt.figure(1)
fig.add_subplot(121)
image = ((val_x[img] * 255) + 121.936059453125)
image = image.astype(np.uint8)
plt.imshow(image)
fig.add_subplot(122)
y = outputs[img][choices]
x = [0, 1, 2, 3, 4]
plt.yticks(np.arange(5), names)
plt.barh(x, y)
plt.show()
print(img)
def main(config):
loader = Loader(config)
base = Base(config, loader)
make_dirs(base.output_path)
make_dirs(base.save_logs_path)
make_dirs(base.save_model_path)
logger = Logger(os.path.join(base.save_logs_path, 'log.txt'))
logger(config)
if config.mode == 'train':
if config.resume_train_epoch >= 0:
base.resume_model(config.resume_train_epoch)
start_train_epoch = config.resume_train_epoch
else:
start_train_epoch = 0
if config.auto_resume_training_from_lastest_step:
root, _, files = os_walk(base.save_model_path)
if len(files) > 0:
indexes = []
for file in files:
indexes.append(int(
file.replace('.pkl', '').split('_')[-1]))
indexes = sorted(list(set(indexes)), reverse=False)
base.resume_model(indexes[-1])
start_train_epoch = indexes[-1]
logger(
'Time: {}, automatically resume training from the latest step (model {})'
.format(time_now(), indexes[-1]))
for current_epoch in range(start_train_epoch,
config.total_train_epoch):
base.save_model(current_epoch)
if current_epoch < config.use_graph:
_, result = train_meta_learning(base, loader)
logger('Time: {}; Epoch: {}; {}'.format(
time_now(), current_epoch, result))
if current_epoch + 1 >= 1 and (current_epoch + 1) % 40 == 0:
mAP, CMC = test(config, base, loader)
logger(
'Time: {}; Test on Target Dataset: {}, \nmAP: {} \n Rank: {}'
.format(time_now(), config.target_dataset, mAP, CMC))
else:
_, result = train_with_graph(config, base, loader)
logger('Time: {}; Epoch: {}; {}'.format(
time_now(), current_epoch, result))
if current_epoch + 1 >= 1 and (current_epoch + 1) % 5 == 0:
mAP, CMC = test_with_graph(config, base, loader)
logger(
'Time: {}; Test on Target Dataset: {}, \nmAP: {} \n Rank: {}'
.format(time_now(), config.target_dataset, mAP, CMC))
elif config.mode == 'test':
base.resume_model(config.resume_test_model)
mAP, CMC = test_with_graph(config, base, loader)
logger('Time: {}; Test on Target Dataset: {}, \nmAP: {} \n Rank: {}'.
format(time_now(), config.target_dataset, mAP, CMC))
def main(args):
vecs_builder = VecsBuilder(vecs_path='./glove/glove.6B.300d.txt')
vecs = vecs_builder.get_data()
train_dataset = Loader(args.max_length,vecs,'train')
train_loader = DataLoader(train_dataset, batch_size = args.batch_size, num_workers = 5)
val_dataset = Loader(args.max_length,vecs,'val')
val_loader = DataLoader(val_dataset, batch_size = args.batch_size)
model = Classifier(args.embed_dim, args.hidden_dim,args.num_classes,args.num_hidden_layers)
if torch.cuda.is_available():
print('Cuda Functioning..')
model.cuda()
best_acc = 0
automated_log = open('models/automated_log.txt','w+')
automated_log.write('Epochs'+'\t'+'Train-Loss'+'\t'+'Train-Accuracy'+'\t'+'Validation Loss'+'\t'+'Validation Accuracy\n')
for epoch in tqdm(range(args.num_epochs)):
train_loss,train_acc = train(model,train_loader)
val_loss,val_acc = eval(model,val_loader)
train_acc = train_acc/train_dataset.num_samples
val_acc = val_acc/val_dataset.num_samples
# print('Epoch : ',epoch)
# print('Train Loss : ',train_loss)
# print('Train Acc : ',train_acc)
# print('Validation Loss : ',val_loss)
# print('Validation Acc : ',val_acc)
automated_log.write(str(epoch)+'\t'+str(train_loss)+'\t'+str(train_acc)+'\t'+str(val_loss)+'\t'+str(val_acc)+'\n')
if epoch%10==0:
model_name = 'models/model_'+str(epoch)+'.pkl'
torch.save(model.state_dict(),model_name)
if val_acc>best_acc:
best_acc = val_acc
best_model = 'best.pkl'
torch.save(model.state_dict(),best_model)
f = open('models/best.txt','w+')
report = 'Epoch : '+str(epoch)+'\t Validation Accuracy : '+str(best_acc)
f.write(report)
f.close()
print('Best Model Saved with Valdn Accuracy :',val_acc)
automated_log.close()
def __init__(self, instructions, name=None):
l = Loader('map.txt')
self.wmap = l.f #frickin mint syntax, world map created
self.boundaries = np.array([self.wmap.shape[0], self.wmap.shape[1]])
if self.boundaries[0] == self.boundaries[1]:
self.boundaries = self.boundaries[0]
self.name = name
pos = np.where(self.wmap == 'S')
self.row = pos[0][0]
self.col = pos[1][0]
self.instructions = instructions
self.__instructions2 = instructions
self.treasures = 0
self.steps = 0
self.path = []
self.complete_path = []
self.onway = []
class CnnMaxPool(object):
def __init__(self):
self.model = dy.Model()
self.options = {'channel_1': 512, 'channel_2': 512, 'channel_3': 512}
self.params = self.init_params()
self.trainer = dy.AdamTrainer(self.model, alpha=0.01)
self.loader = Loader(sanity_check=True)
def load(self, filename):
self.model.load(filename)
def save(self, filename):
self.model.save(filename)
def init_params(self):
params = {}
# cnn层参数
params['conv_W_1'] = self.model.add_parameters(
(1, 4, 100, self.options['channel_1']))
params['conv_b_1'] = self.model.add_parameters(
self.options['channel_1'])
params['conv_W_2'] = self.model.add_parameters(
(1, 8, 100, self.options['channel_2']))
params['conv_b_2'] = self.model.add_parameters(
self.options['channel_2'])
params['conv_W_3'] = self.model.add_parameters(
(1, 12, 100, self.options['channel_3']))
params['conv_b_3'] = self.model.add_parameters(
self.options['channel_3'])
# 输出层参数
params['W'] = self.model.add_parameters(self.options['channel_1'] +
self.options['channel_2'] +
self.options['channel_3'])
params['b'] = self.model.add_parameters(1)
return params
def build_graph(self, x):
conv_W_1 = dy.parameter(self.params['conv_W_1'])
conv_b_1 = dy.parameter(self.params['conv_b_1'])
conv_W_2 = dy.parameter(self.params['conv_W_2'])
conv_b_2 = dy.parameter(self.params['conv_b_2'])
conv_W_3 = dy.parameter(self.params['conv_W_3'])
conv_b_3 = dy.parameter(self.params['conv_b_3'])
W = dy.parameter(self.params['W'])
b = dy.parameter(self.params['b'])
(n, d), _ = x.dim()
x = dy.reshape(x, (1, n, d))
# 一维卷积网络
conv_1 = dy.tanh(
dy.conv2d_bias(x, conv_W_1, conv_b_1, (1, 1), is_valid=False))
conv_2 = dy.tanh(
dy.conv2d_bias(x, conv_W_2, conv_b_2, (1, 1), is_valid=False))
conv_3 = dy.tanh(
dy.conv2d_bias(x, conv_W_3, conv_b_3, (1, 1), is_valid=False))
pool_1 = dy.max_dim(dy.reshape(conv_1, (n, self.options['channel_1'])))
pool_2 = dy.max_dim(dy.reshape(conv_2, (n, self.options['channel_2'])))
pool_3 = dy.max_dim(dy.reshape(conv_3, (n, self.options['channel_3'])))
# 全连接分类
pool = dy.concatenate([pool_1, pool_2, pool_3], 0)
logit = dy.dot_product(pool, W) + b
return logit
def backward(self, word_vectors, label):
dy.renew_cg()
x = dy.inputTensor(word_vectors)
y = dy.inputTensor(label)
logit = self.build_graph(x)
# q表示对正样本的加权
# 公式见https://www.tensorflow.org/api_docs/python/tf/nn/weighted_cross_entropy_with_logits
q = 15
l = 1 + (q - 1) * y
loss = (1 - y) * logit + l * (dy.log(1 + dy.exp(-dy.abs(logit))) +
dy.rectify(-logit))
res = loss.value()
loss.backward()
return res
def train(self):
epoch = 5
for i in xrange(epoch):
for j in xrange(7297 / 4):
for input, label in self.loader.next_batch():
loss = self.backward(input, label)
if np.isnan(loss):
print 'somthing went wrong, loss is nan.'
return
self.trainer.update()
print j, loss
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import f1_score
from tqdm import tqdm
import numpy as np
from data_loader import Loader
loader = Loader()
model = MLPClassifier()
for batch in loader.get_batches(1):
images, targets, ids = batch
processed = np.reshape(
images,
(images.shape[0], images.shape[1] * images.shape[2] * images.shape[3]))
model.fit(processed, targets)
for batch in loader.get_batches(1):
images, targets, ids = batch
processed = np.reshape(
images,
(images.shape[0], images.shape[1] * images.shape[2] * images.shape[3]))
score = model.score(processed, targets)
# score = model.score(targets, prediction)
print(score)
#!/usr/bin/env python3
import sys
from cooking_agent import CookingAgent
from data_loader import Loader
from resolution import Resolution
from node import Node
if __name__ == '__main__':
if len(sys.argv) == 1:
print("Program run without specific system arguments...")
all_facts_list = Loader.load_facts(
"./resolution_examples/chicken_broccoli_alfredo_big.txt")
last_fact = all_facts_list.pop() # last fact is the goal
print(
Resolution.check_deduction(all_facts_list, last_fact,
debug=True)[1])
# ca = CookingAgent(set(Loader.load_facts("cooking_examples/chicken_alfredo.txt")), False)
# ca.interactive()
# ca.executeCommands(Loader.load_commands("cooking_examples/chicken_alfredo_input.txt"))
elif sys.argv[1] == "resolution":
all_facts_list = Loader.load_facts(sys.argv[2])
last_fact = all_facts_list.pop() # last fact is the goal
debug = sys.argv[-1] == "verbose"
print(
Resolution.check_deduction(all_facts_list, last_fact,
debug=debug)[1])
elif sys.argv[1] == "cooking_test":
facts_set = set(Loader.load_facts(sys.argv[2]))
debug = sys.argv[-1] == "verbose"
from data_loader import Loader
from preprocess import Preprocess
loader = Loader("raw_data")
#train_data, train_labels = loader.read_plutchik_data("primary-plutchik-wheel-DFE.csv")
train_data, train_labels = loader.read_twitter_data("twitter_text_emotion.csv")
#train_data, train_labels = loader.read_biggle("training.1600000.processed.noemoticon.csv")
preprocess = Preprocess(train_data, train_labels)
#preprocess.reduce_by_label(["sadness", "happiness", "hate", "love", "worry"])
clean = preprocess.clean_data()
print(train_data[3])
print(clean[3])
print("")
print(train_data[19])
print(clean[19])
print("")
print(train_data[27])
print(clean[27])
print("")
print(train_data[222])
print(clean[222])
print("")
print(train_data[46])
print(clean[46])
print("")
print(train_data[537])
print(clean[537])
print("")
print(train_data[10240])
def do_train(sess, args):
# set CPU as the default device for the graph. Some of the operations will be moved to GPU later.
with tf.device('/cpu:0'):
# Images and labels placeholders
# images_ph = tf.placeholder(tf.float32, shape=(None,) + tuple(args.processed_size), name='input')
images_ph = tf.placeholder(tf.float32, shape=None, name='input')
labels_ph = tf.placeholder(tf.int32, shape=None, name='label')
max_seq_len_ph = tf.placeholder(tf.int32,
shape=None,
name='max_seq_len')
label_length_batch_ph = tf.placeholder(tf.int32,
shape=None,
name='label_length_batch')
# a placeholder for determining if we train or validate the network. This placeholder will be used to set dropout rates and batchnorm paramaters.
is_training_ph = tf.placeholder(tf.bool, name='is_training')
# epoch number
# 值得一看
epoch_number = tf.get_variable(
'epoch_number', [],
dtype=tf.int32,
initializer=tf.constant_initializer(0),
trainable=False,
collections=[tf.GraphKeys.GLOBAL_VARIABLES, SAVE_VARIABLES])
global_step = tf.get_variable(
'global_step', [],
dtype=tf.int32,
initializer=tf.constant_initializer(0),
trainable=False,
collections=[tf.GraphKeys.GLOBAL_VARIABLES, SAVE_VARIABLES])
# Weight Decay policy
wd = utils.get_policy(args.WD_policy, args.WD_details)
# Learning rate decay policy (if needed)
# lr = utils.get_policy(args.LR_policy, args.LR_details)
# TODO: 可能有问题
lr = 0.0001
# Create an optimizer that performs gradient descent.
optimizer = utils.get_optimizer(args.optimizer, lr)
# Create a pipeline to read data from disk
# a placeholder for setting the input pipeline batch size. This is employed to ensure that we feed each validation example only once to the network.
# Because we only use 1 GPU for validation, the validation batch size should not be more than 512.
batch_size_tf = tf.placeholder_with_default(min(512, args.batch_size),
shape=())
# A data loader pipeline to read training images and their labels
train_loader = Loader(args.train_info, args.delimiter, args.raw_size,
args.processed_size, True,
args.chunked_batch_size, args.num_prefetch,
args.num_threads, args.path_prefix, args.shuffle)
# The loader returns images, their labels, and their paths
# images, labels, info = train_loader.load()
mfcc_feat_batch, label_batch, feat_shape_batch, seq_len_batch, max_seq_len, label_length_batch = train_loader.load(
)
# build the computational graph using the provided configuration.
dnn_model = model(images_ph,
labels_ph,
utils.loss,
optimizer,
wd,
args.architecture,
args.depth,
args.num_chars,
args.num_classes,
is_training_ph,
max_seq_len_ph,
label_length_batch_ph,
args.transfer_mode,
num_gpus=args.num_gpus)
# If validation data are provided, we create an input pipeline to load the validation data
if args.run_validation:
val_loader = Loader(args.val_info, args.delimiter, args.raw_size,
args.processed_size, False, batch_size_tf,
args.num_prefetch, args.num_threads,
args.path_prefix)
# TODO: uncomment
# val_images, val_labels, val_info = val_loader.load()
# Get training operations to run from the deep learning model
train_ops = dnn_model.train_ops()
# Build an initialization operation to run below.
init = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess.run(init)
if args.retrain_from is not None:
dnn_model.load(sess, args.retrain_from)
# Set the start epoch number
start_epoch = sess.run(epoch_number + 1)
# Start the queue runners.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
# Setup a summary writer
summary_writer = tf.summary.FileWriter(args.log_dir, sess.graph)
# The main training loop
for epoch in range(start_epoch, start_epoch + args.num_epochs):
# update epoch_number
sess.run(epoch_number.assign(epoch))
print("Epoch %d started" % (epoch))
# Trainig batches
for step in range(args.num_batches):
sess.run(global_step.assign(step + epoch * args.num_batches))
# train the network on a batch of data (It also measures time)
start_time = time.time()
# load a batch from input pipeline
# img, lbl = sess.run([images, labels], options=args.run_options, run_metadata=args.run_metadata)
mfcc_feat_batch, label_batch, feat_shape_batch, seq_len_batch, max_seq_len, label_length_batch \
= sess.run([mfcc_feat_batch, label_batch, feat_shape_batch, seq_len_batch, max_seq_len, label_length_batch],
options=args.run_options, run_metadata=args.run_metadata)
# train on the loaded batch of data
_, loss_value, top1_accuracy, topn_accuracy = \
sess.run(train_ops,
feed_dict={images_ph: mfcc_feat_batch, labels_ph: label_batch,
max_seq_len_ph: max_seq_len, label_length_batch_ph: label_length_batch,
is_training_ph: True},
options=args.run_options, run_metadata=args.run_metadata)
duration = time.time() - start_time
# Check for errors
assert not np.isnan(
loss_value), 'Model diverged with loss = NaN'
# Logging every ten batches and writing tensorboard summaries every hundred batches
if step % 10 == 0:
num_examples_per_step = args.chunked_batch_size * args.num_gpus
examples_per_sec = num_examples_per_step / duration
sec_per_batch = duration / args.num_gpus
# Log
format_str = (
'%s: epoch %d, step %d, loss = %.2f, Top-1 = %.2f Top-'
+ str(args.top_n) +
' = %.2f (%.1f examples/sec; %.3f sec/batch)')
print(format_str %
(datetime.now(), epoch, step, loss_value,
top1_accuracy, topn_accuracy, examples_per_sec,
sec_per_batch))
sys.stdout.flush()
if step % 100 == 0:
summary_str = sess.run(tf.summary.merge_all(),
feed_dict={
images_ph: mfcc_feat_batch,
labels_ph: label_batch,
max_seq_len_ph: max_seq_len,
label_length_batch_ph:
label_length_batch,
is_training_ph: True
})
summary_writer.add_summary(summary_str,
args.num_batches * epoch + step)
# TODO:这里好像有bug
# if args.log_debug_info:
# summary_writer.add_run_metadata(run_metadata, 'epoch%d step%d' % (epoch, step))
# Save the model checkpoint periodically after each training epoch
checkpoint_path = os.path.join(args.log_dir, args.snapshot_prefix)
dnn_model.save(sess, checkpoint_path, global_step=epoch)
print("Epoch %d ended. a checkpoint saved at %s" %
(epoch, args.log_dir))
sys.stdout.flush()
# if validation data are provided, evaluate accuracy on the validation set after the end of each epoch
if args.run_validation:
print("Evaluating on validation set")
"""
true_predictions_count = 0 # Counts the number of correct predictions
true_topn_predictions_count = 0 # Counts the number of top-n correct predictions
total_loss = 0.0 # measures cross entropy loss
all_count = 0 # Count the total number of examples
# The validation loop
for step in range(args.num_val_batches):
# Load a batch of data
val_img, val_lbl = sess.run([val_images, val_labels], feed_dict={
batch_size_tf: args.num_val_samples % min(512, args.batch_size)} if step == args.num_val_batches - 1 else None,
options=args.run_options, run_metadata=args.run_metadata)
# validate the network on the loaded batch
val_loss, top1_predictions, topn_predictions = sess.run([train_ops[1], train_ops[2], train_ops[3]],
feed_dict={images_ph: val_img, labels_ph: val_lbl,
is_training_ph: False},
options=args.run_options, run_metadata=args.run_metadata)
all_count += val_lbl.shape[0]
true_predictions_count += int(round(val_lbl.shape[0] * top1_predictions))
true_topn_predictions_count += int(round(val_lbl.shape[0] * topn_predictions))
total_loss += val_loss * val_lbl.shape[0]
if step % 10 == 0:
print("Validation step %d of %d" % (step, args.num_val_batches))
sys.stdout.flush()
print("Total number of validation examples %d, Loss %.2f, Top-1 Accuracy %.2f, Top-%d Accuracy %.2f" %
(all_count, total_loss / all_count, true_predictions_count / all_count, args.top_n,
true_topn_predictions_count / all_count))
sys.stdout.flush()
"""
coord.request_stop()
coord.join(threads)
sess.close()
parser.add_argument("--pre_train_G_epoch", type=int, default=50, help="generator pre-training epochs")
parser.add_argument("--pre_train_D_epoch", type=int, default=50, help="discriminator pre-training epochs")
a = parser.parse_args()
if not a.output_dir:
output_prepath = 'output'
if not os.path.isdir(output_prepath):
os.makedirs(output_prepath)
a.output_dir = os.path.join(output_prepath, datetime.datetime.now().strftime("%I_%M%p_on_%B_%d_%Y"))
image_path = os.path.join(a.output_dir, 'images')
if not os.path.isdir(image_path):
os.makedirs(image_path)
loader = Loader(a.batch_size)
def append_index(filename, info):
index_path = os.path.join(a.output_dir, filename)
if os.path.exists(index_path):
index = open(index_path, "a")
else:
index = open(index_path, "w")
index.write("<html><body><table><tr>")
index.write("<th>step</th><th>input</th><th>output</th><th>target</th></tr>")
index.write("<tr>")
index.write("<td>%d</td>" % info['step'])
for kind in ["inputs", "outputs", "targets"]:
index.write("<td><img src='images/%s'></td>" % info[kind])
def train(self, restore=False):
saver = tf.train.Saver()
with tf.Session(
config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)) as sess:
try:
run_id = np.random.randint(0, 1e7)
train_writer = tf.summary.FileWriter(logdir="logs/" +
str(run_id),
graph=sess.graph)
if restore:
saver.restore(sess, tf.train.latest_checkpoint('./saves'))
else:
sess.run(tf.global_variables_initializer())
counter = 0
train_loader = Loader(batch_size=256)
val_x, val_y = sess.run(
train_loader.get_dataset(train=False).get_next())
merge = tf.summary.merge_all()
while True:
counter += 1
_, summary = sess.run([self.step, merge], feed_dict={})
train_writer.add_summary(summary, counter)
if counter % 1000 == 0:
# Check validation accuracy on 10 batches
acc = sess.run(
[self.accuracy],
feed_dict={
self.x_placeholder: val_x,
self.y_placeholder: val_y,
self.training: False
})
accuracy_summary = tf.Summary(value=[
tf.Summary.Value(tag='Validation Accuracy',
simple_value=acc)
])
train_writer.add_summary(accuracy_summary, counter)
if counter % 10000 == 0:
# Save model
print("Periodically saving model...")
save_path = saver.save(sess, "./saves/model.ckpt")
except KeyboardInterrupt:
print("Interupted... saving model.")
save_path = saver.save(sess, "./saves/model.ckpt")
def test(dataset_name, epoch, checkpoint_path, cuda):
assert dataset_name in ['mnist', 'mnist_m']
image_root = os.path.join('.', 'data', dataset_name)
batch_size = 128
image_size = 28
alpha = 0
if dataset_name == 'mnist_m':
test_list = os.path.join(image_root, 'mnist_m_test_labels.txt')
img_transform = transforms.Compose([
transforms.Resize(image_size),
transforms.ToTensor(),
transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))
])
dataset = Loader(data_root=os.path.join(image_root, 'mnist_m_test'),
data_list=test_list,
transform=img_transform)
dataloader = torch.utils.data.DataLoader(dataset=dataset,
batch_size=batch_size,
shuffle=False,
num_workers=4)
else:
img_transform_mnist = transforms.Compose([
transforms.Resize(image_size),
transforms.ToTensor(),
transforms.Lambda(lambda x: x.repeat(3, 1, 1)),
transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))
])
dataset = datasets.MNIST(root=image_root,
train=False,
transform=img_transform_mnist)
dataloader = torch.utils.data.DataLoader(dataset=dataset,
batch_size=batch_size,
shuffle=False,
num_workers=4)
model = models_.CNNModel()
ckpt = torch.load(
os.path.join(checkpoint_path, 'cp_{}ep'.format(epoch) + '.pt'))
model.load_state_dict(ckpt['model_state'])
model = model.eval()
if cuda:
model = model.cuda()
len_dataloader = len(dataloader)
data_target_iter = iter(dataloader)
i = 0
n_total = 0
n_correct = 0
while i < len_dataloader:
# test model using target data
batch = data_target_iter.next()
x, y = batch
if cuda:
x = x.cuda()
y = y.cuda()
class_output, _ = model(input_data=x, alpha=alpha)
pred = class_output.data.max(1, keepdim=True)[1]
n_correct += pred.eq(y.data.view_as(pred)).cpu().sum()
n_total += batch_size
i += 1
accu = n_correct.item() * 1.0 / n_total
print('Epoch:{}, accuracy on {}: {}.'.format(epoch + 1, dataset_name,
accu))
def train():
# for PANTHER-mode print slice precision
if (FLAGS.ifpanther):
if (FLAGS.ifmixed):
assert (len(FLAGS.slice_bits_list) == 8), "list length should be 8"
print("PUMA sliced_bits_list", FLAGS.slice_bits_list)
else:
print("PUMA slice bits: ", FLAGS.slice_bits)
# dataloader for validation accuracy computation -dataloader for training data is embedded in model
loader = Loader(FLAGS.batch_size, FLAGS.dataset)
val_iterator = loader.get_dataset(train=False).get_next()
# load model
model = Model(FLAGS.batch_size)
logits = model.logits
labels = model.y_placeholder
# create training op - add fake nodes to simulate quantization in inference
# notice the qunat_delay of num_epochs+1, just adds fake nodes to be used later in inference
softmax = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=labels, logits=logits, name="softmax_cross_entropy")
loss = tf.reduce_mean(softmax)
var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
if (FLAGS.ifpanther):
## NOTE: If not using puma-outerproduct; directly use nonideality class without going through outer_product class
# outer_product is built on example-wise gradients and is slow [To Try for speedup - see Goodfeli blog - https://github.com/tensorflow/tensorflow/issues/4897]
#nonideality = puma.nonideality(sigma=FLAGS.puma_sigma, alpha=FLAGS.puma_alpha)
puma_op = puma.outer_product(var_list=var_list, sigma=FLAGS.puma_sigma, alpha=FLAGS.puma_alpha, \
slice_bits=FLAGS.slice_bits, ifmixed=FLAGS.ifmixed, slice_bits_list=[int(x) for x in FLAGS.slice_bits_list])
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
if (FLAGS.optimizer == "vanilla"):
opt = tf.train.GradientDescentOptimizer(learning_rate=FLAGS.lr)
elif (FLAGS.optimizer == "momentum"):
opt = tf.train.MomentumOptimizer(learning_rate=FLAGS.lr,
momentum=0.9)
elif (FLAGS.optimizer == "nesterov"):
opt = tf.train.MomentumOptimizer(learning_rate=FLAGS.lr,
momentum=0.9,
use_nesterov=True)
else:
opt = tf.train.AdamOptimizer(learning_rate=FLAGS.lr)
# Layer gradient computation
grad = opt.compute_gradients(loss)
# Weight gradient computation and update
if (FLAGS.ifpanther):
print('Adding PANTHER ops')
## NOTE: If not using puma-outerproduct; directly use nonideality class without going through outer_product class
#grad_n = nonideality.apply(grad)
grad_n = puma_op.apply_batch(grad)
train_op = opt.apply_gradients(zip(grad_n[0], var_list))
else:
grad_n = grad # added this placeholder for grad_n for consistency among different run types
train_op = opt.apply_gradients(grad)
# create ops for validation and training accuracy
outputs = tf.nn.softmax(logits)
equality = tf.nn.in_top_k(outputs, labels, 1)
accuracy = tf.reduce_mean(tf.cast(equality, tf.float32))
# create ops for top-5 accuracy
equality5 = tf.nn.in_top_k(outputs, labels, 5)
accuracy5 = tf.reduce_mean(tf.cast(equality5, tf.float32))
tf.summary.scalar("Loss", loss)
tf.summary.scalar("Training accuracy - Top-1", accuracy)
tf.summary.scalar("Training accuracy - Top-5", accuracy5)
# capture slice-wise saturation statistics
if (FLAGS.ifpanther):
puma_sat_stats = grad_n[1]
tf.summary.scalar("PUMA Parallel Write Saturation", puma_sat_stats[1])
for i in range(puma_sat_stats[0].get_shape()[0]):
tf.summary.scalar(
"PUMA Parallel Write Saturation-" + "Slice " + str(i),
puma_sat_stats[0][i])
# puma crs_sync
if (FLAGS.ifpanther):
crs_op = puma_op.crs_sync(var_list)
# set a saver for checkpointing
saver = tf.train.Saver()
# run training within a session
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)) as sess:
#sess = tf_debug.LocalCLIDebugWrapperSession(sess)
try:
# setup logfile for this training session
train_writer = tf.summary.FileWriter(logdir="./" + FLAGS.logdir,
graph=sess.graph)
# setup counter, summary writes and model based on if restore required
# keep track of progress during training with counter - for correct restoring from last checkpoint
if FLAGS.restore:
assert (
tf.gfile.Exists(FLAGS.logdir)
), 'Restore requires log file from previous run, set restore to False and run...'
print('restoring train from: %s' % FLAGS.logdir)
saver.restore(sess,
tf.train.latest_checkpoint("./" + FLAGS.logdir))
ckpt_name = tf.train.get_checkpoint_state(
FLAGS.logdir
).model_checkpoint_path # extract the latest checkpoint
## patch for case where model gets saved naturally, not because it stopped due to interrupt
# counter = int(ckpt_name.split('-')[1]) # extract the counter from checkpoint
temp_l = ckpt_name.split('-')
if (len(temp_l) > 1):
counter = int(ckpt_name.split('-')
[1]) # extract the counter from checkpoint
else:
print("Restoring from counter:", FLAGS.start_counter)
counter = FLAGS.start_counter
else:
counter = 0
sess.run(tf.global_variables_initializer())
merge = tf.summary.merge_all()
print('counter: ', counter)
# NOTE: keep below commented until quantization support is not enabled for training
#print ('Quantization bits: %d delay: %d ' % (FLAGS.quant_bits, FLAGS.quant_delay))
# train network for user-defined epochs
num_batch_per_epoch_train = math.ceil(
loader.num_training_examples / FLAGS.batch_size)
print('number of batches per epoch: ', num_batch_per_epoch_train)
while (counter < FLAGS.epochs * num_batch_per_epoch_train):
counter += 1
# puma carry resolution step
if (FLAGS.ifpanther and (counter % FLAGS.crs_freq == 0)):
print("Performing puma crs.....")
sess.run(crs_op)
# a batch of training
start_time = time.time()
## Uncomment the lines below if running detailed traces - for runtime compute and mmeory data
#run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) #Uncomment these to get run-time compute/memory utilization
#run_metadata = tf.RunMetadata()
#_, _, summary = sess.run([grad_n, train_op, merge],feed_dict={}, options=run_options, run_metadata=run_metadata)
_, _, summary = sess.run([grad_n, train_op, merge],
feed_dict={})
duration = time.time() - start_time
print("Step: %d \t Training time (1 batch): %0.4f" %
(counter, duration))
## Uncomment the lines below if running detailed traces - for runtime compute and mmeory data
#train_writer.add_run_metadata(run_metadata, 'step%d' % counter)
train_writer.add_summary(summary, global_step=counter)
# compute validation accuracy every epoch
if (counter % num_batch_per_epoch_train == 0):
num_batch_per_epoch_val = math.ceil(
loader.num_testing_examples / FLAGS.batch_size)
val_counter = 0
true_count = 0
true_count5 = 0
while (val_counter < num_batch_per_epoch_val):
val_counter += 1
# a batch of validation
val_x, val_y = sess.run(val_iterator)
val_equality, val_equality5 = sess.run(
[equality, equality5],
feed_dict={
model.x_placeholder: val_x,
model.y_placeholder: val_y,
model.training: False
})
true_count += np.sum(val_equality)
true_count5 += np.sum(val_equality5)
val_accuracy = true_count / (FLAGS.batch_size *
num_batch_per_epoch_val)
val_accuracy5 = true_count5 / (FLAGS.batch_size *
num_batch_per_epoch_val)
accuracy_summary = tf.Summary()
accuracy_summary.value.add(
tag='Validation Accuracy - Top-1',
simple_value=val_accuracy)
accuracy_summary.value.add(
tag='Validation Accuracy - Top-5',
simple_value=val_accuracy5)
train_writer.add_summary(accuracy_summary,
global_step=counter)
print('Validation accuracy: Top-1 %.4f \t Top-5 %.4f' %
(val_accuracy, val_accuracy5))
if (counter %
(FLAGS.chpk_freq * num_batch_per_epoch_train) == 0):
# Save model
print("Periodically saving model...")
save_path = saver.save(sess,
"./" + FLAGS.logdir + "/model.ckpt")
except KeyboardInterrupt:
print("Interupted... saving model.")
save_path = saver.save(
sess, "./" + FLAGS.logdir + "/model.ckpt-" + str(counter))
def main():
for k, v in a._get_kwargs():
print(k, "=", v)
with open(os.path.join(a.output_dir, "options.json"), "w") as f:
f.write(json.dumps(vars(a), sort_keys=True, indent=4))
loader = Loader(a.batch_size)
# initialize models here
model = CNN(a)
if a.checkpoint is not None:
print("loading model from checkpoint")
model.load(a.checkpoint)
if a.mode == 'test':
if a.checkpoint is None:
print('need checkpoint to continue')
return
draw(model, os.path.join(a.output_dir, 'test_output.jpg'))
else:
# training
start = time.time()
for epoch in range(a.max_epochs):
def should(freq):
return freq > 0 and ((epoch + 1) % freq == 0
or epoch == a.max_epochs - 1)
training_loss = 0
for _ in range(loader.ntrain):
X, y = loader.next_batch(0)
model.step(X, y)
training_loss += model.loss.data[0]
training_loss /= loader.ntrain
if should(a.validation_freq):
print('validating model')
validation_loss = 0
for _ in range(loader.nval):
X, y = loader.next_batch(1)
model.validate_step(X, y)
validation_loss += model.loss.data[0]
validation_loss /= loader.nval
if should(a.summary_freq):
print("recording summary")
with open(os.path.join(a.output_dir, 'loss_record.txt'),
"a") as loss_file:
loss_file.write("%s\t%s\t%s\n" %
(epoch, training_loss, validation_loss))
if should(a.progress_freq):
rate = (epoch + 1) / (time.time() - start)
remaining = (a.max_epochs - 1 - epoch) / rate
print("progress epoch %d remaining %dh" %
(epoch, remaining / 3600))
print("training loss", training_loss)
if should(a.display_freq):
draw(model, os.path.join(a.output_dir, '%s.jpg' % epoch))
if should(a.save_freq):
print("saving model")
model.save(os.path.join(a.output_dir, '%s.pth' % epoch))
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
source_path = os.path.join('.', 'data', args.source)
target_path = os.path.join('.', 'data', args.target)
image_size = 28
img_transform = transforms.Compose([transforms.Resize(image_size), transforms.ToTensor(), transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))])
img_transform_mnist = transforms.Compose([transforms.Resize(image_size), transforms.ToTensor(), transforms.Lambda(lambda x: x.repeat(3, 1, 1)), transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))])
source_dataset = datasets.MNIST(root=source_path, download=True, train=True, transform=img_transform_mnist)
source_loader = torch.utils.data.DataLoader(dataset=source_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers)
train_list = os.path.join(target_path, 'mnist_m_train_labels.txt')
target_dataset = Loader(data_root=os.path.join(target_path, 'mnist_m_train'), data_list=train_list, transform=img_transform)
target_loader = torch.utils.data.DataLoader(dataset=target_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers)
model = models_.CNNModel()
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=0.9)
if args.cuda:
model = model.cuda()
torch.backends.cudnn.benchmark=True
trainer = TrainLoop(model, optimizer, source_loader, target_loader, checkpoint_path=args.checkpoint_path, checkpoint_epoch=args.checkpoint_epoch, cuda=args.cuda, target_name = args.target)
print('Cuda Mode: {}'.format(args.cuda))
print('Batch size: {}'.format(args.batch_size))
print('LR: {}'.format(args.lr))
print('Source: {}'.format(args.source))
def __init__(self):
self.model = dy.Model()
self.options = {'channel_1': 512, 'channel_2': 512, 'channel_3': 512}
self.params = self.init_params()
self.trainer = dy.AdamTrainer(self.model, alpha=0.01)
self.loader = Loader(sanity_check=True)
def __init__(self):
loader = Loader()
iterator = loader.get_dataset()
def build_model():
with tf.device("/device:GPU:0"):
x_loaded, y_loaded = iterator.get_next()
x = tf.placeholder_with_default(x_loaded, (None, 32, 32, 3),
name="x_placeholder")
y = tf.placeholder_with_default(y_loaded, (None),
name="y_placeholder")
training = tf.placeholder_with_default(True,
name="training_bool",
shape=())
#Layer1 - 64 channels
conv1 = tf.layers.conv2d(
x,
filters=64,
kernel_size=(3, 3),
padding='SAME',
use_bias=True,
kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_1 = tf.contrib.layers.batch_norm(conv1,
activation_fn=tf.nn.relu,
is_training=training)
# Layer2 - 64 channels
conv2 = tf.layers.conv2d(
bn_1,
filters=64,
kernel_size=(3, 3),
padding='SAME',
use_bias=True,
kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_2 = tf.contrib.layers.batch_norm(conv2,
activation_fn=tf.nn.relu,
is_training=training)
pool2 = tf.layers.max_pooling2d(bn_2, (2, 2), (2, 2),
padding='SAME')
dropout_2 = tf.layers.dropout(pool2,
training=training,
rate=0.5)
#Layer 3 - 128 channels
conv3 = tf.layers.conv2d(
dropout_2,
filters=128,
kernel_size=(3, 3),
padding='SAME',
use_bias=True,
kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_3 = tf.contrib.layers.batch_norm(conv3,
activation_fn=tf.nn.relu,
is_training=training)
# Layer 4 - 128 channels
conv4 = tf.layers.conv2d(
bn_3,
filters=128,
kernel_size=(3, 3),
padding='SAME',
use_bias=True,
kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_4 = tf.contrib.layers.batch_norm(conv4,
activation_fn=tf.nn.relu,
is_training=training)
pool4 = tf.layers.max_pooling2d(bn_4, (2, 2), (2, 2),
padding='SAME')
dropout_4 = tf.layers.dropout(pool4,
training=training,
rate=0.5)
#Layer 5 - 256 channels
conv5 = tf.layers.conv2d(
dropout_4,
filters=256,
kernel_size=(3, 3),
padding='SAME',
use_bias=True,
kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_5 = tf.contrib.layers.batch_norm(conv5,
activation_fn=tf.nn.relu,
is_training=training)
# Layer 6 - 256 channels
conv6 = tf.layers.conv2d(
bn_5,
filters=256,
kernel_size=(3, 3),
padding='SAME',
use_bias=True,
kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_6 = tf.contrib.layers.batch_norm(conv6,
activation_fn=tf.nn.relu,
is_training=training)
# Layer 7 - 256 channels
conv7 = tf.layers.conv2d(
bn_6,
filters=256,
kernel_size=(3, 3),
padding='SAME',
use_bias=True,
kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_7 = tf.contrib.layers.batch_norm(conv7,
activation_fn=tf.nn.relu,
is_training=training)
pool7 = tf.layers.max_pooling2d(bn_7, (2, 2), (2, 2),
padding='SAME')
dropout_7 = tf.layers.dropout(pool7,
training=training,
rate=0.5)
# Layer 8 - 512 channels
conv8 = tf.layers.conv2d(
dropout_7,
filters=512,
kernel_size=(3, 3),
padding='SAME',
use_bias=True,
kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_8 = tf.contrib.layers.batch_norm(conv8,
activation_fn=tf.nn.relu,
is_training=training)
# Layer 9 - 512 channels
conv9 = tf.layers.conv2d(
bn_8,
filters=512,
kernel_size=(3, 3),
padding='SAME',
use_bias=True,
kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_9 = tf.contrib.layers.batch_norm(conv9,
activation_fn=tf.nn.relu,
is_training=training)
# Layer 10 - 512 channels
conv10 = tf.layers.conv2d(
bn_9,
filters=512,
kernel_size=(3, 3),
padding='SAME',
use_bias=True,
kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_10 = tf.contrib.layers.batch_norm(conv10,
activation_fn=tf.nn.relu,
is_training=training)
pool10 = tf.layers.max_pooling2d(bn_10, (2, 2), (2, 2),
padding='SAME')
dropout_7 = tf.layers.dropout(pool10,
training=training,
rate=0.5)
# Layer 11 - 512 channels
conv11 = tf.layers.conv2d(
dropout_7,
filters=512,
kernel_size=(3, 3),
padding='SAME',
use_bias=True,
kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_11 = tf.contrib.layers.batch_norm(conv11,
activation_fn=tf.nn.relu,
is_training=training)
# Layer 12 - 512 channels
conv12 = tf.layers.conv2d(
bn_11,
filters=512,
kernel_size=(3, 3),
padding='SAME',
use_bias=True,
kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_12 = tf.contrib.layers.batch_norm(conv12,
activation_fn=tf.nn.relu,
is_training=training)
# Layer 13 - 512 channels
conv13 = tf.layers.conv2d(
bn_12,
filters=512,
kernel_size=(3, 3),
padding='SAME',
use_bias=True,
kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_13 = tf.contrib.layers.batch_norm(conv13,
activation_fn=tf.nn.relu,
is_training=training)
pool13 = tf.layers.max_pooling2d(bn_13, (2, 2), (2, 2),
padding='SAME')
dropout_13 = tf.layers.dropout(pool13,
training=training,
rate=0.5)
flattened = tf.contrib.layers.flatten(dropout_13)
dense14 = tf.layers.dense(
inputs=flattened,
units=4096,
kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_14 = tf.contrib.layers.batch_norm(dense14,
activation_fn=tf.nn.relu,
is_training=training)
dropout_14 = tf.layers.dropout(bn_14,
training=training,
rate=0.5)
dense15 = tf.layers.dense(
inputs=dropout_14,
units=4096,
kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_15 = tf.contrib.layers.batch_norm(dense15,
activation_fn=tf.nn.relu,
is_training=training)
dense16 = tf.layers.dense(
inputs=bn_15,
units=100,
activation=None,
kernel_initializer=tf.contrib.layers.xavier_initializer())
tf.summary.scalar("dense16_mean", tf.reduce_mean(dense16))
# Predict
outputs = tf.nn.softmax(dense16)
prediction = tf.argmax(outputs, axis=1)
equality = tf.equal(prediction, y)
accuracy = tf.reduce_mean(tf.cast(equality, tf.float32))
# Train
softmax = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=y, logits=dense16, name="softmax")
loss = tf.reduce_mean(softmax)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
step = tf.train.AdamOptimizer(
learning_rate=0.0001).minimize(loss)
self.loss = loss
self.x_placeholder = x
self.y_placeholder = y
self.training = training
self.accuracy = accuracy
self.outputs = outputs
self.prediction = prediction
self.step = step
tf.summary.scalar("Loss", loss)
tf.summary.scalar("Train Accuracy", accuracy)
build_model()
def main():
z_dim = 100
t_dim = 256
image_size = 64
gf_dim = 64
df_dim = 64
gfc_dim = 1024
caption_vector_length = 2400
batch_size = 64
options = {
'z_dim': z_dim,
't_dim': t_dim,
'image_size': image_size,
'gf_dim': gf_dim,
'df_dim': df_dim,
'gfc_dim': gfc_dim,
'caption_vector_length': caption_vector_length,
'batch_size': batch_size
}
epochs = 300
learning_rate = 0.0002
beta1 = 0.5
gan = model.GAN(options)
input_tensors, variables, loss, outputs, checks = gan.build_model()
d_optim = tf.train.AdamOptimizer(learning_rate, beta1=beta1).minimize(
loss['d_loss'], var_list=variables['d_vars'])
g_optim = tf.train.AdamOptimizer(learning_rate, beta1=beta1).minimize(
loss['g_loss'], var_list=variables['g_vars'])
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
saver = tf.train.Saver()
# if args.resume_model:
# saver.restore(sess, args.resume_model)
loader = Loader()
for i in range(epochs):
batch_no = 0
for real_images, wrong_images, captions in loader.train_data(
batch_size):
real_images /= 255.0
wrong_images /= 255.0
captions = captions[:, :caption_vector_length]
z_noise = np.random.normal(0, 1, [batch_size, z_dim])
# DISCR UPDATE
check_ts = [
checks['d_loss1'], checks['d_loss2'], checks['d_loss3']
]
_, d_loss, gen, d1, d2, d3 = sess.run(
[d_optim, loss['d_loss'], outputs['generator']] + check_ts,
feed_dict={
input_tensors['t_real_image']: real_images,
input_tensors['t_wrong_image']: wrong_images,
input_tensors['t_real_caption']: captions,
input_tensors['t_z']: z_noise,
})
# GEN UPDATE
_, g_loss, gen = sess.run(
[g_optim, loss['g_loss'], outputs['generator']],
feed_dict={
input_tensors['t_real_image']: real_images,
input_tensors['t_wrong_image']: wrong_images,
input_tensors['t_real_caption']: captions,
input_tensors['t_z']: z_noise,
})
# GEN UPDATE TWICE, to make sure d_loss does not go to 0
_, g_loss, gen = sess.run(
[g_optim, loss['g_loss'], outputs['generator']],
feed_dict={
input_tensors['t_real_image']: real_images,
input_tensors['t_wrong_image']: wrong_images,
input_tensors['t_real_caption']: captions,
input_tensors['t_z']: z_noise,
})
print(i, batch_no, d_loss, g_loss)
with open('loss_log', 'a+') as out:
out.write('%s,%s,%s,%s\n' % (i, batch_no, d_loss, g_loss))
batch_no += 1
if (batch_no % 30) == 0:
print("Saving Images, Model")
save_for_vis(real_images, gen)
save_path = saver.save(sess, "models/latest_model_temp.ckpt")
if i % 5 == 0:
save_path = saver.save(sess, "models/model_epoch_%d.ckpt" % i)
def restore_robot(self):
l = Loader('map2.txt')
self.wmap = l.f
pos = np.where(self.wmap == 'S')
self.row = pos[0][0]
self.col = pos[1][0]
def __init__(self, batch_size):
loader = Loader(batch_size, FLAGS.dataset)
iterator = loader.get_dataset()
def build_model():
with tf.device("/device:GPU:0"):
x_loaded,y_loaded = iterator.get_next()
x = tf.placeholder_with_default(x_loaded,(None,32,32,3),name="x_placeholder")
y = tf.placeholder_with_default(y_loaded,(None),name="y_placeholder")
training = tf.placeholder_with_default(True,name="training_bool",shape=())
#Layer1 - 64 channels
conv1 = tf.layers.conv2d(x, filters=64,kernel_size=(3,3),padding='SAME',
use_bias=True,kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_1 = tf.contrib.layers.batch_norm(conv1,activation_fn=tf.nn.relu,is_training=training)
# Layer2 - 64 channels
conv2 = tf.layers.conv2d(bn_1, filters=64,kernel_size=(3,3),padding='SAME',
use_bias=True,kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_2 = tf.contrib.layers.batch_norm(conv2,activation_fn=tf.nn.relu,is_training=training)
pool2 = tf.layers.max_pooling2d(bn_2, (2,2), (2,2), padding='SAME')
dropout_2 = tf.layers.dropout(pool2,training=training,rate=0.5)
#Layer 3 - 128 channels
conv3 = tf.layers.conv2d(dropout_2, filters=128,kernel_size=(3,3),padding='SAME',
use_bias=True,kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_3 = tf.contrib.layers.batch_norm(conv3,activation_fn=tf.nn.relu,is_training=training)
# Layer 4 - 128 channels
conv4 = tf.layers.conv2d(bn_3, filters=128,kernel_size=(3,3),padding='SAME',
use_bias=True,kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_4 = tf.contrib.layers.batch_norm(conv4,activation_fn=tf.nn.relu,is_training=training)
pool4 = tf.layers.max_pooling2d(bn_4, (2,2), (2,2), padding='SAME')
dropout_4 = tf.layers.dropout(pool4,training=training,rate=0.5)
#Layer 5 - 256 channels
conv5 = tf.layers.conv2d(dropout_4, filters=256,kernel_size=(3,3),padding='SAME',
use_bias=True,kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_5 = tf.contrib.layers.batch_norm(conv5,activation_fn=tf.nn.relu,is_training=training)
# Layer 6 - 256 channels
conv6 = tf.layers.conv2d(bn_5, filters=256,kernel_size=(3,3),padding='SAME',
use_bias=True,kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_6 = tf.contrib.layers.batch_norm(conv6,activation_fn=tf.nn.relu,is_training=training)
# Layer 7 - 256 channels
conv7 = tf.layers.conv2d(bn_6, filters=256,kernel_size=(3,3),padding='SAME',
use_bias=True,kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_7 = tf.contrib.layers.batch_norm(conv7,activation_fn=tf.nn.relu,is_training=training)
pool7 = tf.layers.max_pooling2d(bn_7, (2,2), (2,2), padding='SAME')
dropout_7 = tf.layers.dropout(pool7,training=training,rate=0.5)
# Layer 8 - 512 channels
conv8 = tf.layers.conv2d(dropout_7, filters=512,kernel_size=(3,3),padding='SAME',
use_bias=True,kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_8 = tf.contrib.layers.batch_norm(conv8,activation_fn=tf.nn.relu,is_training=training)
# Layer 9 - 512 channels
conv9 = tf.layers.conv2d(bn_8, filters=512,kernel_size=(3,3),padding='SAME',
use_bias=True,kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_9 = tf.contrib.layers.batch_norm(conv9,activation_fn=tf.nn.relu,is_training=training)
# Layer 10 - 512 channels
conv10 = tf.layers.conv2d(bn_9, filters=512,kernel_size=(3,3),padding='SAME',
use_bias=True,kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_10 = tf.contrib.layers.batch_norm(conv10,activation_fn=tf.nn.relu,is_training=training)
pool10 = tf.layers.max_pooling2d(bn_10, (2,2), (2,2), padding='SAME')
dropout_7 = tf.layers.dropout(pool10,training=training,rate=0.5)
# Layer 11 - 512 channels
conv11 = tf.layers.conv2d(dropout_7, filters=512,kernel_size=(3,3),padding='SAME',
use_bias=True,kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_11 = tf.contrib.layers.batch_norm(conv11,activation_fn=tf.nn.relu,is_training=training)
# Layer 12 - 512 channels
conv12 = tf.layers.conv2d(bn_11, filters=512,kernel_size=(3,3),padding='SAME',
use_bias=True,kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_12 = tf.contrib.layers.batch_norm(conv12,activation_fn=tf.nn.relu,is_training=training)
# Layer 13 - 512 channels
conv13 = tf.layers.conv2d(bn_12, filters=512,kernel_size=(3,3),padding='SAME',
use_bias=True,kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_13 = tf.contrib.layers.batch_norm(conv13,activation_fn=tf.nn.relu,is_training=training)
pool13 = tf.layers.max_pooling2d(bn_13, (2,2), (2,2), padding='SAME')
dropout_13 = tf.layers.dropout(pool13,training=training,rate=0.5)
flattened = tf.contrib.layers.flatten(dropout_13)
# Layer 14
dense14 = tf.layers.dense(inputs=flattened, units=4096,kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_14 = tf.contrib.layers.batch_norm(dense14,activation_fn=tf.nn.relu,is_training=training)
dropout_14 = tf.layers.dropout(bn_14,training=training,rate=0.5)
# Layer 15
dense15 = tf.layers.dense(inputs=dropout_14, units=4096,kernel_initializer=tf.contrib.layers.xavier_initializer())
bn_15 = tf.contrib.layers.batch_norm(dense15,activation_fn=tf.nn.relu,is_training=training)
## Layer 16
dense16 = tf.layers.dense(inputs=bn_15, units=100,activation=None,kernel_initializer=tf.contrib.layers.xavier_initializer())
tf.summary.scalar("dense16_mean",tf.reduce_mean(dense16))
# data used in the model_op
self.x_placeholder = x
self.y_placeholder = y
self.training = training
self.logits = dense16
build_model()
for p in path:
if os.path.exists(p) is False:
os.mkdir(p)
VERBOSE = 0
early_stopping = EarlyStopping(patience=20)
PRE_EPOCH = 100
PRE_BATCH = 200
SEQ_EPOCH = 100
SEQ_BATCH = 10
for i in range(4, 20, 1):
print(i, '-th fold out of 20-fold cross validation')
d = Loader()
# pre training
d.load_pretrain(i)
pre_model = sleepNetwork()
pre_history = pre_model.fit(d.X_train,
d.y_train,
batch_size=PRE_BATCH,
epochs=PRE_EPOCH,
verbose=VERBOSE,
validation_data=(d.X_valid, d.y_valid),
callbacks=[early_stopping])
get_acc(pre_model.predict(d.X_test), d.y_test)
pre_model.save(join(path[0], 'pre_model_' + str(i) + '.weights'))
def do_evaluate(sess, args):
with tf.device('/cpu:0'):
# Images and labels placeholders
images_ph = tf.placeholder(tf.float32,
shape=(None, ) + tuple(args.processed_size),
name='input')
labels_ph = tf.placeholder(tf.int32, shape=(None), name='label')
# a placeholder for determining if we train or validate the network. This placeholder will be used to set dropout rates and batchnorm paramaters.
is_training_ph = tf.placeholder(tf.bool, name='is_training')
# build a deep learning model using the provided configuration
dnn_model = model(images_ph, labels_ph, utils.loss, None, 0.0,
args.architecture, args.depth, args.num_chars,
args.num_classes, is_training_ph, args.transfer_mode)
# creating an input pipeline to read data from disk
# a placeholder for setting the input pipeline batch size. This is employed to ensure that we feed each validation example only once to the network.
batch_size_tf = tf.placeholder_with_default(args.batch_size, shape=())
# a data loader pipeline to read test data
val_loader = Loader(args.val_info,
args.delimiter,
args.raw_size,
args.processed_size,
False,
batch_size_tf,
args.num_prefetch,
args.num_threads,
args.path_prefix,
inference_only=args.inference_only)
# if we want to do inference only (i.e. no label is provided) we only load images and their paths
if not args.inference_only:
val_images, val_labels, val_info = val_loader.load()
else:
val_images, val_info = val_loader.load()
# get evaluation operations from the dnn model
eval_ops = dnn_model.evaluate_ops(args.inference_only)
# Build an initialization operation to run below.
init = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess.run(init)
# Load pretrained parameters from disk
dnn_model.load(sess, args.log_dir)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
# evaluation
if not args.inference_only:
true_predictions_count = 0 # Counts the number of correct predictions
true_topn_predictions_count = 0 # Counts the number of correct top-n predictions
total_loss = 0.0 # Measures cross entropy loss
all_count = 0 # Counts the total number of examples
# Open an output file to write predictions
out_file = open(args.save_predictions, 'w')
predictions_format_str = ('%d, %s, %d, %s, %s\n')
for step in range(args.num_val_batches):
# Load a batch of data
val_img, val_lbl, val_inf = sess.run(
[val_images, val_labels, val_info],
feed_dict={
batch_size_tf: args.num_val_samples % args.batch_size
} if step == args.num_val_batches - 1 else None)
# Evaluate the network on the loaded batch
val_loss, top1_predictions, topn_predictions, topnguesses, topnconf = sess.run(
eval_ops,
feed_dict={
images_ph: val_img,
labels_ph: val_lbl,
is_training_ph: False
},
options=args.run_options,
run_metadata=args.run_metadata)
true_predictions_count += np.sum(top1_predictions)
true_topn_predictions_count += np.sum(topn_predictions)
all_count += top1_predictions.shape[0]
total_loss += val_loss * val_lbl.shape[0]
print(
'Batch Number: %d, Top-1 Hit: %d, Top-%d Hit: %d, Loss %.2f, Top-1 Accuracy: %.3f, Top-%d Accuracy: %.3f'
% (step, true_predictions_count, args.top_n,
true_topn_predictions_count, total_loss / all_count,
true_predictions_count / all_count, args.top_n,
true_topn_predictions_count / all_count))
# log results into an output file
for i in range(0, val_inf.shape[0]):
out_file.write(
predictions_format_str %
(step * args.batch_size + i + 1, str(
val_inf[i]).encode('utf-8'), val_lbl[i], ', '.join(
'%d' % item
for item in topnguesses[i]), ', '.join(
'%.4f' % item for item in topnconf[i])))
out_file.flush()
out_file.close()
# inference
else:
# Open an output file to write predictions
out_file = open(args.save_predictions, 'w')
predictions_format_str = ('%d, %s, %s, %s\n')
for step in range(args.num_val_batches):
# Load a batch of data
val_img, val_inf = sess.run(
[val_images, val_info],
feed_dict={
batch_size_tf: args.num_val_samples % args.batch_size
} if step == args.num_val_batches - 1 else None)
# Run the network on the loaded batch
topnguesses, topnconf = sess.run(
eval_ops,
feed_dict={
images_ph: val_img,
is_training_ph: False
},
options=args.run_options,
run_metadata=args.run_metadata)
print('Batch Number: %d of %d is done' %
(step, args.num_val_batches))
# Log to an output file
for i in range(0, val_inf.shape[0]):
out_file.write(predictions_format_str %
(step * args.batch_size + i + 1,
str(val_inf[i]).encode('utf-8'), ', '.join(
'%d' % item
for item in topnguesses[i]), ', '.join(
'%.4f' % item
for item in topnconf[i])))
out_file.flush()
out_file.close()
coord.request_stop()
coord.join(threads)
sess.close()