def get_a_r_psi(folder):
if not folder.endswith('/'): folder += '/'
files = sorted([f for f in os.listdir(folder) if f.startswith('batch-N')])
a = None
l = []
r_series = []
psi_series = []
for f in files:
h, data = load_batch(folder + f)
if a is None:
a = data.coupling.values
elif not all(a == data.coupling.values):
print("coupling values mismatch at file", f)
sys.exit()
l.append(1 / h['N'])
r_series.append(data.r.values)
psi_series.append(data.psi.values)
l = np.array(l)
r_series = np.vstack(r_series).transpose()
psi_series = np.vstack(psi_series).transpose()
return l, a, r_series, psi_series
def run():
if not os.path.exists(log_eval):
os.mkdir(log_eval)
#Just construct the graph from scratch again
with tf.Graph().as_default() as graph:
tf.logging.set_verbosity(tf.logging.INFO)
#Get the dataset first and load one batch of validation images and labels tensors. Set is_training as False so as to use the evaluation preprocessing
dataset = get_split('validation', dataset_dir)
images, raw_images, labels = load_batch(dataset,
batch_size=batch_size,
is_training=False)
#Create some information about the training steps
num_batches_per_epoch = dataset.num_samples / batch_size
num_steps_per_epoch = num_batches_per_epoch
#Now create the inference model but set is_training=False
with slim.arg_scope(inception_resnet_v2_arg_scope()):
logits, end_points = inception_resnet_v2(
images, num_classes=dataset.num_classes, is_training=False)
# #get all the variables to restore from the checkpoint file and create the saver function to restore
variables_to_restore = slim.get_variables_to_restore()
saver = tf.train.Saver(variables_to_restore)
def restore_fn(sess):
return saver.restore(sess, checkpoint_file)
#Just define the metrics to track without the loss or whatsoever
predictions = tf.argmax(end_points['Predictions'], 1)
accuracy, accuracy_update = tf.contrib.metrics.streaming_accuracy(
predictions, labels)
metrics_op = tf.group(accuracy_update)
#Create the global step and an increment op for monitoring
global_step = get_or_create_global_step()
global_step_op = tf.assign(
global_step, global_step + 1
) #no apply_gradient method so manually increasing the global_step
#Create a evaluation step function
def eval_step(sess, metrics_op, global_step):
'''
Simply takes in a session, runs the metrics op and some logging information.
'''
start_time = time.time()
_, global_step_count, accuracy_value = sess.run(
[metrics_op, global_step_op, accuracy])
time_elapsed = time.time() - start_time
#Log some information
logging.info(
'Global Step %s: Streaming Accuracy: %.4f (%.2f sec/step)',
global_step_count, accuracy_value, time_elapsed)
return accuracy_value
#Define some scalar quantities to monitor
tf.summary.scalar('Validation_Accuracy', accuracy)
my_summary_op = tf.summary.merge_all()
#Get your supervisor
sv = tf.train.Supervisor(logdir=log_eval,
summary_op=None,
saver=None,
init_fn=restore_fn)
#Now we are ready to run in one session
with sv.managed_session() as sess:
for step in range(int(num_steps_per_epoch * num_epochs)):
sess.run(sv.global_step)
#print vital information every start of the epoch as always
if step % num_batches_per_epoch == 0:
logging.info('Epoch: %s/%s',
step / num_batches_per_epoch + 1, num_epochs)
logging.info('Current Streaming Accuracy: %.4f',
sess.run(accuracy))
#Compute summaries every 10 steps and continue evaluating
if step % 10 == 0:
eval_step(sess,
metrics_op=metrics_op,
global_step=sv.global_step)
summaries = sess.run(my_summary_op)
sv.summary_computed(sess, summaries)
#Otherwise just run as per normal
else:
eval_step(sess,
metrics_op=metrics_op,
global_step=sv.global_step)
#At the end of all the evaluation, show the final accuracy
logging.info('Final Streaming Accuracy: %.4f', sess.run(accuracy))
#Now we want to visualize the last batch's images just to see what our model has predicted
raw_images, labels, predictions = sess.run(
[raw_images, labels, predictions])
for i in range(10):
image, label, prediction = raw_images[i], labels[
i], predictions[i]
prediction_name, label_name = dataset.labels_to_name[
prediction], dataset.labels_to_name[label]
text = 'Prediction: %s \n Ground Truth: %s' % (prediction_name,
label_name)
img_plot = plt.imshow(image)
#Set up the plot and hide axes
plt.title(text)
img_plot.axes.get_yaxis().set_ticks([])
img_plot.axes.get_xaxis().set_ticks([])
plt.show()
logging.info(
'Model evaluation has completed! Visit TensorBoard for more information regarding your evaluation.'
)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# vocabulary options
parser.add_argument("--vocab_path",
type=str,
default="./data/vocab.txt",
help="Path of the vocabulary file.")
# model configuration options
parser.add_argument("--hidden_size",
type=int,
default=768,
help="Size of the hidden states.")
parser.add_argument("--intermediate_size",
type=int,
default=3072,
help="Size of the intermediate layer.")
parser.add_argument("--max_sequence_length",
type=int,
default=512,
help="Maximum length of table string.")
parser.add_argument("--num_attention_heads",
type=int,
default=12,
help="Number of the atttention heads.")
parser.add_argument("--num_hidden_layers",
type=int,
default=12,
help="Number of the hidden layers.")
parser.add_argument("--hidden_dropout_prob",
type=int,
default=0.1,
help="Dropout probability for hidden layers.")
parser.add_argument("--attention_dropout_prob",
type=int,
default=0.1,
help="Dropout probability for attention.")
parser.add_argument("--layer_norm_eps", type=float, default=1e-6)
# training options
parser.add_argument("--batch_size",
type=int,
default=32,
help="Size of the input batch.")
parser.add_argument("--seq_length",
type=int,
default=128,
help="Length of pre-processed sequences.")
parser.add_argument("--topk",
type=int,
default=10,
help="Top k choice for cloze tasks.")
# io options
parser.add_argument("--data_path",
type=str,
default="./data/knowledge/fact.txt")
parser.add_argument("--pretrained_model_path",
type=str,
default=None,
help="Path of the pretrained model.")
# fine-tune options.
parser.add_argument("--device_id",
type=int,
default=None,
help="Single GPU assignment.")
parser.add_argument("--target",
choices=["bert", "mlm", "sbo", "sop"],
default="bert")
args = parser.parse_args()
tokenizer = CKETokenizer(args.vocab_path)
args.vocab_size = len(tokenizer.vocab)
# Model initialization
model = MODELS[args.target](args)
# model.init_weights(model_path=args.pretrained_model_path)
predictor = ClozePreds[args.target](model)
if args.device_id is not None:
device = predictor.set_device(do_report=True)
else:
device = torch.device("cpu")
dataset = create_dataset(args.data_path,
tokenizer,
seq_length=args.seq_length)
query_ids = torch.LongTensor([sample[0] for sample in dataset])
token_type_ids = torch.LongTensor([sample[1] for sample in dataset])
attn_mask = torch.LongTensor([sample[2] for sample in dataset])
mlm_labels = torch.LongTensor([sample[3] for sample in dataset])
instances_num = len(dataset)
print("The number of evaluation instances: ", instances_num)
predictor.eval()
corrects = 0.
for i, (query_batch, type_batch, attn_batch, label_batch) in enumerate(
load_batch(args.batch_size, query_ids, token_type_ids, attn_mask,
mlm_labels)):
with torch.no_grad():
correct, _, _ = predictor(query_batch.to(device),
type_batch.to(device),
attn_batch.to(device),
label_batch.to(device),
topk=args.topk)
corrects += correct.item()
print("{} right predictions out of {} instances".format(
corrects, instances_num))
acc = corrects / instances_num
print("Correct rate: {:.4f}\n".format(acc))
def main(_):
data_tr, labels_tr, data_te, labels_te, unlabeled = input_data.load_mnist(
num_labeled=FLAGS.num_labeled)
print(" train shapes:", data_tr.shape, labels_tr.shape)
print(" test shapes:", data_te.shape, labels_te.shape)
print("unlabeled shapes:", unlabeled.shape)
data_tr_batch, labels_tr_batch = u.load_shuffle_batch(
data_tr,
labels_tr,
batch_size=FLAGS.batch_size,
capacity=FLAGS.batch_size * 100,
min_after_dequeue=FLAGS.batch_size * 20)
data_te_batch, labels_te_batch = u.load_batch(data_te, labels_te,
FLAGS.batch_size)
with tf.variable_scope('model') as scope:
logits_tr = models.mnist_supervised_rasmus(data_tr_batch,
is_training=True)
scope.reuse_variables()
logits_te = models.mnist_supervised_rasmus(data_te_batch,
is_training=False)
loss_tr = u.get_supervised_loss(logits=logits_tr, labels=labels_tr_batch)
loss_te = u.get_supervised_loss(logits=logits_te, labels=labels_te_batch)
acc_tr = u.get_accuracy(logits_tr, labels_tr_batch)
acc_te = u.get_accuracy(logits_te, labels_te_batch)
step = tf.Variable(0, trainable=False, dtype=tf.int32)
optimizer = u.get_adam_rasmus(step=step,
learning_rate=FLAGS.learning_rate,
num_total_iters=FLAGS.num_iters,
decay_first=FLAGS.decay_first)
train_op = u.get_train_op(optimizer, loss_tr, step)
with tf.Session() as sess:
def eval_test():
loss = 0.0
acc = 0.0
eval_iters = int(data_te.shape[0] / FLAGS.batch_size)
for j in range(eval_iters):
l, a = sess.run([loss_te, acc_te])
loss += l
acc += a
loss /= eval_iters
acc /= eval_iters
return loss, acc
# initialize the variables
init_op = tf.global_variables_initializer()
sess.run(init_op)
# initialize the queue threads to start to shovel data
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for i in tqdm(range(FLAGS.num_iters)):
_, cur_loss_tr, cur_acc_tr = sess.run([train_op, loss_tr, acc_tr])
if FLAGS.eval_interval is not None and i % FLAGS.eval_interval == 0:
print('train loss: %.4f train acc: %.4f' %
(cur_loss_tr, cur_acc_tr))
cur_loss_te, cur_acc_te = eval_test()
print(' test loss: %.4f test acc: %.4f' %
(cur_loss_te, cur_acc_te))
print('\nOPTIMIZATION FINISHED!')
final_loss_te, final_acc_te = eval_test()
print('FINAL TEST LOSS: %.4f FINAL TEST ACC: %.4f' %
(final_loss_te, final_acc_te))
# stop our queue threads and properly close the session
coord.request_stop()
coord.join(threads)
sess.close()
def main(_):
os.environ["CUDA_VISIBLE_DEVICES"] = str(FLAGS.gpu_id)
FLAGS.num_preprocessing_threads = 10
FLAGS.max_epoch_num = 100
FLAGS.train_split_name = 'train'
FLAGS.test_split_name = 'test'
FLAGS.model_name = 'inception_v3'
FLAGS.dataset_dir = 'datasets'
FLAGS.attr2class_file = os.path.join(FLAGS.dataset_dir, 'attr2class.txt')
FLAGS.train_dir = 'output'
FLAGS.checkpoint_path = os.path.join('pretrained_models',
'%s.ckpt' % (FLAGS.model_name))
log_file_path = os.path.join(FLAGS.train_dir, 'log')
if not os.path.isdir(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
FLAGS.checkpoint_exclude_scopes = 'InceptionV3/Logits,InceptionV3/AuxLogits,VAE'
FLAGS.trainable_scopes = 'VAE' # if learning all parameters including CNN, set FLAGS.trainable_scopes=None
FLAGS.checkpoint_exclude_keywords = None
FLAGS.batch_size = 64
with tf.Graph().as_default():
# load dataset
dataset = dataset_factory.get_dataset(FLAGS.train_split_name,
FLAGS.dataset_dir)
test_dataset = dataset_factory.get_dataset(FLAGS.test_split_name,
FLAGS.dataset_dir)
num_batches = int(
math.ceil(dataset.num_samples / float(FLAGS.batch_size)))
num_test_batches = int(
math.ceil(test_dataset.num_samples / float(FLAGS.batch_size)))
train_image_size = nets_factory.get_network_fn(
FLAGS.model_name, dataset.num_classes).default_image_size
images, labels = load_batch(FLAGS,
dataset,
train_image_size,
train_image_size,
is_training=True)
test_images, test_labels = load_batch(FLAGS,
test_dataset,
train_image_size,
train_image_size,
is_training=False)
# load class attributes
attr2class = np.loadtxt(FLAGS.attr2class_file, np.float32)
# build networks
train_batch_loss, train_summary = WSCI_network('train',
FLAGS.model_name,
images, labels,
attr2class, False,
FLAGS.beta)
test_correct_arr = WSCI_network('test', FLAGS.model_name, test_images,
test_labels, attr2class, True)
# optimizer
global_step = slim.create_global_step()
config_lr = configure_learning_rate(FLAGS, dataset.num_samples,
global_step)
optimizer = configure_optimizer(learning_rate=config_lr)
variables_to_train = get_variables_to_train(FLAGS)
grads_and_vars = optimizer.compute_gradients(train_batch_loss,
variables_to_train)
update_ops = [
optimizer.apply_gradients(grads_and_vars, global_step=global_step)
]
# update moving_mean and moving_variance for batch normalization
# update_ops += tf.get_collection(tf.GraphKeys.UPDATE_OPS)
update_op = tf.group(*update_ops)
with tf.control_dependencies([update_op]):
train_op = tf.identity(train_batch_loss)
# main code
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
# initialization
sess.run(tf.global_variables_initializer())
with slim.queues.QueueRunners(sess):
# initialization
iepoch = 0
init_fn = get_init_fn(FLAGS.checkpoint_path,
FLAGS.checkpoint_exclude_scopes,
FLAGS.checkpoint_exclude_keywords)
init_fn(sess)
while iepoch < FLAGS.max_epoch_num:
#training
for ibatch in range(num_batches):
print 'iepoch %d: train %d/%d' % (iepoch, ibatch,
num_batches)
sess.run([train_op, train_summary])
# test
correct_num = 0
for ibatch in range(num_test_batches):
print 'iepoch %d: test %d/%d' % (iepoch, ibatch,
num_test_batches)
correct_arr = sess.run(test_correct_arr)
correct_num += np.sum(correct_arr)
test_acc = float(correct_num) / (num_test_batches *
FLAGS.batch_size)
fid = open(log_file_path, 'a+')
fid.write('%d %f\n' % (iepoch, test_acc))
fid.close()
train_x, train_t, test_x, test_t = utils.divide_train_test(data, label,
fold*test_size,
fold*test_size+test_size)
else:
train_x, train_t, test_x, test_t = utils.divide_train_test(data, label,
data.shape[0]-test_size,
data.shape[0])
print('present fold train shape: ', train_x.shape)
print('present fold test shape: ', test_x.shape)
max_fold_acc = 0
for epoch in range(FLAGS.num_epochs):
train_loss = 0
train_acc = 0
batch_num = 0
for i in range(0, train_size, FLAGS.batch_size):
x_batch, t_batch = utils.load_batch(train_x, train_t, FLAGS.batch_size)
loss, acc, pred = net.train(x_batch, t_batch, FLAGS.learning_rate, FLAGS.momentum)
batch_num += 1
train_loss += loss
train_acc += acc
if batch_num % 4 == 0:
print('training loss {:.4f}'.format(loss))
print('batch accuracy {:.4f}'.format(acc))
print('epoch train loss: ', train_loss/batch_num)
print('epoch train acc: ', train_acc/batch_num)
eva_acc, eva_pred = net.evaluate(test_x, test_t)
print('epoch'+str(epoch)+': ', eva_acc)
if eva_acc > max_fold_acc:
max_fold_acc = eva_acc
def main(_):
data_tr, labels_tr, data_te, labels_te, unlabeled = input_data.load_data(
FLAGS.dataset_name, FLAGS.num_labeled)
print(" train shapes:", data_tr.shape, labels_tr.shape)
print(" test shapes:", data_te.shape, labels_te.shape)
print("unlabeled shapes:", unlabeled.shape)
data_tr_batch, labels_tr_batch = u.load_shuffle_batch(
data_tr,
labels_tr,
batch_size=FLAGS.batch_size,
capacity=FLAGS.batch_size * 100,
min_after_dequeue=FLAGS.batch_size * 20)
data_te_batch, labels_te_batch = u.load_batch(data_te, labels_te,
FLAGS.batch_size)
data_unlabeled_batch, _ = u.load_batch(unlabeled,
np.zeros(unlabeled.shape[0]),
FLAGS.batch_size)
with tf.variable_scope('model') as scope:
model = models.get_model(FLAGS.model_name)
logits_tr = model(data_tr_batch, is_training=True)
scope.reuse_variables()
logits_te = model(data_te_batch, is_training=False)
loss_tr = u.get_supervised_loss(logits=logits_tr, labels=labels_tr_batch)
loss_te = u.get_supervised_loss(logits=logits_te, labels=labels_te_batch)
acc_tr = u.get_accuracy(logits_tr, labels_tr_batch)
acc_te = u.get_accuracy(logits_te, labels_te_batch)
step = tf.Variable(0, trainable=False, dtype=tf.int32)
optimizer = u.get_optimizer(FLAGS.optimizer_type, FLAGS.learning_rate,
step, FLAGS.lr_decay_steps,
FLAGS.lr_decay_factor)
train_op = u.get_train_op(optimizer, loss_tr, step)
with tf.Session() as sess:
def eval_test():
loss = 0.0
acc = 0.0
eval_iters = int(data_te.shape[0] / FLAGS.batch_size)
for j in range(eval_iters):
l, a = sess.run([loss_te, acc_te])
loss += l
acc += a
loss /= eval_iters
acc /= eval_iters
return loss, acc
# initialize the variables
init_op = tf.global_variables_initializer()
sess.run(init_op)
# initialize the queue threads to start to shovel data
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for i in tqdm(range(FLAGS.num_iters)):
_, cur_loss_tr, cur_acc_tr = sess.run([train_op, loss_tr, acc_tr])
if i % FLAGS.eval_interval == 0:
print('train loss: %.4f train acc: %.4f' %
(cur_loss_tr, cur_acc_tr))
cur_loss_te, cur_acc_te = eval_test()
print(' test loss: %.4f test acc: %.4f' %
(cur_loss_te, cur_acc_te))
# stop our queue threads and properly close the session
coord.request_stop()
coord.join(threads)
sess.close()
def main():
is_training = tf.placeholder(tf.bool, name='is_training')
num_class = args.num_way
num_shot = args.num_shot
num_query = args.num_query
keep_prob = tf.placeholder(tf.float32, name='keep_prob')
support_label = tf.placeholder(tf.int32, (None, ), 'support_label')
query_label = tf.placeholder(tf.int32, (None, ), 'query_label')
support_x = tf.placeholder(tf.float32, (None, 640), 'support_x')
query_x = tf.placeholder(tf.float32, (None, 640), 'query_x')
support_feature = support_x
query_feature = query_x
support_feature = tf.reshape(support_feature,
(batch_size, num_class, num_shot, 640))
query_feature = tf.reshape(query_feature,
(batch_size, num_class, num_query, 640))
support_label_reshape = tf.reshape(support_label,
(batch_size, num_class, num_shot))
query_label_reshape = tf.reshape(query_label,
(batch_size, num_class, num_query))
awgim = model.AWGIM(args, keep_prob, is_training)
loss_cls, accuracy, tr_loss, tr_accuracy, support_reconstruction, query_reconstruction = \
awgim.forward(support_feature, support_label_reshape, query_feature, query_label_reshape)
reg_term = tf.add_n([
tf.nn.l2_loss(v) for v in tf.trainable_variables()
if 'kernel' in v.name
])
loss_meta = loss_cls + args.alpha_1 * tr_loss + args.alpha_2 * support_reconstruction + args.alpha_3 * query_reconstruction
Batch = tf.Variable(0,
trainable=False,
dtype=tf.float32,
name='global_step')
learning_rate = tf.train.exponential_decay(
learning_rate=args.learning_rate,
global_step=Batch,
decay_steps=args.step_size,
decay_rate=0.2,
staircase=True)
optim = tf.contrib.opt.AdamWOptimizer(learning_rate=learning_rate,
weight_decay=args.weight_decay)
meta_weights = [v for v in tf.trainable_variables()]
print(meta_weights)
if args.stage == 'train':
meta_gradients = utils.grads_and_vars(loss_meta, meta_weights,
reg_term)
meta_gradients = utils.clip_gradients(meta_gradients,
args.gradient_threshold,
args.gradient_norm_threshold)
train_op = optim.apply_gradients(zip(meta_gradients, meta_weights),
global_step=Batch)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
save_path = utils.save(args)
print(save_path)
os.makedirs(save_path, exist_ok=True)
if args.stage == 'test':
print(tf.train.latest_checkpoint(save_path))
saver.restore(sess, tf.train.latest_checkpoint(save_path))
print('load model')
if args.data_set == 'mini':
loader_train = dataset_mini.dataset_mini('train', args)
loader_val = dataset_mini.dataset_mini('val', args)
loader_test = dataset_mini.dataset_mini('test', args)
else:
loader_train = dataset_tiered.dataset_tiered('train', args)
loader_val = dataset_tiered.dataset_tiered('val', args)
loader_test = dataset_tiered.dataset_tiered('test', args)
if args.stage == 'train':
print('Load PKL data')
loader_train.load_data_pkl()
loader_val.load_data_pkl()
else:
loader_test.load_data_pkl()
val_best_accuracy = 0.
n_iter = 0
record_val_acc = []
if args.stage == 'train':
for epoch in range(args.epoch):
training_accuracy, training_loss, acc_cp, acc_real, c_loss, d_loss, g_loss = [], [], [], [], [], [], []
# training_loss_cls = []
for epi in range(100):
support_input, s_labels, query_input, q_labels = utils.load_batch(
args, loader_train, args.batch_size, True, loader_val)
feed_dict = {
support_x: support_input,
support_label: s_labels,
query_x: query_input,
query_label: q_labels,
is_training: True,
keep_prob: 1. - args.dropout
}
outs = sess.run([train_op, loss_meta, accuracy, Batch],
feed_dict=feed_dict)
training_accuracy.append(outs[2])
training_loss.append(outs[1])
n_iter += 1
if (epoch + 1) % 3 == 0:
log = 'epoch: ', epoch + 1, 'accuracy: ', np.mean(
training_accuracy), 'loss: ', np.mean(training_loss)
print(log)
if (epoch + 1) % 3 == 0:
accuracy_val = []
loss_val = []
for epi in range(100):
support_input, s_labels, query_input, q_labels = utils.load_batch(
args, loader_val, args.batch_size, training=False)
outs = sess.run(
[loss_meta, accuracy, Batch],
feed_dict={
support_x: support_input,
support_label: s_labels,
query_x: query_input,
query_label: q_labels,
is_training: False,
keep_prob: 1.
})
accuracy_val.append(outs[1])
loss_val.append(outs[0])
mean_acc = np.mean(accuracy_val)
std_acc = np.std(accuracy_val)
ci95 = 1.96 * std_acc / np.sqrt(100)
print(
' Val Acc:{:.4f},std:{:.4f},ci95:{:.4f}'.format(
mean_acc, std_acc, ci95), 'at epoch: ', epoch + 1)
record_val_acc.append(mean_acc)
if mean_acc > val_best_accuracy:
val_best_accuracy = mean_acc
saver.save(sess,
save_path=save_path + 'model.ckpt',
global_step=Batch)
if (epoch + 1) % 100 == 0:
saver.save(sess,
save_path=save_path + 'model.ckpt',
global_step=Batch)
elif args.stage == 'test':
accuracy_test = []
loss_test = []
num = 600
for epi in range(num):
support_input, s_labels, query_input, q_labels = utils.load_batch(
args, loader_test, args.batch_size, False)
outs = sess.run(
[loss_meta, accuracy],
feed_dict={
support_x: support_input,
support_label: s_labels,
query_x: query_input,
query_label: q_labels,
is_training: False,
keep_prob: 1.
})
accuracy_test.append(outs[1])
loss_test.append(outs[0])
mean_acc = np.mean(accuracy_test)
std_acc = np.std(accuracy_test)
ci95 = 1.96 * std_acc / np.sqrt(num)
print('Acc:{:.4f},std:{:.4f},ci95:{:.4f}'.format(
mean_acc, std_acc, ci95))
sess.close()
def train(self):
logging.info('Training...')
best_hr = [0.]
best_ndcg = [0.]
# file to store the generated topic words
if os.path.exists('res/topics_' + self.dataset + '.txt'):
os.remove('res/topics_' + self.dataset + '.txt')
logging.info('Successfully remove existing topic file!')
batch_num = int(len(self.train_pairs) / self.batch_size) + 1
for epoch in range(self.max_epoch):
if (epoch + 1) % 20 == 0:
self.init_lr = self.init_lr * self.lr_decay
logging.info('Training at epoch ' + str(epoch + 1) + ' ...')
loss_total, gen_loss_total, latent_loss_total, reg_loss_total, cf_loss_total = 0., 0., 0., 0., 0.
for batch in range(batch_num):
batch_data, batch_labels = utils.load_batch(
self.train_pairs, self.train_labels, batch,
self.batch_size)
batch_data = np.transpose(batch_data)
docu1 = batch_data[0]
docu2 = batch_data[1]
docus = np.concatenate((docu1, docu2), axis=0)
get_emb = [
list(range(len(docu1))),
list(range(len(docu1),
len(docu1) + len(docu2)))
]
feed_dict = {
self.batch_data: self.doc_contents[docus],
self.batch_labels: np.array(batch_labels),
self.learning_rate: self.init_lr,
self.keep_prob: 1.,
self.get_emb: get_emb
}
_, loss_tmp, gen_loss_tmp, latent_loss_tmp, reg_loss_tmp, cf_loss_tmp = self.sess.run(
(self.train_op, self.loss, self.gen_loss, self.latent_loss,
self.reg_loss, self.cf_loss),
feed_dict=feed_dict)
loss_total += loss_tmp
gen_loss_total += gen_loss_tmp
latent_loss_total += latent_loss_tmp
reg_loss_total += reg_loss_tmp
cf_loss_total += cf_loss_tmp
if (epoch + 1) % self.trained_print_step == 0:
logging.info(
'Epoch {0}: avg batch loss = {1}, gen loss = {2}, latent loss = {3}, reg loss = {4}, cf loss = {5}\n'
.format(epoch + 1, loss_total / batch_num,
gen_loss_total / batch_num,
latent_loss_total / batch_num,
reg_loss_total / batch_num,
1000. * cf_loss_total / batch_num))
if (epoch + 1) % self.test_step == 0:
logging.info('Testing at epoch ' + str(epoch + 1) + ' ...')
z_test = self.sess.run(self.z,
feed_dict={
self.batch_data: self.doc_contents,
self.keep_prob: 1.0
})
feed_dict = {self.z_test: z_test, self.keep_prob: 1.0}
# ave_rank, ave_auc = self._auc_test(feed_dict)
# logging.info('ave rank = ' + str(ave_rank) + ', ave auc = ' + str(ave_auc) + '\n')
hits, ndcgs = self._hit_test(feed_dict)
logging.info('HR = ' + str(hits))
logging.info('NDCGS = ' + str(ndcgs) + '\n')
if best_hr[-1] < hits[-1]:
best_hr = hits
if best_ndcg[-1] < ndcgs[-1]:
best_ndcg = ndcgs
if (epoch + 1) % self.print_words_step == 0:
utils.print_top_words(self.sess.run(self.weights_words),
self.vocab, self.dataset)
logging.info('BEST HR = ' + str(best_hr))
logging.info('BEST NDCGS = ' + str(best_ndcg) + '\n\n\n')
ax4.set_xlabel('Coupling Strength', fontsize=fs)
for ax in axes.flatten():
ax.tick_params(labelsize=fs)
handles = []
labels = []
ic = ''
for i, arg in enumerate(sys.argv):
if arg.startswith("--step="):
step = int(sys.argv.pop(i).split('=')[1])
break
else:
step = 1
colors = _get_colors(int((len(sys.argv) - 1) / step))
for i, filename in enumerate(sys.argv[1:][::step]):
header, data = load_batch(filename)
if not ic:
ic = header['initial_condition']
elif ic != header['initial_condition']:
ic = 'mixed'
N, K = header['N'], header['K']
data['chi_r'] = data['chi_r'] * N
data['chi_psi'] = data['chi_psi'] * N
print(header)
plotkwargs = dict(color=colors[i], marker=ms(i), mfc='w', ms=fs * 0.5)
plot_batch(axes, data, ylog=True, **plotkwargs)
handles += [Line2D([], [], **plotkwargs)]
alpha = K / N
if header['p']:
BATCHSIZE = 32
BATCHES_PER_EPOCH = 1000
epoch = 0
while epoch < 30:
loss_cur_epoch = 0
batch_counter = 0
for _ in range(BATCHES_PER_EPOCH):
# Load a batch of training data
x_batch, y_batch = utils.load_batch(dataset_npy,
batch_size=BATCHSIZE,
augmentation=True)
# feed batches and calculate loss
_, loss_this_batch = sess.run(fetches=[train_step, loss],
feed_dict={
x: x_batch,
y: y_batch,
keep_prob: 0.5
})
loss_cur_epoch += loss_this_batch
batch_counter += 1
if batch_counter % 200 == 0:
print("BATCH {} / 1000".format(batch_counter), end='\r')
loss_cur_epoch /= BATCHES_PER_EPOCH
import numpy as np
import utils
import utils.load_batch
from utils.load_batch import load_batch
from utils.load_batch import cifar10_DataLoader
from utils.handle_data import data_split
from utils.handle_data import get_all_train_data
from utils.preprocess import StandardScaler
from clsr.nn_2l import TwoLayerNeuralNetwork
if __name__ == "__main__":
merged_data = get_all_train_data("cifar-10-batches-py")
test_data = load_batch("cifar-10-batches-py/test_batch")
loader = cifar10_DataLoader(test_data, batch_size=100)
cnt = 0
for inputs, labels in loader:
cnt += labels.shape[0]
print(cnt)
cnt = 0
for inputs, labels in loader:
cnt += labels.shape[0]
print(cnt)
def main(_):
data_tr, labels_tr, data_te, labels_te, unlabeled = input_data.load_cifar10(
num_labeled=FLAGS.num_labeled)
print(" train shapes:", data_tr.shape, labels_tr.shape)
print(" test shapes:", data_te.shape, labels_te.shape)
print("unlabeled shapes:", unlabeled.shape)
data_tr_batch, labels_tr_batch = u.load_shuffle_batch(
data_tr,
labels_tr,
batch_size=FLAGS.batch_size,
capacity=FLAGS.batch_size * 100,
min_after_dequeue=FLAGS.batch_size * 20)
data_te_batch, labels_te_batch = u.load_batch(data_te, labels_te,
FLAGS.batch_size)
unlabeled_batch, _ = u.load_shuffle_batch(
unlabeled,
unlabeled,
batch_size=FLAGS.batch_size,
capacity=FLAGS.batch_size * 100,
min_after_dequeue=FLAGS.batch_size * 20)
bn_decay = 0.9
ema = tf.train.ExponentialMovingAverage(decay=bn_decay)
bn_assigns = []
logits_tr, _, _, _ = models.cifar10_gamma(data_tr_batch,
is_training=True,
is_unlabeled=False,
ema=ema,
bn_assigns=bn_assigns,
batch_norm_decay=bn_decay,
noise_std=0.3)
# _, _, crt, cln = models.cifar10_gamma(unlabeled_batch, is_training=False, is_unlabeled=True,
# ema=ema, bn_assigns=bn_assigns, batch_norm_decay=bn_decay, noise_std=0.3)
_, logits_te, _, _ = models.cifar10_gamma(data_te_batch,
is_training=False,
is_unlabeled=False,
ema=ema,
bn_assigns=bn_assigns,
batch_norm_decay=bn_decay,
noise_std=0.0)
loss_tr = u.get_supervised_loss(logits=logits_tr, labels=labels_tr_batch)
# loss_tr = u.get_supervised_loss(logits=logits_tr, labels=labels_tr_batch) + u.get_denoising_loss(crt, cln, 4.0)
loss_te = u.get_supervised_loss(logits=logits_te, labels=labels_te_batch)
acc_tr = u.get_accuracy(logits_tr, labels_tr_batch)
acc_te = u.get_accuracy(logits_te, labels_te_batch)
step = tf.Variable(0, trainable=False, dtype=tf.int32)
optimizer = u.get_adam_rasmus(step=step,
learning_rate=FLAGS.learning_rate,
num_total_iters=FLAGS.num_iters,
decay_first=FLAGS.decay_first)
train_op = u.get_train_op(optimizer, loss_tr, step, bn_assigns=bn_assigns)
with tf.Session() as sess:
def eval_test():
loss = 0.0
acc = 0.0
eval_iters = int(data_te.shape[0] / FLAGS.batch_size)
for j in range(eval_iters):
l, a = sess.run([loss_te, acc_te])
loss += l
acc += a
loss /= eval_iters
acc /= eval_iters
return loss, acc
# initialize the variables
init_op = tf.global_variables_initializer()
sess.run(init_op)
# initialize the queue threads to start to shovel data
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
print('\nOPTIMIZATION STARTED!')
for i in tqdm(range(FLAGS.num_iters)):
_, cur_loss_tr, cur_acc_tr = sess.run([train_op, loss_tr, acc_tr])
if FLAGS.eval_interval is not None and i % FLAGS.eval_interval == 0:
print('train loss: %.4f train acc: %.4f' %
(cur_loss_tr, cur_acc_tr))
cur_loss_te, cur_acc_te = eval_test()
print(' test loss: %.4f test acc: %.4f' %
(cur_loss_te, cur_acc_te))
print('\nOPTIMIZATION FINISHED!')
final_loss_te, final_acc_te = eval_test()
print('FINAL TEST LOSS: %.4f FINAL TEST ACC: %.4f' %
(final_loss_te, final_acc_te))
# stop our queue threads and properly close the session
coord.request_stop()
coord.join(threads)
sess.close()
