def validate(model_def):
"""
Validate my alexnet implementation
Args:
model_def: the model class/definition
"""
img_dir = os.path.join('.', 'images')
images = []
print "loading images ..."
files = fnmatch.filter(os.listdir(img_dir), '*.jpeg')
for f in files:
print "> " + f
img_file = tf.read_file(os.path.join(img_dir, f))
img_decoded = tf.image.decode_jpeg(img_file, channels=3)
img_processed = model_def.image_prep.preprocess_image(
image=img_decoded,
output_height=model_def.image_size,
output_width=model_def.image_size,
is_training=False)
images.append(img_processed)
# create TensorFlow Iterator object
images = Dataset.from_tensors(images)
iterator = Iterator.from_structure(images.output_types,
images.output_shapes)
next_element = iterator.get_next()
iterator_init_op = iterator.make_initializer(images)
# create the model and get scores (pipe to softmax)
model = model_def(next_element)
scores = model.get_final_op()
softmax = tf.nn.softmax(scores)
print 'start validation ...'
with tf.Session() as sess:
# Initialize all variables and the iterator
sess.run(tf.global_variables_initializer())
sess.run(iterator_init_op)
# Load the pretrained weights into the model
model.load_initial_weights(sess)
# run the graph
probs = sess.run(softmax)
if model_def is ResNetV2:
probs = prep_resnet_results(probs)
# sometime we have an offset
offset = len(class_names) - len(probs[0])
# print the results
for prob in probs:
best_index = np.argmax(prob)
print "> " + class_names[best_index +
offset] + " -> %.4f" % prob[best_index]
def train(self, Datasize=None):
with tf.name_scope("Dataset"):
fin = int(min(self.cut, Datasize) if Datasize else self.cut)
choices = self.train_choices[:fin]
dat = (Dataset.from_tensors([
d[choices, :].astype(np.float32) for d in [self.X, self.Y]
]).make_initializable_iterator())
tf.add_to_collection("batch_init", dat.initializer)
return dat.get_next("bs_dat")
def split_multi_string(self, x):
"""
Splits a list of strings on whitespaces and casts them to int
:param x: list of sequences
:return: list of int tensors
"""
# Split best answers on comma
n_best_answers = tf.string_split([x], delimiter=",")
# Reformat data to sparse tensor
n_best_answers = tf.SparseTensorValue(indices=n_best_answers.indices,
values=tf.string_split(n_best_answers.values),
dense_shape=n_best_answers.dense_shape)
# Get data as sparse tensor
up_ba = n_best_answers.values
# Sparse tensor to dense Tensor with padding '0'
up_ba = tf.sparse_to_dense(up_ba.indices, (up_ba.dense_shape[0], up_ba.dense_shape[1] + 1), up_ba.values, '0')
# If n_best_answer not empty, convert every answer to int
up_ba = tf.cond(tf.greater(tf.size(up_ba), 0),
lambda: tf.map_fn(lambda s: tf.string_to_number(s, out_type=tf.int32), up_ba, dtype=tf.int32),
lambda: tf.string_to_number(up_ba, out_type=tf.int32))
# Get length of all sequences
seq_len = tf.argmin(tf.to_int32(tf.not_equal(up_ba, 0)), axis=1)
# Create filter mask
idx = tf.less_equal(seq_len, self.max_seq_len - 1)
# Filter sequences and length
up_ba = tf.boolean_mask(up_ba, idx)
up_ba = up_ba[:, 0:tf.cond(tf.greater(tf.size(up_ba), 0), lambda: self.max_seq_len, lambda: 0)]
up_ba = tf.pad(up_ba, [[0, 0], [0, self.max_seq_len - tf.shape(up_ba)[1]]])
seq_len = tf.boolean_mask(seq_len, idx) + 1
# Make datasets
sequences_nba = Dataset.from_tensors(up_ba)
len_nba = Dataset.from_tensors(seq_len)
return Dataset.zip((sequences_nba, len_nba))
def create_dataset(batch_size):
files, labels = list_files_and_labels()
files_const = tf.constant(files)
labels_const = tf.one_hot(tf.constant(labels), depth=10)
dataset = Dataset.from_tensor_slices((files_const, labels_const))
dataset = dataset.interleave(lambda filename, label: Dataset.from_tensors(
(filename, label)).map(_parse_function, num_threads=1),
cycle_length=10)
# dataset = dataset.shuffle(buffer_size=10000)
dataset = dataset.batch(batch_size)
return dataset
def process_target(self, sub, len_sub, cont, len_cont, nba, nba_len):
"""
Processes target sequence with setting GO-Symbol in front of target input sequence and EOS-Symbol at the end
of target output sequence
:param sub: subject sequences
:param len_sub: length of subject sequences
:param cont: content sequences
:param len_cont: length of content sequences
:param nba: n best answers sequences
:param nba_len: length of n best answers
:return: Dataset with processed target input and output
"""
def set_eos(nba):
"""
Receives one answer out of all answers and sets an EOS-Symbol at the end
:param nba: One answer out of n best answers
:return: Answer with following EOS-Symbol
"""
len_nba = tf.argmin(nba, output_type=tf.int32)
nba = tf.concat([nba[:len_nba], [3], nba[len_nba + 1:]], axis=0)
return nba
# Set number 2 (GO) at the beginning of the sequences
target_input = tf.concat([tf.fill((tf.size(nba_len), 1), 2), nba[:, :-1]], axis=1)
# Set number 3 (EOS) at the end of the sequences
target_output = tf.map_fn(set_eos, nba, dtype=tf.int32)
# Convert function input back to dataset
sub = Dataset.from_tensors(sub)
len_sub = Dataset.from_tensors(len_sub)
cont = Dataset.from_tensors(cont)
len_cont = Dataset.from_tensors(len_cont)
target_input = Dataset.from_tensors(target_input)
target_output = Dataset.from_tensors(target_output)
len_nba = Dataset.from_tensors(nba_len)
return Dataset.zip((sub, len_sub, cont, len_cont, target_input, target_output, len_nba))
def get_seq_len_and_join_ba(self, sub, cont, nba_and_len):
"""
For getting length of subject sequences, content sequences and answer sequences
:param sub: subject sequences
:param cont: content sequences
:param nba_and_len: n best answer ans length
:return: Dataset with sequences and length
"""
# Seperate n best answers and length
nba = nba_and_len[0]
nba_len = nba_and_len[1]
# Count sequence length of subject and content
len_sub = Dataset.from_tensors(tf.size(sub))
len_cont = Dataset.from_tensors(tf.size(cont))
# Make dataset from tensors
sub = Dataset.from_tensors(sub)
cont = Dataset.from_tensors(cont)
nba = Dataset.from_tensors(nba)
nba_len = Dataset.from_tensors(nba_len)
return Dataset.zip((sub, len_sub, cont, len_cont, nba, nba_len))
def main():
"""Run the demo."""
data = fetch_gpml_sarcos_data()
Xr = data.train.data.astype(np.float32)
Yr = data.train.targets.astype(np.float32)[:, np.newaxis]
Xs = data.test.data.astype(np.float32)
Ys = data.test.targets.astype(np.float32)[:, np.newaxis]
N, D = Xr.shape
print("Iterations: {}".format(int(round(N * NEPOCHS / BATCH_SIZE))))
# Scale and centre the data, as per the original experiment
ss = StandardScaler()
Xr = ss.fit_transform(Xr)
Xs = ss.transform(Xs)
ym = Yr.mean()
Yr -= ym
Ys -= ym
# Training batches
data_tr = Dataset.from_tensor_slices({'X': Xr, 'Y': Yr}) \
.shuffle(buffer_size=1000) \
.batch(BATCH_SIZE)
# Testing iterators
data_ts = Dataset.from_tensors({'X': Xs, 'Y': Ys}).repeat()
with tf.name_scope("DataIterators"):
iterator = Iterator.from_structure(data_tr.output_types,
data_tr.output_shapes)
data = iterator.get_next()
training_init = iterator.make_initializer(data_tr)
testing_init = iterator.make_initializer(data_ts)
with tf.name_scope("Deepnet"):
phi, kl = net(X=data['X'])
std = tf.Variable(NOISE, name="noise")
lkhood = tf.distributions.Normal(phi, scale=ab.pos(std))
loss = ab.elbo(lkhood, data['Y'], N, kl)
tf.summary.scalar('loss', loss)
with tf.name_scope("Train"):
optimizer = tf.train.AdamOptimizer()
global_step = tf.train.create_global_step()
train = optimizer.minimize(loss, global_step=global_step)
with tf.name_scope("Test"):
r2 = rsquare(data['Y'], phi)
# Logging
log = tf.train.LoggingTensorHook(
{'step': global_step, 'loss': loss},
every_n_iter=1000
)
with tf.train.MonitoredTrainingSession(
config=CONFIG,
scaffold=tf.train.Scaffold(local_init_op=training_init),
checkpoint_dir="./sarcos/",
save_summaries_steps=None,
save_checkpoint_secs=20,
save_summaries_secs=20,
hooks=[log]
) as sess:
summary_writer = sess._hooks[1]._summary_writer
for i in range(NEPOCHS):
# Train for one epoch
try:
while not sess.should_stop():
sess.run(train)
except tf.errors.OutOfRangeError:
pass
# Init testing and assess and log R-square score on test set
sess.run(testing_init)
r2_score = sess.run(r2)
score_sum = tf.Summary(value=[
tf.Summary.Value(tag='r-square', simple_value=r2_score)
])
summary_writer.add_summary(score_sum, sess.run(global_step))
# Re-init training
sess.run(training_init)
# Prediction
sess.run(testing_init)
Ey = ab.predict_samples(phi, feed_dict=None, n_groups=NPREDICTSAMPLES,
session=sess)
sigma = sess.run(std)
r2_score = sess.run(r2)
# Score mean standardised log likelihood
Eymean = Ey.mean(axis=0)
Eyvar = Ey.var(axis=0) + sigma**2 # add sigma2 for obervation noise
snlp = msll(Ys.flatten(), Eymean, Eyvar, Yr.flatten())
print("------------")
print("r-square: {:.4f}, smse: {:.4f}, msll: {:.4f}."
.format(r2_score, 1 - r2_score, snlp))
def main(_):
FLAGS.eval_interval = 1000 # todo remove
if FLAGS.logdir is not None:
if FLAGS.taskid is not None:
FLAGS.logdir = FLAGS.logdir + '/t_' + str(FLAGS.taskid)
else:
FLAGS.logdir = FLAGS.logdir + '/t_' + str(random.randint(0,99999))
dataset_tools = import_module('tools.' + FLAGS.dataset)
NUM_LABELS = dataset_tools.NUM_LABELS
num_labels = NUM_LABELS
IMAGE_SHAPE = dataset_tools.IMAGE_SHAPE
image_shape = IMAGE_SHAPE
train_images, train_labels_svm = dataset_tools.get_data('train') # no train labels nowhere
test_images, test_labels = dataset_tools.get_data('test')
if FLAGS.zero_fact < 1:
# exclude a random set of zeros (not at the end, then there would be many batches without zeros)
keep = np.ones(len(train_labels_svm), dtype=bool)
zero_indices = np.where((train_labels_svm == 0))[0]
remove = np.random.uniform(0, 1, len(zero_indices))
zero_indices_to_remove = zero_indices[remove > FLAGS.zero_fact]
keep[zero_indices_to_remove] = False
train_images = train_images[keep]
train_labels_svm = train_labels_svm[keep]
print('using only a fraction of zeros, resulting in the following shape:', train_images.shape)
if FLAGS.num_unlabeled_images > 0:
unlabeled_train_images, _ = dataset_tools.get_data('unlabeled', max_num=np.min([FLAGS.num_unlabeled_images, 50000]))
train_images = np.vstack([train_images, unlabeled_train_images])
if FLAGS.normalize_input:
train_images = (train_images - 128.) / 128.
test_images = (test_images - 128.) / 128.
if FLAGS.use_test:
train_images = np.vstack([train_images, test_images])
train_labels_svm = np.hstack([train_labels_svm, test_labels])
#if FLAGS.dataset == 'svhn' and FLAGS.architecture == 'resnet_cifar_model':
# FLAGS.emb_size = 64
image_shape_crop = image_shape
c_test_imgs = test_images
c_train_imgs = train_images
# crop images to some random region. Intuitively, images should belong to the same cluster,
# even if a part of the image is missing
# (no padding, because the net could detect padding easily, and match it to other augmented samples that have
# padding)
if FLAGS.dataset == 'stl10':
image_shape_crop = [64, 64, 3]
c_test_imgs = test_images[:, 16:80, 16:80]
c_train_imgs = train_images[:, 16:80, 16:80]
def aug(image):
return apply_augmentation(image, target_shape=image_shape_crop, params=dataset_tools.augmentation_params)
def random_crop(image):
image_size = image_shape_crop[0]
image = tf.random_crop(image, [image_size, image_size, image_shape[2]])
return image
graph = tf.Graph()
with graph.as_default():
t_images = tf.placeholder("float", shape=[None] + image_shape)
dataset = Dataset.from_tensor_slices(t_images)
dataset = dataset.shuffle(buffer_size=10000, seed=47) # important, so that we have the same images in both sets
# parameters for buffering during augmentation. Only influence training speed.
nt = 8 if FLAGS.volta else 4 # that's not even enough, but there are no more CPUs
b = 10000
rf = FLAGS.num_augmented_samples
augmented_set = dataset
if FLAGS.shuffle_augmented_samples:
augmented_set = augmented_set.shuffle(buffer_size=10000, seed=47)
# get multiple augmented versions of the same image - they should later have similar embeddings
augmented_set = augmented_set.flat_map(lambda x: Dataset.from_tensors(x).repeat(rf))
augmented_set = augmented_set.map(aug, num_threads=nt, output_buffer_size=b)
dataset = dataset.map(random_crop, num_threads=1, output_buffer_size=b)
dataset = dataset.repeat().batch(FLAGS.unsup_batch_size)
augmented_set = augmented_set.repeat().batch(FLAGS.unsup_batch_size * rf)
iterator = dataset.make_initializable_iterator()
reg_iterator = augmented_set.make_initializable_iterator()
t_unsup_images = iterator.get_next()
t_reg_unsup_images = reg_iterator.get_next()
model_func = getattr(semisup.architectures, FLAGS.architecture)
model = semisup.SemisupModel(model_func, num_labels, image_shape_crop, optimizer='adam',
emb_size=FLAGS.emb_size,
dropout_keep_prob=FLAGS.dropout_keep_prob, num_blocks=FLAGS.num_blocks,
normalize_embeddings=FLAGS.normalize_embeddings, beta1=FLAGS.beta1,
beta2=FLAGS.beta2)
init_virt = []
for c in range(num_labels):
center = np.random.normal(0, 0.3, size=[1, FLAGS.emb_size])
noise = np.random.uniform(-0.01, 0.01, size=[FLAGS.virtual_embeddings_per_class, FLAGS.emb_size])
centroids = noise + center
init_virt.extend(centroids)
t_sup_emb = tf.Variable(tf.cast(np.array(init_virt), tf.float32), name="virtual_centroids")
t_sup_labels = tf.constant(
np.concatenate([[i] * FLAGS.virtual_embeddings_per_class for i in range(num_labels)]))
visit_weight = tf.placeholder("float", shape=[])
walker_weight = tf.placeholder("float", shape=[])
t_logit_weight = tf.placeholder("float", shape=[])
t_trafo_weight = tf.placeholder("float", shape=[])
t_l1_weight = tf.placeholder("float", shape=[])
t_norm_weight = tf.placeholder("float", shape=[])
t_learning_rate = tf.placeholder("float", shape=[])
t_sat_loss_weight = tf.placeholder("float", shape=[])
t_unsup_emb = model.image_to_embedding(t_unsup_images)
t_reg_unsup_emb = model.image_to_embedding(t_reg_unsup_images)
t_all_unsup_emb = tf.concat([t_unsup_emb, t_reg_unsup_emb], axis=0)
t_rsup_labels = tf.constant(np.concatenate([[i] * rf for i in range(FLAGS.unsup_batch_size)]))
rwalker_weight = tf.placeholder("float", shape=[])
rvisit_weight = tf.placeholder("float", shape=[])
if FLAGS.normalize_embeddings:
t_sup_logit = model.embedding_to_logit(tf.nn.l2_normalize(t_sup_emb, dim=1))
model.add_semisup_loss(
tf.nn.l2_normalize(t_sup_emb, dim=1), tf.nn.l2_normalize(t_unsup_emb, dim=1), t_sup_labels,
walker_weight=walker_weight, visit_weight=visit_weight,
match_scale=FLAGS.scale_match_ab)
model.reg_loss_aba = model.add_semisup_loss(
tf.nn.l2_normalize(t_reg_unsup_emb, dim=1), tf.nn.l2_normalize(t_unsup_emb, dim=1), t_rsup_labels,
walker_weight=rwalker_weight, visit_weight=rvisit_weight, match_scale=FLAGS.scale_match_ab,
est_err=False)
else:
t_sup_logit = model.embedding_to_logit(t_sup_emb)
model.add_semisup_loss(
t_sup_emb, t_unsup_emb, t_sup_labels,
walker_weight=walker_weight, visit_weight=visit_weight,
match_scale=FLAGS.scale_match_ab, est_err=True, name='c_association')
model.reg_loss_aba = model.add_semisup_loss(
t_reg_unsup_emb, t_unsup_emb, t_rsup_labels,
walker_weight=rwalker_weight, visit_weight=rvisit_weight, match_scale=FLAGS.scale_match_ab, est_err=False, name='aug_association')
model.add_logit_loss(t_sup_logit, t_sup_labels, weight=t_logit_weight)
t_reg_unsup_emb_singled = t_reg_unsup_emb[::FLAGS.num_augmented_samples]
t_unsup_logit = model.embedding_to_logit(t_unsup_emb)
t_reg_unsup_logit = model.embedding_to_logit(t_reg_unsup_emb_singled)
model.add_sat_loss(t_unsup_logit, t_reg_unsup_logit, weight=t_sat_loss_weight)
trafo_lc = semisup.NO_FC_COLLECTION if FLAGS.trafo_separate_loss_collection else semisup.LOSSES_COLLECTION
if FLAGS.trafo_weight > 0:
# only use a single augmented sample per sample
t_trafo_loss = model.add_transformation_loss(t_unsup_emb, t_reg_unsup_emb_singled, t_unsup_logit,
t_reg_unsup_logit, FLAGS.unsup_batch_size, weight=t_trafo_weight, label_smoothing=0, loss_collection=trafo_lc)
else:
t_trafo_loss = tf.constant(0)
model.add_emb_regularization(t_all_unsup_emb, weight=t_l1_weight)
model.add_emb_regularization(t_sup_emb, weight=t_l1_weight)
# make l2 norm = 3
model.add_emb_normalization(t_sup_emb, weight=t_norm_weight, target=FLAGS.norm_target)
model.add_emb_normalization(t_all_unsup_emb, weight=t_norm_weight, target=FLAGS.norm_target)
gradient_multipliers = {t_sup_emb: 1 }
[train_op, train_op_sat] = model.create_train_op(t_learning_rate, gradient_multipliers=gradient_multipliers)
summary_op = tf.summary.merge_all()
if FLAGS.logdir is not None:
summary_writer = tf.summary.FileWriter(FLAGS.logdir, graph)
saver = tf.train.Saver()
with tf.Session(graph=graph) as sess:
tf.global_variables_initializer().run()
sess.run(iterator.initializer, feed_dict={t_images: train_images})
sess.run(reg_iterator.initializer, feed_dict={t_images: train_images})
# optional: init from autoencoder
if FLAGS.restore_checkpoint is not None:
# logit fc layer cannot be restored
def is_main_net(x):
return 'logit_fc' not in x.name and 'Adam' not in x.name
variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='net')
variables = list(filter(is_main_net, variables))
restorer = tf.train.Saver(var_list=variables)
restorer.restore(sess, FLAGS.restore_checkpoint)
extra_feed_dict = {}
from numpy.linalg import norm
reg_warmup_steps = FLAGS.reg_warmup_steps
logit_weight_ = FLAGS.logit_weight
rwalker_weight_ = FLAGS.rwalker_weight
rvisit_weight_ = FLAGS.rvisit_weight
learning_rate_ = FLAGS.learning_rate
trafo_weight = FLAGS.trafo_weight
kmeans_initialized = False
for step in range(FLAGS.max_steps):
import time
start = time.time()
if FLAGS.init_with_kmeans:
if FLAGS.kmeans_sat_thresh is not None and not kmeans_initialized or \
FLAGS.kmeans_sat_thresh is None and step <= reg_warmup_steps:
walker_weight_ = 0
visit_weight_ = 0
logit_weight_ = 0
trafo_weight = 0
else:
walker_weight_ = FLAGS.walker_weight
visit_weight_ = FLAGS.visit_weight_base
logit_weight_ = FLAGS.logit_weight
trafo_weight = FLAGS.trafo_weight
else:
walker_weight_ = apply_envelope("log", step, FLAGS.walker_weight, reg_warmup_steps, 0)
visit_weight_ = apply_envelope("log", step, FLAGS.visit_weight_base, reg_warmup_steps, 0)
feed_dict = {rwalker_weight: rwalker_weight_ * FLAGS.reg_association_weight,
rvisit_weight: rvisit_weight_ * FLAGS.reg_association_weight,
walker_weight: walker_weight_ * FLAGS.cluster_association_weight,
visit_weight: visit_weight_ * FLAGS.cluster_association_weight,
t_l1_weight: FLAGS.l1_weight,
t_norm_weight: FLAGS.norm_weight,
t_logit_weight: logit_weight_,
t_trafo_weight: trafo_weight,
t_sat_loss_weight: 0,
t_learning_rate: 1e-6 + apply_envelope("log", step, learning_rate_, FLAGS.warmup_steps, 0)
}
_, sat_loss, train_loss, summaries, centroids, unsup_emb, reg_unsup_emb, estimated_error, p_ab, p_ba, p_aba, \
reg_loss, trafo_loss = sess.run(
[train_op, train_op_sat, model.train_loss, summary_op, t_sup_emb, t_unsup_emb, t_reg_unsup_emb,
model.estimate_error, model.p_ab,
model.p_ba, model.p_aba, model.reg_loss_aba, t_trafo_loss], {**extra_feed_dict, **feed_dict})
if FLAGS.kmeans_sat_thresh is not None and step % 200 == 0 and not kmeans_initialized:
sat_score = semisup.calc_sat_score(unsup_emb, reg_unsup_emb)
if sat_score > FLAGS.kmeans_sat_thresh:
print('initializing with kmeans', step, sat_score)
FLAGS.init_with_kmeans = True
kmeans_initialized = True
reg_warmup_steps = step # -> jump to next if clause
if FLAGS.init_with_kmeans and step == reg_warmup_steps:
# do kmeans, initialize with kmeans
embs = model.calc_embedding(c_train_imgs, model.test_emb, sess, extra_feed_dict)
kmeans = semisup.KMeans(n_clusters=num_labels, random_state=0).fit(embs)
init_virt = []
noise = 0.0001
for c in range(num_labels):
center = kmeans.cluster_centers_[c]
noise = np.random.uniform(-noise, noise, size=[FLAGS.virtual_embeddings_per_class, FLAGS.emb_size])
centroids = noise + center
init_virt.extend(centroids)
# init with K-Means
assign_op = t_sup_emb.assign(np.array(init_virt))
sess.run(assign_op)
model.reset_optimizer(sess)
rwalker_weight_ *= FLAGS.reg_decay_factor
rvisit_weight_ *= FLAGS.reg_decay_factor
if FLAGS.svm_test_interval is not None and step % FLAGS.svm_test_interval == 0 and step > 0:
svm_test_score, _ = model.train_and_eval_svm(c_train_imgs, train_labels_svm, c_test_imgs, test_labels,
sess, num_samples=5000)
print('svm score:', svm_test_score)
test_pred = model.classify(c_test_imgs, sess)
train_pred = model.classify(c_train_imgs, sess)
svm_test_score, _ = model.train_and_eval_svm_on_preds(train_pred, train_labels_svm, test_pred, test_labels,
sess, num_samples=5000)
print('svm score on logits:', svm_test_score)
if step % FLAGS.decay_steps == 0 and step > 0:
learning_rate_ = learning_rate_ * FLAGS.decay_factor
if step == 0 or (step + 1) % FLAGS.eval_interval == 0 or step == 99:
print('Step: %d' % step)
print('trafo loss', trafo_loss)
print('reg loss' , reg_loss)
print('Time for step', time.time() - start)
test_pred = model.classify(c_test_imgs, sess, extra_feed_dict).argmax(-1)
nmi = semisup.calc_nmi(test_pred, test_labels)
conf_mtx, score = semisup.calc_correct_logit_score(test_pred, test_labels, num_labels)
print(conf_mtx)
print('Test error: %.2f %%' % (100 - score * 100))
print('Test NMI: %.2f %%' % (nmi * 100))
print('Train loss: %.2f ' % train_loss)
print('Train loss no fc: %.2f ' % sat_loss)
print('Reg loss aba: %.2f ' % reg_loss)
print('Estimated Accuracy: %.2f ' % estimated_error)
sat_score = semisup.calc_sat_score(unsup_emb, reg_unsup_emb)
print('sat accuracy', sat_score)
embs = model.calc_embedding(c_test_imgs, model.test_emb, sess, extra_feed_dict)
c_n = norm(centroids, axis=1, ord=2)
e_n = norm(embs[0:100], axis=1, ord=2)
print('centroid norm', np.mean(c_n))
print('embedding norm', np.mean(e_n))
k_conf_mtx, k_score = semisup.do_kmeans(embs, test_labels, num_labels)
print(k_conf_mtx)
print('k means score:', k_score) # sometimes that kmeans is better than the logits
if FLAGS.logdir is not None:
sum_values = {
'test score': score,
'reg loss': reg_loss,
'centroid norm': np.mean(c_n),
'embedding norm': np.mean(c_n),
'k means score': k_score
}
summary_writer.add_summary(summaries, step)
for key, value in sum_values.items():
summary = tf.Summary(
value=[tf.Summary.Value(tag=key, simple_value=value)])
summary_writer.add_summary(summary, step)
# early stopping to save some time
if step == 34999 and score < 0.45:
break
if step == 14999 and score < 0.225:
break
if dataset == 'mnist' and step == 6999 and score < 0.25:
break
svm_test_score, _ = model.train_and_eval_svm(c_train_imgs, train_labels_svm, c_test_imgs, test_labels, sess,
num_samples=10000)
if FLAGS.logdir is not None:
path = saver.save(sess, FLAGS.logdir, model.step)
print('@@model_path:%s' % path)
print('FINAL RESULTS:')
print(conf_mtx)
print('Test error: %.2f %%' % (100 - score * 100))
print('final_score', score)
print('@@test_error:%.4f' % score)
print('@@train_loss:%.4f' % train_loss)
print('@@reg_loss:%.4f' % reg_loss)
print('@@estimated_error:%.4f' % estimated_error)
print('@@centroid_norm:%.4f' % np.mean(c_n))
print('@@emb_norm:%.4f' % np.mean(e_n))
print('@@k_score:%.4f' % k_score)
print('@@svm_score:%.4f' % svm_test_score)
def task(task_name):
s = tf.constant(task_name, tf.string)
return Dataset.from_tensors(s).repeat()
def dataset_with_label(self, label_int, src_pattern):
label = tf.constant(label_int, tf.int32, name="label")
lines = Dataset.list_files(src_pattern).flat_map(
lambda fn: TextLineDataset(fn))
labels = Dataset.from_tensors(label).repeat()
return Dataset.zip((lines, labels))
def random_image():
return None, Dataset.from_tensors(
np.random.random((784,)).astype(np.float32))
def random_embedding():
embedding = np.random.random((128,)).astype(np.float32)
return None, Dataset.from_tensors({'z': embedding})
DIR = '/home/mtb/test.npy'
# a = np.asarray([1,2,3,4,5,6])
# np.save(DIR, a)
# b =np.load(DIR)
# print b.shape
filename = DIR
# input_=np.load(DIR + 'input.npy')
output_ = np.load(DIR + 'output.npy')
input_placeholder = tf.placeholder(tf.float32, shape=None)
# output_placeholder = tf.placeholder(tf.float32, shape = output_.shape)
a = Dataset.from_tensors(input_placeholder)
b = Dataset.from_tensors(output_placeholder)
# c = Dataset.zip((a,b))
# batch_ = a.batch(100)
iterator = a.make_initializable_iterator()
next_elem = iterator.get_next()
# # b = Dataset.from_tensor_slices(input_placeholder)
# # tf.Session().run(tf.global_variables_initializer(),)
# # b = Dataset.from_tensor_slices(tf.Session().run(v,{input_placeholder: input_}))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
# print a.shape
# print sess.run(next_elem, {input_placeholder:input_})
input_dict = OrderedDict()
def random():
image = np.random.random((784,)).astype(np.float32)
embedding = np.random.random((128,)).astype(np.float32)
return None, Dataset.from_tensors({'x': image, 'z': embedding})
def flat_map_fn(iterator_id_t):
repeated_id = Dataset.from_tensors(iterator_id_t).repeat(None)
return repeated_id.map(generator_map_fn)
def read(self, batch_size, num_epochs=1, shuffle=False, task_spec=None):
"""
Reads the data and return a tuple of (inputs,outputs)
:param batch_size: the batch size of the returned inputs/outputs
:param num_epochs: the number of epochs to read the dataset
:param shuffle: whether to shuffle the data or not
:param task_spec: the task spec of the training. I will help to know whether it is
distributed training or not
:return: The result of calling dataset.make_one_shot_iterator().get_next()
"""
# TODO in TF 1.4 use: dataset = Dataset.from_generator(self.generator)
# FIXME repeat doesn't work with generators, so we can encapsulate the generator here
def _epochs():
for _ in range(num_epochs):
for item in self.generator():
yield item
generator_state = _GeneratorState(_epochs)
output_types = self.output_types
output_shapes = self.output_shapes
output_shapes = nest.map_structure(
lambda _: tensor_shape.TensorShape(None), output_types)
flattened_types = nest.flatten(output_types)
flattened_shapes = nest.flatten(output_shapes)
def get_iterator_id_map_fn(dummy):
return script_ops.py_func(generator_state.get_next_id, [],
tf.int64,
stateful=True)
def generator_map_fn(iterator_id_t):
def generator_py_func(iterator_id):
try:
values = next(generator_state.get_iterator(iterator_id))
except StopIteration:
generator_state.iterator_completed(iterator_id)
raise StopIteration("Iteration finished.")
ret_arrays = [
script_ops.FuncRegistry._convert(ret)
for ret in nest.flatten_up_to(output_types, values)
]
for (ret_array, expected_dtype,
expected_shape) in zip(ret_arrays, flattened_types,
flattened_shapes):
if ret_array.dtype != expected_dtype.as_numpy_dtype:
raise TypeError(
"`generator` yielded an element of type %s where an element "
"of type %s was expected." %
(ret_array.dtype, expected_dtype.as_numpy_dtype))
if not expected_shape.is_compatible_with(ret_array.shape):
raise ValueError(
"`generator` yielded an element of shape %s where an element "
"of shape %s was expected." %
(ret_array.shape, expected_shape))
return ret_arrays
flat_values = script_ops.py_func(generator_py_func,
[iterator_id_t],
self.output_types,
stateful=True)
if output_shapes is not None:
for ret_t, shape in zip(flat_values, flattened_shapes):
ret_t.set_shape(shape)
return nest.pack_sequence_as(output_types, flat_values)
def flat_map_fn(iterator_id_t):
repeated_id = Dataset.from_tensors(iterator_id_t).repeat(None)
return repeated_id.map(generator_map_fn)
id_dataset = Dataset.from_tensors(0).map(get_iterator_id_map_fn)
dataset = id_dataset.flat_map(flat_map_fn)
############################################################################################
# set the number of epochs
# FIXME repeat doesn't work with generators
# dataset = dataset.repeat(num_epochs)
if task_spec and task_spec.num_workers > 1:
# split the dataset in shards
# TODO in TF 1.4 use: dataset = dataset.shard(task_spec.num_workers, task_spec.index)
from tensorflow.python.ops import math_ops
def filter_fn(elem_index, _):
mod_result = math_ops.mod(elem_index, task_spec.num_workers)
return math_ops.equal(mod_result, task_spec.index)
dataset = dataset.enumerate().filter(filter_fn).map(
lambda _, elem: elem)
if shuffle:
# shuffle the samples
if self.shuffle_size is None:
raise ValueError('shuffle_size has not been set')
dataset = dataset.shuffle(buffer_size=self.shuffle_size)
# process each example. We check the method is defined in the child class:
if self._map.__func__ not in TFDataSetGenerator.__dict__.values():
dataset = dataset.map(
self._map,
# use as many threads as CPUs + 1
# TODO in TF 1.4 use: num_parallel_calls=multiprocessing.cpu_count() + 1,
num_threads=multiprocessing.cpu_count() + 1,
# buffer the data as CPUs * batch_size + minimum_size
output_buffer_size=batch_size * multiprocessing.cpu_count() +
self.min_queue_examples)
if self.padded_shapes:
dataset = dataset.padded_batch(batch_size, self.padded_shapes,
self.padded_values)
else:
dataset = dataset.batch(batch_size)
return dataset.make_one_shot_iterator().get_next()
def random_image_ae():
image = np.random.random((784,)).astype(np.float32)
return None, Dataset.from_tensors({'x': image})
