def validate(model_def): """ Validate my alexnet implementation Args: model_def: the model class/definition """ img_dir = os.path.join('.', 'images') images = [] print "loading images ..." files = fnmatch.filter(os.listdir(img_dir), '*.jpeg') for f in files: print "> " + f img_file = tf.read_file(os.path.join(img_dir, f)) img_decoded = tf.image.decode_jpeg(img_file, channels=3) img_processed = model_def.image_prep.preprocess_image( image=img_decoded, output_height=model_def.image_size, output_width=model_def.image_size, is_training=False) images.append(img_processed) # create TensorFlow Iterator object images = Dataset.from_tensors(images) iterator = Iterator.from_structure(images.output_types, images.output_shapes) next_element = iterator.get_next() iterator_init_op = iterator.make_initializer(images) # create the model and get scores (pipe to softmax) model = model_def(next_element) scores = model.get_final_op() softmax = tf.nn.softmax(scores) print 'start validation ...' with tf.Session() as sess: # Initialize all variables and the iterator sess.run(tf.global_variables_initializer()) sess.run(iterator_init_op) # Load the pretrained weights into the model model.load_initial_weights(sess) # run the graph probs = sess.run(softmax) if model_def is ResNetV2: probs = prep_resnet_results(probs) # sometime we have an offset offset = len(class_names) - len(probs[0]) # print the results for prob in probs: best_index = np.argmax(prob) print "> " + class_names[best_index + offset] + " -> %.4f" % prob[best_index]
def train(self, Datasize=None): with tf.name_scope("Dataset"): fin = int(min(self.cut, Datasize) if Datasize else self.cut) choices = self.train_choices[:fin] dat = (Dataset.from_tensors([ d[choices, :].astype(np.float32) for d in [self.X, self.Y] ]).make_initializable_iterator()) tf.add_to_collection("batch_init", dat.initializer) return dat.get_next("bs_dat")
def split_multi_string(self, x): """ Splits a list of strings on whitespaces and casts them to int :param x: list of sequences :return: list of int tensors """ # Split best answers on comma n_best_answers = tf.string_split([x], delimiter=",") # Reformat data to sparse tensor n_best_answers = tf.SparseTensorValue(indices=n_best_answers.indices, values=tf.string_split(n_best_answers.values), dense_shape=n_best_answers.dense_shape) # Get data as sparse tensor up_ba = n_best_answers.values # Sparse tensor to dense Tensor with padding '0' up_ba = tf.sparse_to_dense(up_ba.indices, (up_ba.dense_shape[0], up_ba.dense_shape[1] + 1), up_ba.values, '0') # If n_best_answer not empty, convert every answer to int up_ba = tf.cond(tf.greater(tf.size(up_ba), 0), lambda: tf.map_fn(lambda s: tf.string_to_number(s, out_type=tf.int32), up_ba, dtype=tf.int32), lambda: tf.string_to_number(up_ba, out_type=tf.int32)) # Get length of all sequences seq_len = tf.argmin(tf.to_int32(tf.not_equal(up_ba, 0)), axis=1) # Create filter mask idx = tf.less_equal(seq_len, self.max_seq_len - 1) # Filter sequences and length up_ba = tf.boolean_mask(up_ba, idx) up_ba = up_ba[:, 0:tf.cond(tf.greater(tf.size(up_ba), 0), lambda: self.max_seq_len, lambda: 0)] up_ba = tf.pad(up_ba, [[0, 0], [0, self.max_seq_len - tf.shape(up_ba)[1]]]) seq_len = tf.boolean_mask(seq_len, idx) + 1 # Make datasets sequences_nba = Dataset.from_tensors(up_ba) len_nba = Dataset.from_tensors(seq_len) return Dataset.zip((sequences_nba, len_nba))
def create_dataset(batch_size): files, labels = list_files_and_labels() files_const = tf.constant(files) labels_const = tf.one_hot(tf.constant(labels), depth=10) dataset = Dataset.from_tensor_slices((files_const, labels_const)) dataset = dataset.interleave(lambda filename, label: Dataset.from_tensors( (filename, label)).map(_parse_function, num_threads=1), cycle_length=10) # dataset = dataset.shuffle(buffer_size=10000) dataset = dataset.batch(batch_size) return dataset
def process_target(self, sub, len_sub, cont, len_cont, nba, nba_len): """ Processes target sequence with setting GO-Symbol in front of target input sequence and EOS-Symbol at the end of target output sequence :param sub: subject sequences :param len_sub: length of subject sequences :param cont: content sequences :param len_cont: length of content sequences :param nba: n best answers sequences :param nba_len: length of n best answers :return: Dataset with processed target input and output """ def set_eos(nba): """ Receives one answer out of all answers and sets an EOS-Symbol at the end :param nba: One answer out of n best answers :return: Answer with following EOS-Symbol """ len_nba = tf.argmin(nba, output_type=tf.int32) nba = tf.concat([nba[:len_nba], [3], nba[len_nba + 1:]], axis=0) return nba # Set number 2 (GO) at the beginning of the sequences target_input = tf.concat([tf.fill((tf.size(nba_len), 1), 2), nba[:, :-1]], axis=1) # Set number 3 (EOS) at the end of the sequences target_output = tf.map_fn(set_eos, nba, dtype=tf.int32) # Convert function input back to dataset sub = Dataset.from_tensors(sub) len_sub = Dataset.from_tensors(len_sub) cont = Dataset.from_tensors(cont) len_cont = Dataset.from_tensors(len_cont) target_input = Dataset.from_tensors(target_input) target_output = Dataset.from_tensors(target_output) len_nba = Dataset.from_tensors(nba_len) return Dataset.zip((sub, len_sub, cont, len_cont, target_input, target_output, len_nba))
def get_seq_len_and_join_ba(self, sub, cont, nba_and_len): """ For getting length of subject sequences, content sequences and answer sequences :param sub: subject sequences :param cont: content sequences :param nba_and_len: n best answer ans length :return: Dataset with sequences and length """ # Seperate n best answers and length nba = nba_and_len[0] nba_len = nba_and_len[1] # Count sequence length of subject and content len_sub = Dataset.from_tensors(tf.size(sub)) len_cont = Dataset.from_tensors(tf.size(cont)) # Make dataset from tensors sub = Dataset.from_tensors(sub) cont = Dataset.from_tensors(cont) nba = Dataset.from_tensors(nba) nba_len = Dataset.from_tensors(nba_len) return Dataset.zip((sub, len_sub, cont, len_cont, nba, nba_len))
def main(): """Run the demo.""" data = fetch_gpml_sarcos_data() Xr = data.train.data.astype(np.float32) Yr = data.train.targets.astype(np.float32)[:, np.newaxis] Xs = data.test.data.astype(np.float32) Ys = data.test.targets.astype(np.float32)[:, np.newaxis] N, D = Xr.shape print("Iterations: {}".format(int(round(N * NEPOCHS / BATCH_SIZE)))) # Scale and centre the data, as per the original experiment ss = StandardScaler() Xr = ss.fit_transform(Xr) Xs = ss.transform(Xs) ym = Yr.mean() Yr -= ym Ys -= ym # Training batches data_tr = Dataset.from_tensor_slices({'X': Xr, 'Y': Yr}) \ .shuffle(buffer_size=1000) \ .batch(BATCH_SIZE) # Testing iterators data_ts = Dataset.from_tensors({'X': Xs, 'Y': Ys}).repeat() with tf.name_scope("DataIterators"): iterator = Iterator.from_structure(data_tr.output_types, data_tr.output_shapes) data = iterator.get_next() training_init = iterator.make_initializer(data_tr) testing_init = iterator.make_initializer(data_ts) with tf.name_scope("Deepnet"): phi, kl = net(X=data['X']) std = tf.Variable(NOISE, name="noise") lkhood = tf.distributions.Normal(phi, scale=ab.pos(std)) loss = ab.elbo(lkhood, data['Y'], N, kl) tf.summary.scalar('loss', loss) with tf.name_scope("Train"): optimizer = tf.train.AdamOptimizer() global_step = tf.train.create_global_step() train = optimizer.minimize(loss, global_step=global_step) with tf.name_scope("Test"): r2 = rsquare(data['Y'], phi) # Logging log = tf.train.LoggingTensorHook( {'step': global_step, 'loss': loss}, every_n_iter=1000 ) with tf.train.MonitoredTrainingSession( config=CONFIG, scaffold=tf.train.Scaffold(local_init_op=training_init), checkpoint_dir="./sarcos/", save_summaries_steps=None, save_checkpoint_secs=20, save_summaries_secs=20, hooks=[log] ) as sess: summary_writer = sess._hooks[1]._summary_writer for i in range(NEPOCHS): # Train for one epoch try: while not sess.should_stop(): sess.run(train) except tf.errors.OutOfRangeError: pass # Init testing and assess and log R-square score on test set sess.run(testing_init) r2_score = sess.run(r2) score_sum = tf.Summary(value=[ tf.Summary.Value(tag='r-square', simple_value=r2_score) ]) summary_writer.add_summary(score_sum, sess.run(global_step)) # Re-init training sess.run(training_init) # Prediction sess.run(testing_init) Ey = ab.predict_samples(phi, feed_dict=None, n_groups=NPREDICTSAMPLES, session=sess) sigma = sess.run(std) r2_score = sess.run(r2) # Score mean standardised log likelihood Eymean = Ey.mean(axis=0) Eyvar = Ey.var(axis=0) + sigma**2 # add sigma2 for obervation noise snlp = msll(Ys.flatten(), Eymean, Eyvar, Yr.flatten()) print("------------") print("r-square: {:.4f}, smse: {:.4f}, msll: {:.4f}." .format(r2_score, 1 - r2_score, snlp))
def main(_): FLAGS.eval_interval = 1000 # todo remove if FLAGS.logdir is not None: if FLAGS.taskid is not None: FLAGS.logdir = FLAGS.logdir + '/t_' + str(FLAGS.taskid) else: FLAGS.logdir = FLAGS.logdir + '/t_' + str(random.randint(0,99999)) dataset_tools = import_module('tools.' + FLAGS.dataset) NUM_LABELS = dataset_tools.NUM_LABELS num_labels = NUM_LABELS IMAGE_SHAPE = dataset_tools.IMAGE_SHAPE image_shape = IMAGE_SHAPE train_images, train_labels_svm = dataset_tools.get_data('train') # no train labels nowhere test_images, test_labels = dataset_tools.get_data('test') if FLAGS.zero_fact < 1: # exclude a random set of zeros (not at the end, then there would be many batches without zeros) keep = np.ones(len(train_labels_svm), dtype=bool) zero_indices = np.where((train_labels_svm == 0))[0] remove = np.random.uniform(0, 1, len(zero_indices)) zero_indices_to_remove = zero_indices[remove > FLAGS.zero_fact] keep[zero_indices_to_remove] = False train_images = train_images[keep] train_labels_svm = train_labels_svm[keep] print('using only a fraction of zeros, resulting in the following shape:', train_images.shape) if FLAGS.num_unlabeled_images > 0: unlabeled_train_images, _ = dataset_tools.get_data('unlabeled', max_num=np.min([FLAGS.num_unlabeled_images, 50000])) train_images = np.vstack([train_images, unlabeled_train_images]) if FLAGS.normalize_input: train_images = (train_images - 128.) / 128. test_images = (test_images - 128.) / 128. if FLAGS.use_test: train_images = np.vstack([train_images, test_images]) train_labels_svm = np.hstack([train_labels_svm, test_labels]) #if FLAGS.dataset == 'svhn' and FLAGS.architecture == 'resnet_cifar_model': # FLAGS.emb_size = 64 image_shape_crop = image_shape c_test_imgs = test_images c_train_imgs = train_images # crop images to some random region. Intuitively, images should belong to the same cluster, # even if a part of the image is missing # (no padding, because the net could detect padding easily, and match it to other augmented samples that have # padding) if FLAGS.dataset == 'stl10': image_shape_crop = [64, 64, 3] c_test_imgs = test_images[:, 16:80, 16:80] c_train_imgs = train_images[:, 16:80, 16:80] def aug(image): return apply_augmentation(image, target_shape=image_shape_crop, params=dataset_tools.augmentation_params) def random_crop(image): image_size = image_shape_crop[0] image = tf.random_crop(image, [image_size, image_size, image_shape[2]]) return image graph = tf.Graph() with graph.as_default(): t_images = tf.placeholder("float", shape=[None] + image_shape) dataset = Dataset.from_tensor_slices(t_images) dataset = dataset.shuffle(buffer_size=10000, seed=47) # important, so that we have the same images in both sets # parameters for buffering during augmentation. Only influence training speed. nt = 8 if FLAGS.volta else 4 # that's not even enough, but there are no more CPUs b = 10000 rf = FLAGS.num_augmented_samples augmented_set = dataset if FLAGS.shuffle_augmented_samples: augmented_set = augmented_set.shuffle(buffer_size=10000, seed=47) # get multiple augmented versions of the same image - they should later have similar embeddings augmented_set = augmented_set.flat_map(lambda x: Dataset.from_tensors(x).repeat(rf)) augmented_set = augmented_set.map(aug, num_threads=nt, output_buffer_size=b) dataset = dataset.map(random_crop, num_threads=1, output_buffer_size=b) dataset = dataset.repeat().batch(FLAGS.unsup_batch_size) augmented_set = augmented_set.repeat().batch(FLAGS.unsup_batch_size * rf) iterator = dataset.make_initializable_iterator() reg_iterator = augmented_set.make_initializable_iterator() t_unsup_images = iterator.get_next() t_reg_unsup_images = reg_iterator.get_next() model_func = getattr(semisup.architectures, FLAGS.architecture) model = semisup.SemisupModel(model_func, num_labels, image_shape_crop, optimizer='adam', emb_size=FLAGS.emb_size, dropout_keep_prob=FLAGS.dropout_keep_prob, num_blocks=FLAGS.num_blocks, normalize_embeddings=FLAGS.normalize_embeddings, beta1=FLAGS.beta1, beta2=FLAGS.beta2) init_virt = [] for c in range(num_labels): center = np.random.normal(0, 0.3, size=[1, FLAGS.emb_size]) noise = np.random.uniform(-0.01, 0.01, size=[FLAGS.virtual_embeddings_per_class, FLAGS.emb_size]) centroids = noise + center init_virt.extend(centroids) t_sup_emb = tf.Variable(tf.cast(np.array(init_virt), tf.float32), name="virtual_centroids") t_sup_labels = tf.constant( np.concatenate([[i] * FLAGS.virtual_embeddings_per_class for i in range(num_labels)])) visit_weight = tf.placeholder("float", shape=[]) walker_weight = tf.placeholder("float", shape=[]) t_logit_weight = tf.placeholder("float", shape=[]) t_trafo_weight = tf.placeholder("float", shape=[]) t_l1_weight = tf.placeholder("float", shape=[]) t_norm_weight = tf.placeholder("float", shape=[]) t_learning_rate = tf.placeholder("float", shape=[]) t_sat_loss_weight = tf.placeholder("float", shape=[]) t_unsup_emb = model.image_to_embedding(t_unsup_images) t_reg_unsup_emb = model.image_to_embedding(t_reg_unsup_images) t_all_unsup_emb = tf.concat([t_unsup_emb, t_reg_unsup_emb], axis=0) t_rsup_labels = tf.constant(np.concatenate([[i] * rf for i in range(FLAGS.unsup_batch_size)])) rwalker_weight = tf.placeholder("float", shape=[]) rvisit_weight = tf.placeholder("float", shape=[]) if FLAGS.normalize_embeddings: t_sup_logit = model.embedding_to_logit(tf.nn.l2_normalize(t_sup_emb, dim=1)) model.add_semisup_loss( tf.nn.l2_normalize(t_sup_emb, dim=1), tf.nn.l2_normalize(t_unsup_emb, dim=1), t_sup_labels, walker_weight=walker_weight, visit_weight=visit_weight, match_scale=FLAGS.scale_match_ab) model.reg_loss_aba = model.add_semisup_loss( tf.nn.l2_normalize(t_reg_unsup_emb, dim=1), tf.nn.l2_normalize(t_unsup_emb, dim=1), t_rsup_labels, walker_weight=rwalker_weight, visit_weight=rvisit_weight, match_scale=FLAGS.scale_match_ab, est_err=False) else: t_sup_logit = model.embedding_to_logit(t_sup_emb) model.add_semisup_loss( t_sup_emb, t_unsup_emb, t_sup_labels, walker_weight=walker_weight, visit_weight=visit_weight, match_scale=FLAGS.scale_match_ab, est_err=True, name='c_association') model.reg_loss_aba = model.add_semisup_loss( t_reg_unsup_emb, t_unsup_emb, t_rsup_labels, walker_weight=rwalker_weight, visit_weight=rvisit_weight, match_scale=FLAGS.scale_match_ab, est_err=False, name='aug_association') model.add_logit_loss(t_sup_logit, t_sup_labels, weight=t_logit_weight) t_reg_unsup_emb_singled = t_reg_unsup_emb[::FLAGS.num_augmented_samples] t_unsup_logit = model.embedding_to_logit(t_unsup_emb) t_reg_unsup_logit = model.embedding_to_logit(t_reg_unsup_emb_singled) model.add_sat_loss(t_unsup_logit, t_reg_unsup_logit, weight=t_sat_loss_weight) trafo_lc = semisup.NO_FC_COLLECTION if FLAGS.trafo_separate_loss_collection else semisup.LOSSES_COLLECTION if FLAGS.trafo_weight > 0: # only use a single augmented sample per sample t_trafo_loss = model.add_transformation_loss(t_unsup_emb, t_reg_unsup_emb_singled, t_unsup_logit, t_reg_unsup_logit, FLAGS.unsup_batch_size, weight=t_trafo_weight, label_smoothing=0, loss_collection=trafo_lc) else: t_trafo_loss = tf.constant(0) model.add_emb_regularization(t_all_unsup_emb, weight=t_l1_weight) model.add_emb_regularization(t_sup_emb, weight=t_l1_weight) # make l2 norm = 3 model.add_emb_normalization(t_sup_emb, weight=t_norm_weight, target=FLAGS.norm_target) model.add_emb_normalization(t_all_unsup_emb, weight=t_norm_weight, target=FLAGS.norm_target) gradient_multipliers = {t_sup_emb: 1 } [train_op, train_op_sat] = model.create_train_op(t_learning_rate, gradient_multipliers=gradient_multipliers) summary_op = tf.summary.merge_all() if FLAGS.logdir is not None: summary_writer = tf.summary.FileWriter(FLAGS.logdir, graph) saver = tf.train.Saver() with tf.Session(graph=graph) as sess: tf.global_variables_initializer().run() sess.run(iterator.initializer, feed_dict={t_images: train_images}) sess.run(reg_iterator.initializer, feed_dict={t_images: train_images}) # optional: init from autoencoder if FLAGS.restore_checkpoint is not None: # logit fc layer cannot be restored def is_main_net(x): return 'logit_fc' not in x.name and 'Adam' not in x.name variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='net') variables = list(filter(is_main_net, variables)) restorer = tf.train.Saver(var_list=variables) restorer.restore(sess, FLAGS.restore_checkpoint) extra_feed_dict = {} from numpy.linalg import norm reg_warmup_steps = FLAGS.reg_warmup_steps logit_weight_ = FLAGS.logit_weight rwalker_weight_ = FLAGS.rwalker_weight rvisit_weight_ = FLAGS.rvisit_weight learning_rate_ = FLAGS.learning_rate trafo_weight = FLAGS.trafo_weight kmeans_initialized = False for step in range(FLAGS.max_steps): import time start = time.time() if FLAGS.init_with_kmeans: if FLAGS.kmeans_sat_thresh is not None and not kmeans_initialized or \ FLAGS.kmeans_sat_thresh is None and step <= reg_warmup_steps: walker_weight_ = 0 visit_weight_ = 0 logit_weight_ = 0 trafo_weight = 0 else: walker_weight_ = FLAGS.walker_weight visit_weight_ = FLAGS.visit_weight_base logit_weight_ = FLAGS.logit_weight trafo_weight = FLAGS.trafo_weight else: walker_weight_ = apply_envelope("log", step, FLAGS.walker_weight, reg_warmup_steps, 0) visit_weight_ = apply_envelope("log", step, FLAGS.visit_weight_base, reg_warmup_steps, 0) feed_dict = {rwalker_weight: rwalker_weight_ * FLAGS.reg_association_weight, rvisit_weight: rvisit_weight_ * FLAGS.reg_association_weight, walker_weight: walker_weight_ * FLAGS.cluster_association_weight, visit_weight: visit_weight_ * FLAGS.cluster_association_weight, t_l1_weight: FLAGS.l1_weight, t_norm_weight: FLAGS.norm_weight, t_logit_weight: logit_weight_, t_trafo_weight: trafo_weight, t_sat_loss_weight: 0, t_learning_rate: 1e-6 + apply_envelope("log", step, learning_rate_, FLAGS.warmup_steps, 0) } _, sat_loss, train_loss, summaries, centroids, unsup_emb, reg_unsup_emb, estimated_error, p_ab, p_ba, p_aba, \ reg_loss, trafo_loss = sess.run( [train_op, train_op_sat, model.train_loss, summary_op, t_sup_emb, t_unsup_emb, t_reg_unsup_emb, model.estimate_error, model.p_ab, model.p_ba, model.p_aba, model.reg_loss_aba, t_trafo_loss], {**extra_feed_dict, **feed_dict}) if FLAGS.kmeans_sat_thresh is not None and step % 200 == 0 and not kmeans_initialized: sat_score = semisup.calc_sat_score(unsup_emb, reg_unsup_emb) if sat_score > FLAGS.kmeans_sat_thresh: print('initializing with kmeans', step, sat_score) FLAGS.init_with_kmeans = True kmeans_initialized = True reg_warmup_steps = step # -> jump to next if clause if FLAGS.init_with_kmeans and step == reg_warmup_steps: # do kmeans, initialize with kmeans embs = model.calc_embedding(c_train_imgs, model.test_emb, sess, extra_feed_dict) kmeans = semisup.KMeans(n_clusters=num_labels, random_state=0).fit(embs) init_virt = [] noise = 0.0001 for c in range(num_labels): center = kmeans.cluster_centers_[c] noise = np.random.uniform(-noise, noise, size=[FLAGS.virtual_embeddings_per_class, FLAGS.emb_size]) centroids = noise + center init_virt.extend(centroids) # init with K-Means assign_op = t_sup_emb.assign(np.array(init_virt)) sess.run(assign_op) model.reset_optimizer(sess) rwalker_weight_ *= FLAGS.reg_decay_factor rvisit_weight_ *= FLAGS.reg_decay_factor if FLAGS.svm_test_interval is not None and step % FLAGS.svm_test_interval == 0 and step > 0: svm_test_score, _ = model.train_and_eval_svm(c_train_imgs, train_labels_svm, c_test_imgs, test_labels, sess, num_samples=5000) print('svm score:', svm_test_score) test_pred = model.classify(c_test_imgs, sess) train_pred = model.classify(c_train_imgs, sess) svm_test_score, _ = model.train_and_eval_svm_on_preds(train_pred, train_labels_svm, test_pred, test_labels, sess, num_samples=5000) print('svm score on logits:', svm_test_score) if step % FLAGS.decay_steps == 0 and step > 0: learning_rate_ = learning_rate_ * FLAGS.decay_factor if step == 0 or (step + 1) % FLAGS.eval_interval == 0 or step == 99: print('Step: %d' % step) print('trafo loss', trafo_loss) print('reg loss' , reg_loss) print('Time for step', time.time() - start) test_pred = model.classify(c_test_imgs, sess, extra_feed_dict).argmax(-1) nmi = semisup.calc_nmi(test_pred, test_labels) conf_mtx, score = semisup.calc_correct_logit_score(test_pred, test_labels, num_labels) print(conf_mtx) print('Test error: %.2f %%' % (100 - score * 100)) print('Test NMI: %.2f %%' % (nmi * 100)) print('Train loss: %.2f ' % train_loss) print('Train loss no fc: %.2f ' % sat_loss) print('Reg loss aba: %.2f ' % reg_loss) print('Estimated Accuracy: %.2f ' % estimated_error) sat_score = semisup.calc_sat_score(unsup_emb, reg_unsup_emb) print('sat accuracy', sat_score) embs = model.calc_embedding(c_test_imgs, model.test_emb, sess, extra_feed_dict) c_n = norm(centroids, axis=1, ord=2) e_n = norm(embs[0:100], axis=1, ord=2) print('centroid norm', np.mean(c_n)) print('embedding norm', np.mean(e_n)) k_conf_mtx, k_score = semisup.do_kmeans(embs, test_labels, num_labels) print(k_conf_mtx) print('k means score:', k_score) # sometimes that kmeans is better than the logits if FLAGS.logdir is not None: sum_values = { 'test score': score, 'reg loss': reg_loss, 'centroid norm': np.mean(c_n), 'embedding norm': np.mean(c_n), 'k means score': k_score } summary_writer.add_summary(summaries, step) for key, value in sum_values.items(): summary = tf.Summary( value=[tf.Summary.Value(tag=key, simple_value=value)]) summary_writer.add_summary(summary, step) # early stopping to save some time if step == 34999 and score < 0.45: break if step == 14999 and score < 0.225: break if dataset == 'mnist' and step == 6999 and score < 0.25: break svm_test_score, _ = model.train_and_eval_svm(c_train_imgs, train_labels_svm, c_test_imgs, test_labels, sess, num_samples=10000) if FLAGS.logdir is not None: path = saver.save(sess, FLAGS.logdir, model.step) print('@@model_path:%s' % path) print('FINAL RESULTS:') print(conf_mtx) print('Test error: %.2f %%' % (100 - score * 100)) print('final_score', score) print('@@test_error:%.4f' % score) print('@@train_loss:%.4f' % train_loss) print('@@reg_loss:%.4f' % reg_loss) print('@@estimated_error:%.4f' % estimated_error) print('@@centroid_norm:%.4f' % np.mean(c_n)) print('@@emb_norm:%.4f' % np.mean(e_n)) print('@@k_score:%.4f' % k_score) print('@@svm_score:%.4f' % svm_test_score)
def task(task_name): s = tf.constant(task_name, tf.string) return Dataset.from_tensors(s).repeat()
def dataset_with_label(self, label_int, src_pattern): label = tf.constant(label_int, tf.int32, name="label") lines = Dataset.list_files(src_pattern).flat_map( lambda fn: TextLineDataset(fn)) labels = Dataset.from_tensors(label).repeat() return Dataset.zip((lines, labels))
def random_image(): return None, Dataset.from_tensors( np.random.random((784,)).astype(np.float32))
def random_embedding(): embedding = np.random.random((128,)).astype(np.float32) return None, Dataset.from_tensors({'z': embedding})
DIR = '/home/mtb/test.npy' # a = np.asarray([1,2,3,4,5,6]) # np.save(DIR, a) # b =np.load(DIR) # print b.shape filename = DIR # input_=np.load(DIR + 'input.npy') output_ = np.load(DIR + 'output.npy') input_placeholder = tf.placeholder(tf.float32, shape=None) # output_placeholder = tf.placeholder(tf.float32, shape = output_.shape) a = Dataset.from_tensors(input_placeholder) b = Dataset.from_tensors(output_placeholder) # c = Dataset.zip((a,b)) # batch_ = a.batch(100) iterator = a.make_initializable_iterator() next_elem = iterator.get_next() # # b = Dataset.from_tensor_slices(input_placeholder) # # tf.Session().run(tf.global_variables_initializer(),) # # b = Dataset.from_tensor_slices(tf.Session().run(v,{input_placeholder: input_})) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) # print a.shape # print sess.run(next_elem, {input_placeholder:input_}) input_dict = OrderedDict()
def random(): image = np.random.random((784,)).astype(np.float32) embedding = np.random.random((128,)).astype(np.float32) return None, Dataset.from_tensors({'x': image, 'z': embedding})
def flat_map_fn(iterator_id_t): repeated_id = Dataset.from_tensors(iterator_id_t).repeat(None) return repeated_id.map(generator_map_fn)
def read(self, batch_size, num_epochs=1, shuffle=False, task_spec=None): """ Reads the data and return a tuple of (inputs,outputs) :param batch_size: the batch size of the returned inputs/outputs :param num_epochs: the number of epochs to read the dataset :param shuffle: whether to shuffle the data or not :param task_spec: the task spec of the training. I will help to know whether it is distributed training or not :return: The result of calling dataset.make_one_shot_iterator().get_next() """ # TODO in TF 1.4 use: dataset = Dataset.from_generator(self.generator) # FIXME repeat doesn't work with generators, so we can encapsulate the generator here def _epochs(): for _ in range(num_epochs): for item in self.generator(): yield item generator_state = _GeneratorState(_epochs) output_types = self.output_types output_shapes = self.output_shapes output_shapes = nest.map_structure( lambda _: tensor_shape.TensorShape(None), output_types) flattened_types = nest.flatten(output_types) flattened_shapes = nest.flatten(output_shapes) def get_iterator_id_map_fn(dummy): return script_ops.py_func(generator_state.get_next_id, [], tf.int64, stateful=True) def generator_map_fn(iterator_id_t): def generator_py_func(iterator_id): try: values = next(generator_state.get_iterator(iterator_id)) except StopIteration: generator_state.iterator_completed(iterator_id) raise StopIteration("Iteration finished.") ret_arrays = [ script_ops.FuncRegistry._convert(ret) for ret in nest.flatten_up_to(output_types, values) ] for (ret_array, expected_dtype, expected_shape) in zip(ret_arrays, flattened_types, flattened_shapes): if ret_array.dtype != expected_dtype.as_numpy_dtype: raise TypeError( "`generator` yielded an element of type %s where an element " "of type %s was expected." % (ret_array.dtype, expected_dtype.as_numpy_dtype)) if not expected_shape.is_compatible_with(ret_array.shape): raise ValueError( "`generator` yielded an element of shape %s where an element " "of shape %s was expected." % (ret_array.shape, expected_shape)) return ret_arrays flat_values = script_ops.py_func(generator_py_func, [iterator_id_t], self.output_types, stateful=True) if output_shapes is not None: for ret_t, shape in zip(flat_values, flattened_shapes): ret_t.set_shape(shape) return nest.pack_sequence_as(output_types, flat_values) def flat_map_fn(iterator_id_t): repeated_id = Dataset.from_tensors(iterator_id_t).repeat(None) return repeated_id.map(generator_map_fn) id_dataset = Dataset.from_tensors(0).map(get_iterator_id_map_fn) dataset = id_dataset.flat_map(flat_map_fn) ############################################################################################ # set the number of epochs # FIXME repeat doesn't work with generators # dataset = dataset.repeat(num_epochs) if task_spec and task_spec.num_workers > 1: # split the dataset in shards # TODO in TF 1.4 use: dataset = dataset.shard(task_spec.num_workers, task_spec.index) from tensorflow.python.ops import math_ops def filter_fn(elem_index, _): mod_result = math_ops.mod(elem_index, task_spec.num_workers) return math_ops.equal(mod_result, task_spec.index) dataset = dataset.enumerate().filter(filter_fn).map( lambda _, elem: elem) if shuffle: # shuffle the samples if self.shuffle_size is None: raise ValueError('shuffle_size has not been set') dataset = dataset.shuffle(buffer_size=self.shuffle_size) # process each example. We check the method is defined in the child class: if self._map.__func__ not in TFDataSetGenerator.__dict__.values(): dataset = dataset.map( self._map, # use as many threads as CPUs + 1 # TODO in TF 1.4 use: num_parallel_calls=multiprocessing.cpu_count() + 1, num_threads=multiprocessing.cpu_count() + 1, # buffer the data as CPUs * batch_size + minimum_size output_buffer_size=batch_size * multiprocessing.cpu_count() + self.min_queue_examples) if self.padded_shapes: dataset = dataset.padded_batch(batch_size, self.padded_shapes, self.padded_values) else: dataset = dataset.batch(batch_size) return dataset.make_one_shot_iterator().get_next()
def random_image_ae(): image = np.random.random((784,)).astype(np.float32) return None, Dataset.from_tensors({'x': image})