def make_iterator(tensors): with tf.device('/device:CPU:0'): ds = tf.data.Dataset.from_tensors(tensors).repeat() return tfe.Iterator(ds)
parsed_line = tf.decode_csv(line, example_defaults) # First 4 fields are features, combine into single tensor features = tf.reshape(parsed_line[:-1], shape=(4, )) # Last field is the label label = tf.reshape(parsed_line[-1], shape=()) return features, label train_dataset = tf.data.TextLineDataset(train_dataset_fp) train_dataset = train_dataset.skip(1) # skip the first header row train_dataset = train_dataset.map(parse_csv) # parse each row train_dataset = train_dataset.shuffle(buffer_size=1000) # randomize train_dataset = train_dataset.batch(32) # View a single example entry from a batch features, label = tfe.Iterator(train_dataset).next() print("example features:", features[0]) print("example label:", label[0]) model = tf.keras.Sequential([ tf.keras.layers.Dense(10, activation="relu", input_shape=(4, )), # input shape required tf.keras.layers.Dense(10, activation="relu"), tf.keras.layers.Dense(3) ]) optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01) def loss(model, x, y): y_ = model(x)
labels = [ pClasses[cn]['type'] == 'ann' for cn in (filtGenPClassNames + annPClassNames) ] labels = np.array([tf.to_int32(x) for x in labels]) dataset = tf.data.Dataset.from_tensor_slices((featDict, labels)) dataset = dataset.shuffle(1000).repeat().batch(2) #train_dataset = tf.data.TextLineDataset(train_dataset_fp) #train_dataset = train_dataset.skip(1) # skip the first header row #train_dataset = train_dataset.map(parse_csv) # parse each row #train_dataset = train_dataset.shuffle(buffer_size=1000) # randomize #train_dataset = train_dataset.batch(32) features, label = tfe.Iterator(dataset).next() model = tf.keras.Sequential([ tf.keras.layers.Dense(10, activation="relu", input_shape=(5, )), # input shape required tf.keras.layers.Dense(10, activation="relu"), tf.keras.layers.Dense(1) ]) def loss(model, x, y): y_ = model(x) return tf.losses.sparse_softmax_cross_entropy(labels=y, logits=y_) def grad(model, inputs, targets):
shape=(basePairs, )) # Last field is the label labels = tf.reshape(parsed_line[0:statCount], shape=(statCount, )) return features, labels train_dataset = train_dataset.skip(1) # skip the first header row print("Mapping Database...") train_dataset = train_dataset.map(parse_csv) # parse each row print("Shuffling Database...") train_dataset = train_dataset.shuffle(buffer_size=100) # randomize print("Batching Database...") train_dataset = train_dataset.batch(batchSize) # View a single example entry from a batch features, label = tfe.Iterator(train_dataset).next() print("example features:", features[0]) print("example label:", label[0]) print("Generating Network...") neuralNetwork = tf.keras.Sequential([ tf.keras.layers.Dense(basePairs, activation="relu", input_shape=(basePairs, )), # input shape required tf.keras.layers.Dense(basePairs * 3, activation="relu"), tf.keras.layers.Dense(basePairs * 3, activation="relu"), tf.keras.layers.Dense(statCount) ]) def loss(prediction, groundTruth):
# Parameters learning_rate = 0.001 num_steps = 1000 batch_size = 128 display_step = 100 # Network Parameters n_hidden_1 = 256 # 1st layer number of neurons n_hidden_2 = 256 # 2nd layer number of neurons num_input = 784 # MNIST data input (img shape: 28*28) num_classes = 10 # MNIST total classes (0-9 digits) # Using TF Dataset to split data into batches dataset = tf.data.Dataset.from_tensor_slices( (mnist.train.images, mnist.train.labels)).batch(batch_size) dataset_iter = tfe.Iterator(dataset) # Define the neural network. To use eager API and tf.layers API together, # we must instantiate a tfe.Network class as follow: class NeuralNet(tfe.Network): def __init__(self): # Define each layer super(NeuralNet, self).__init__() # Hidden fully connected layer with 256 neurons self.layer1 = self.track_layer( tf.layers.Conv2D(32, 5, padding='SAME', activation=tf.nn.relu)) # [n,28,28,32] self.pool1 = self.track_layer( tf.layers.AveragePooling2D(2, 2, padding='SAME')) # [n,14,14,32]
def train(self, epoch, dataloader): dataset, data_len = dataloader[0], dataloader[1] if self.log_type == 'progressbar': # Progress bar processed_data_len = 0 bar = plugins.Bar('{:<10}'.format('Train'), max=data_len // self.batch_size) end = time.time() with self.summary_writer.as_default(): with tf.device(self.device): for i, (inputs, labels) in enumerate(tfe.Iterator(dataset)): # keeps track of data loading time data_time = time.time() - end ############################ # Update network ############################ inputs = tf.reshape(inputs, shape=(-1, self.nc, self.resolution_high, self.resolution_wide)) with tf.device('/cpu:0'): tf.assign_add(self.step_counter, 1) with tf.contrib.summary.always_record_summaries(): with tf.GradientTape() as tape: # get outputs outputs = self.model(inputs) # compute loss loss = self.criterion(outputs, labels) # perform evaluation accuracy = self.evaluation(outputs, labels) # logging and visualization tf.contrib.summary.scalar('loss', loss) tf.contrib.summary.scalar('accuracy', accuracy) self.params['Loss'] = loss.cpu()._numpy() self.params['Accuracy'] = accuracy.cpu()._numpy() # compute gradients grads = tape.gradient(loss, self.model.variables) # optimize the network self.optimizer.apply_gradients( zip(grads, self.model.variables), global_step=self.step_counter) # print the progress if self.log_type == 'traditional': # print batch progress print( self.print_formatter % tuple([epoch + 1, self.nepochs, i, data_len] + [ self.params[key] for key in self.training_params ])) elif self.log_type == 'progressbar': # update progress bar batch_time = time.time() - end processed_data_len += inputs._shape_as_list()[0] bar.suffix = self.print_formatter.format(*[ processed_data_len, data_len, data_time, batch_time, bar.elapsed_td, bar.eta_td ] + [ self.params[key] for key in self.training_params ] + [self.optimizer._learning_rate]) bar.next() end = time.time() if self.log_type == 'progressbar': bar.finish() return loss.cpu()._numpy()
reduction_indices=[1])) return loss @tfe.implicit_value_and_gradients def cal_gradient(image_batch, label_batch): return cross_entropy(softmax_model(image_batch), label_batch) if __name__ == '__main__': data = input_data.read_data_sets("data/MNIST_data/", one_hot=True) train_ds = tf.data.Dataset.from_tensor_slices((data.train.images, data.train.labels))\ .map(lambda x, y: (x, tf.cast(y, tf.float32)))\ .shuffle(buffer_size=1000)\ .batch(100)\ optimizer = tf.train.GradientDescentOptimizer(0.5) for step, (image_batch, label_batch) in enumerate(tfe.Iterator(train_ds)): loss, grads_and_vars = cal_gradient(image_batch, label_batch) optimizer.apply_gradients(grads_and_vars) print("step: {} loss: {}".format(step, loss.numpy())) model_test_output = softmax_model(data.test.images) model_test_label = data.test.labels correct_prediction = tf.equal(tf.argmax(model_test_output, 1), tf.argmax(model_test_label, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) print("test accuracy = {}".format(accuracy.numpy()))
def __init__(self): super(MNISTModel, self).__init__() self.layer1 = self.track_layer(tf.layers.Dense(units=10)) self.layer2 = self.track_layer(tf.layers.Dense(units=10)) def call(self, input): """Actually runs the model.""" result = self.layer1(input) result = self.layer2(result) return result # Let's make up a blank input image model = MNISTModel() batch = tf.zeros([1, 1, 784]) print(batch.shape) result = model(batch) print(result) def loss_function(model, x, y): y_ = model(x) return tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=y_) optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001) for (x, y) in tfe.Iterator(batch): grads = tfe.implicit_gradients(loss_function)(model, x, y) optimizer.apply_gradients(grads)
def train_one_epoch(generator, discriminator, generator_optimizer, discriminator_optimizer, dataset, step_counter, log_interval, noise_dim): """Trains `generator` and `discriminator` models on `dataset`. Args: generator: Generator model. discriminator: Discriminator model. generator_optimizer: Optimizer to use for generator. discriminator_optimizer: Optimizer to use for discriminator. dataset: Dataset of images to train on. step_counter: An integer variable, used to write summaries regularly. log_interval: How many steps to wait between logging and collecting summaries. noise_dim: Dimension of noise vector to use. """ total_generator_loss = 0.0 total_discriminator_loss = 0.0 for (batch_index, images) in enumerate(tfe.Iterator(dataset)): with tf.device('/cpu:0'): tf.assign_add(step_counter, 1) with tf.contrib.summary.record_summaries_every_n_global_steps( log_interval, global_step=step_counter): current_batch_size = images.shape[0] noise = tf.random_uniform( shape=[current_batch_size, noise_dim], minval=-1., maxval=1., seed=batch_index) with tfe.GradientTape(persistent=True) as g: generated_images = generator(noise) tf.contrib.summary.image( 'generated_images', tf.reshape(generated_images, [-1, 28, 28, 1]), max_images=10) discriminator_gen_outputs = discriminator(generated_images) discriminator_real_outputs = discriminator(images) discriminator_loss_val = discriminator_loss(discriminator_real_outputs, discriminator_gen_outputs) total_discriminator_loss += discriminator_loss_val generator_loss_val = generator_loss(discriminator_gen_outputs) total_generator_loss += generator_loss_val generator_grad = g.gradient(generator_loss_val, generator.variables) discriminator_grad = g.gradient(discriminator_loss_val, discriminator.variables) generator_optimizer.apply_gradients( zip(generator_grad, generator.variables)) discriminator_optimizer.apply_gradients( zip(discriminator_grad, discriminator.variables)) if log_interval and batch_index > 0 and batch_index % log_interval == 0: print('Batch #%d\tAverage Generator Loss: %.6f\t' 'Average Discriminator Loss: %.6f' % (batch_index, total_generator_loss / batch_index, total_discriminator_loss / batch_index))
def process_user(user_name, outfile): num_epochs = 2 batch_size = 512 data_dir = '{0}/{1}/'.format(users_indir, u) # keep results for plotting train_loss_results = [] model = getModel() for d in range(16): # training phase dataset_fname = data_dir + '{0}.txt'.format(d) # print('df:', dataset_fname) input_data, target_data, red_events = process_file(dataset_fname) print('processing:', dataset_fname, " - num events:", len(input_data), " - red events:", len(red_events)) training_dataset = tf.data.Dataset.from_tensor_slices( (input_data, target_data)) training_dataset = training_dataset.batch(batch_size) # train model on a day loss_results = train(model, training_dataset, num_epochs) train_loss_results.append(loss_results) print('loss_results:', loss_results) """ Evaluation phase """ dataset_fname = data_dir + '{0}.txt'.format(d + 1) input_data, target_data, red_events = process_file(dataset_fname) print(' evaluating:', dataset_fname, " - num events:", len(input_data), " - red events:", len(red_events)) eval_dataset = tf.data.Dataset.from_tensor_slices( (input_data, target_data)) eval_dataset = eval_dataset.batch(batch_size) line_losses = np.array([]) # eval using batches of 'batch_size' for X, y in tfe.Iterator(eval_dataset): batch_loss = loss(model, X, y) line_losses = np.append(line_losses, batch_loss) possible_anomalies = [(i, v) for i, v in enumerate(line_losses)] possible_anomalies.sort(key=lambda x: x[1], reverse=True) print(' max:', possible_anomalies[:10]) print(' red events:', [a for a, b in red_events]) # write top 10 losses to a file with the format (day, score, redevent) for i, v in possible_anomalies[:20]: red = '0' for a, b in red_events: if a == i: red = '1' break line = '{0},{1},{2}\n'.format(d, v, red) outfile.write(line) # Save model to a file model_filepath = '{0}/{1}_simple_lm.hdfs'.format(users_modeldir, user_name) tf.keras.models.save_model(model, model_filepath, overwrite=True, include_optimizer=False) model = None
example_defaults = [[0.], [0.], [0.], [0.], [0]] # sets field types parsed_line = tf.decode_csv(line, example_defaults) # First 4 fields are features, combine into single tensor features = tf.reshape(parsed_line[:-1], shape=(4,)) # Last field is the label label = tf.reshape(parsed_line[-1], shape=()) return features, label train_dataset = tf.data.TextLineDataset(train_dataset_fp) train_dataset = train_dataset.skip(1) # skip the first header row train_dataset = train_dataset.map(parse_csv) # parse each row train_dataset = train_dataset.shuffle(buffer_size=1000) # randomize train_dataset = train_dataset.batch(32) # View a single example entry from a batch features, label = tfe.Iterator(train_dataset).next() print("example features:", features[0]) print("example label:", label[0]) #adventures in ML tutorial const = tf.constant(2.0, name = 'const') #b = tf.Variable(2.0, name='b') b = tf.placeholder(tf.float32, [None, 1], name='b') c = tf.Variable(1.0, name='c') d = tf.add(b,c, name='d') e = tf.add(c, const, name = 'e') a = tf.multiply(d, e, name = 'a') # setup the variable initialisation init_op = tf.global_variables_initializer()
def main(): parser = argparse.ArgumentParser(description='TensorFlow Pascal Example') parser.add_argument('--batch-size', type=int, default=20, help='input batch size for training') parser.add_argument('--epochs', type=int, default=60, help='number of epochs to train') parser.add_argument('--lr', type=float, default=0.001, help='learning rate') parser.add_argument('--seed', type=int, default=1, help='random seed') parser.add_argument('--log-interval', type=int, default=50, help='how many batches to wait before' ' logging training status') parser.add_argument('--eval-interval', type=int, default=50, help='how many batches to wait before' ' evaluate the model') parser.add_argument('--log-dir', type=str, default='02_pascal_simplecnn_tb', help='path for logging directory') parser.add_argument('--data-dir', type=str, default='./VOCdevkit/VOC2007', help='Path to PASCAL data storage') args = parser.parse_args() util.set_random_seed(args.seed) sess = util.set_session() splt = "trainval" trainval_npz = splt + '.npz' test_npz = 'test.npz' if (os.path.isfile(trainval_npz)): print("\nFound trainval npz file\n") with np.load(trainval_npz) as tr_npzfile: train_images = tr_npzfile['imgs'] train_labels = tr_npzfile['labels'] train_weights = tr_npzfile['weights'] else: train_images, train_labels, train_weights = util.load_pascal( args.data_dir, class_names=CLASS_NAMES, split=splt) np.savez(trainval_npz, imgs=train_images, labels=train_labels, weights=train_weights) ##TEST## if (os.path.isfile(test_npz)): print("\nFound test npz file\n") # npzfile = np.load(test_npz) with np.load(test_npz) as test_npzfile: test_images = test_npzfile['imgs'] test_labels = test_npzfile['labels'] test_weights = test_npzfile['weights'] else: test_images, test_labels, test_weights = util.load_pascal( args.data_dir, class_names=CLASS_NAMES, split='test') np.savez(test_npz, imgs=test_images, labels=test_labels, weights=test_weights) ## TODO modify the following code to apply data augmentation here rgb_mean = np.array([123.68, 116.78, 103.94], dtype=np.float32) / 256.0 train_images = (train_images - rgb_mean).astype(np.float32) test_images = (test_images - rgb_mean).astype(np.float32) flip_fn = lambda img, lbl, wts: flip(img, lbl, wts) crop_fn = lambda img, lbl, wts: crop(img, lbl, wts) ccrop_fn = lambda img, lbl, wts: center_crop(img, lbl, wts) train_dataset = tf.data.Dataset.from_tensor_slices( (train_images, train_labels, train_weights)) flipped_train = train_dataset.map(flip_fn, num_parallel_calls=4) train_dataset = train_dataset.concatenate(flipped_train) train_dataset = train_dataset.map(crop_fn, num_parallel_calls=4) train_dataset = train_dataset.shuffle(10000).batch(args.batch_size) test_dataset = tf.data.Dataset.from_tensor_slices( (test_images, test_labels, test_weights)) test_dataset = test_dataset.map(ccrop_fn, num_parallel_calls=4) test_dataset = test_dataset.batch(args.batch_size) model = SimpleCNN(num_classes=len(CLASS_NAMES)) logdir = os.path.join(args.log_dir, datetime.now().strftime('%Y-%m-%d_%H-%M-%S')) if os.path.exists(logdir): shutil.rmtree(logdir) os.makedirs(logdir) writer = tf.contrib.summary.create_file_writer(logdir) writer.set_as_default() tf.contrib.summary.initialize() global_step = tf.train.get_or_create_global_step() # optimizer = tf.train.AdamOptimizer(learning_rate=args.lr) ##decay lr using callback learning_rate = tf.Variable(args.lr) # decay_interval = 5000 # decay_op = tf.train.exponential_decay(args.lr,global_step,decay_interval,0.5) ##optimizer : sgd , momentum, 0.9 # optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate, momentum=0.9) optimizer = tf.train.AdamOptimizer(learning_rate=args.lr) train_log = {'iter': [], 'loss': []} test_log = {'iter': [], 'mAP': []} checkpoint_directory = "./02_pascal_simplecnn/" if not os.path.exists(checkpoint_directory): os.makedirs(checkpoint_directory) checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=model) # pdb.set_trace() latest = tf.train.latest_checkpoint(checkpoint_directory) load_flag = 0 if (latest is not None): print("Loading checkpoint ", latest) status = checkpoint.restore( tf.train.latest_checkpoint(checkpoint_directory)) load_flag = 1 print("\nUsing eval interval: ", args.eval_interval) print("\nUsing batch size: ", args.batch_size) for ep in range(args.epochs): epoch_loss_avg = tfe.metrics.Mean() # for batch, (images, labels,weights) in enumerate(train_dataset): for (images, labels, weights) in tfe.Iterator(train_dataset): with tf.GradientTape() as tape: logits = model(images, training=True) loss_value = tf.losses.sigmoid_cross_entropy( labels, logits, weights) grads = tape.gradient(loss_value, model.trainable_variables) optimizer.apply_gradients(zip(grads, model.trainable_variables), global_step) epoch_loss_avg(loss_value) if global_step.numpy() % args.log_interval == 0: # pdb.set_trace() print( 'Epoch: {0:d}/{1:d} Iteration:{2:d} Training Loss:{3:.4f} ' .format(ep, args.epochs, global_step.numpy(), epoch_loss_avg.result())) train_log['iter'].append(global_step.numpy()) train_log['loss'].append(epoch_loss_avg.result()) with tf.contrib.summary.always_record_summaries(): tf.contrib.summary.scalar('Training loss', loss_value) tf.contrib.summary.image('Training images', images) tf.contrib.summary.scalar('Learning rate', learning_rate) for i, variable in enumerate(model.trainable_variables): tf.contrib.summary.histogram("grad_" + variable.name, grads[i]) if global_step.numpy() % args.eval_interval == 0: print("\n **** Running Eval *****\n") test_AP, test_mAP = util.eval_dataset_map(model, test_dataset) print("Eval finsished with test mAP : ", test_mAP) test_log['iter'].append(global_step.numpy()) test_log['mAP'].append(test_mAP) with tf.contrib.summary.always_record_summaries(): tf.contrib.summary.scalar('Testing mAP', test_mAP) # learning_rate.assign(tf.train.exponential_decay(args.lr, global_step, decay_interval, 0.5)()) # print("Learning rate:", learning_rate) checkpoint.save(checkpoint_prefix) ## TODO write the training and testing code for multi-label classification AP, mAP = util.eval_dataset_map(model, test_dataset) rand_AP = util.compute_ap(test_labels, np.random.random(test_labels.shape), test_weights, average=None) print('Random AP: {} mAP'.format(np.mean(rand_AP))) gt_AP = util.compute_ap(test_labels, test_labels, test_weights, average=None) print('GT AP: {} mAP'.format(np.mean(gt_AP))) print('Obtained {} mAP'.format(mAP)) print('Per class:') for cid, cname in enumerate(CLASS_NAMES): print('{}: {}'.format(cname, util.get_el(AP, cid)))
resize_side_min=256, resize_side_max=512) one_hot = tf.one_hot(data['label'], FLAGS.num_classes) return image, one_hot # Load a given dataset by name, along with the DatasetInfo # Imagenet2012: train=1,281,167 / validation=50,000 dataset = tfds.load(name="imagenet2012", split=tfds.Split.TRAIN) dataset = dataset.map(parse_fn, FLAGS.num_map_threads) dataset = dataset.shuffle(FLAGS.shuffle_buffer_size) dataset = dataset.batch(FLAGS.batch_size) dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE) #print(dataset) #for features in dataset.take(1): # print(features) #vgg = Vgg16(vgg16_npy_path="vgg16.npy") print('!!!!!!!!!!!!') for images, labels in tfe.Iterator(dataset): print(images.shape, labels.shape) #for features in dataset.take(2): # print(features['image']) #vgg.build(features['image']) #print(vgg.prob)
def main(argv): parser = argparse.ArgumentParser( description= 'Create tfrecords dataset holding patches of images specified by filename in input dataset.' ) parser.add_argument('input_dataset', type=str, help='Path to dataset holding image filenames') parser.add_argument('output_dataset', type=str, help='Path where to store the output dataset') parser.add_argument( 'patch_size', type=int, help='Patch size which to use in the preprocessed dataset') parser.add_argument('num_samples', type=int, help='Size of output dataset') parser.add_argument( 'labels', type=lambda s: [item for item in s.split(',')], help="Comma separated list of labels to find in filenames.") parser.add_argument('--image_size', type=int, dest='image_size', help='Image size for files pointed to by filename') parser.add_argument( '--no_filter', dest='no_filter', action='store_true', default=False, help='Whether to apply total image variation filtering.') parser.add_argument( '--threshold', type=float, dest='threshold', help='Threshold for filtering the samples according to variation.') parser.add_argument('--subsampling_factor', type=int, dest='subsampling_factor', default=1, help='Subsampling factor to use to downsample images.') args = parser.parse_args() labels_table = tf.contrib.lookup.index_table_from_tensor( mapping=args.labels) filename_dataset = tf.data.TFRecordDataset( args.input_dataset, num_parallel_reads=8).map(_decode_example_filename).shuffle(100000) functions = [ tf.Variable(label, name='const_' + label).value for label in args.labels ] false_fn = tf.Variable('None', name='none_label').value def _extract_label(filename): #base_size = tf.size(tf.string_split([filename],"")) #predicates = [tf.equal(base_size, tf.size(tf.string_split([tf.regex_replace(filename, "/"+ label + "/", "")]))) for label in args.labels] match = [ tf.math.reduce_any( tf.strings.regex_full_match( tf.string_split([filename], '/').values, label)) for label in args.labels ] pred_fn_pairs = list(zip(match, functions)) return tf.case(pred_fn_pairs, default=false_fn, exclusive=True) # Load images and extract the label from the filename if args.image_size is not None: images_dataset = filename_dataset.map( lambda feature: { 'image': ctfi.load(feature['filename'], channels=3, width=args.image_size, height=args.image_size), 'label': labels_table.lookup(_extract_label(feature['filename'])) }) else: images_dataset = filename_dataset.map( lambda feature: { 'image': ctfi.load(feature['filename'], channels=3), 'label': labels_table.lookup( _extract_label(feature['filename'])) }) if args.subsampling_factor > 1: images_dataset = images_dataset.map( lambda feature: { 'image': ctfi.subsample(feature['image'], args. subsampling_factor), 'label': feature['label'] }) def _filter_func_label(features): label = features['label'] result = label > -1 return result images_dataset = images_dataset.filter(_filter_func_label).shuffle(100) # Extract image patches #for sample in tfe.Iterator(images_dataset): # print(sample['label']) def _split_patches(features): patches = ctfi.extract_patches(features['image'], args.patch_size) labels = tf.expand_dims(tf.reshape(features['label'], [1]), 0) labels = tf.tile(labels, tf.stack([tf.shape(patches)[0], 1])) return (patches, labels) patches_dataset = images_dataset.map(_split_patches).apply( tf.data.experimental.unbatch()) patches_dataset = patches_dataset.map(lambda patch, label: { 'patch': patch, 'label': label }) if args.threshold is not None: threshold = args.threshold else: threshold = 0.08 num_filtered_patches = tf.Variable(0) filtered_patch_ratio = 10 # Filter function which filters the dataset after total image variation. # See: https://www.tensorflow.org/versions/r1.12/api_docs/python/tf/image/total_variation def add_background_info(sample): variation = tf.image.total_variation(sample['patch']) num_pixels = sample['patch'].get_shape().num_elements() var_per_pixel = (variation / num_pixels) no_background = var_per_pixel > threshold sample['no_background'] = no_background return sample #def true_fn(): # sample.update({'no_background': True}) # return sample #def false_fn(): # def _true_fn_lvl2(): # sample.update({'label':tf.reshape(tf.convert_to_tensor(len(args.labels), dtype=tf.int64), [1]),'no_background': True}) # return sample # def _false_fn_lvl2(): # sample.update({'no_background': False}) # return sample # pred = tf.equal(num_filtered_patches.assign_add(1) % 10, 0) # return tf.cond(pred,true_fn=_true_fn_lvl2,false_fn=_false_fn_lvl2) #return tf.cond(no_background,true_fn=true_fn, false_fn=false_fn) if args.no_filter == True: dataset = patches_dataset else: dataset = patches_dataset.map(add_background_info) filtered_elements_dataset = dataset.filter( lambda sample: tf.logical_not(sample['no_background'])) def change_label(sample): return { 'patch': sample['patch'], 'label': tf.reshape( tf.convert_to_tensor(len(args.labels), dtype=tf.int64), [1]) } filtered_elements_dataset = filtered_elements_dataset.map(change_label) filtered_dataset = dataset.filter(lambda sample: sample[ 'no_background']).map(lambda sample: { 'patch': sample['patch'], 'label': sample['label'] }) dataset = tf.data.experimental.sample_from_datasets( [filtered_dataset, filtered_elements_dataset], weights=[0.95, 0.05]) dataset = dataset.map(lambda sample: (sample['patch'], sample['label'])) dataset = dataset.take(args.num_samples).shuffle(100000) writer = tf.io.TFRecordWriter(args.output_dataset) # Make file readable for all users cutil.publish(args.output_dataset) def _encode_func(sample): patch_np = sample[0].numpy().flatten() label_np = sample[1].numpy() return ctfd.encode({ 'patch': ctf.float_feature(patch_np), 'label': ctf.int64_feature(label_np) }) # Iterate over whole dataset and write serialized examples to file. # See: https://www.tensorflow.org/versions/r1.12/api_docs/python/tf/contrib/eager/Iterator for sample in tfe.Iterator(dataset): example = _encode_func(sample) writer.write(example.SerializeToString()) # Flush and close the writer. writer.flush() writer.close()
def train_or_infer_spinn(vocab, trans, params, train_dataset, val_dataset, model_dir, embeddings_matrix): use_gpu = tfe.num_gpus() > 0 device = "gpu:0" if use_gpu else "cpu:0" print("Using device: %s" % device) train_len = params.train_size log_header = ( " Time Epoch Iteration Progress (%Epoch) Loss Dev/Loss" " Accuracy Dev/Accuracy") log_template = ( "{:>6.0f} {:>5.0f} {:>9.0f} {:>5.0f}/{:<5.0f} {:>7.0f}% {:>8.6f} {} " "{:12.4f} {}") dev_log_template = ( "{:>6.0f} {:>5.0f} {:>9.0f} {:>5.0f}/{:<5.0f} {:>7.0f}% {:>8.6f} " "{:8.6f} {:12.4f} {:12.4f}") summary_writer = tf.contrib.summary.create_file_writer(model_dir, flush_millis=10000) with tf.device(device), \ summary_writer.as_default(), \ tf.contrib.summary.always_record_summaries(): model = THANOS(params, embeddings_matrix) global_step = tf.train.get_or_create_global_step() trainer = THANOSTrainer(model, params) checkpoint = tf.train.Checkpoint(trainer=trainer, global_step=global_step) checkpoint.restore(tf.train.latest_checkpoint(model_dir)) best_save_path = model_dir + '/' + 'best_weights' start = time.time() iterations = 0 mean_loss = tfe.metrics.Mean() accuracy = tfe.metrics.Accuracy() logging.info(log_header) best_eval_acc = 0.0 for epoch in xrange(params.num_epochs): batch_idx = 0 for labels, document_sizes, sentence_lengths, sentences, transitions in tfe.Iterator( train_dataset): #print (sentences) if use_gpu: labels, document_sizes, sentence_lengths, sentences, transitions = labels.gpu( ), document_sizes.gpu(), sentence_lengths.gpu( ), sentences.gpu(), transitions.gpu() inputs = { 'labels': labels, 'document_sizes': document_sizes, 'sentence_lengths': sentence_lengths, 'sentences': sentences, 'transitions': transitions } iterations += 1 batch_train_loss, batch_train_logits = trainer.train_batch( inputs) batch_size = tf.shape(labels)[0] mean_loss(batch_train_loss.numpy(), weights=batch_size.gpu() if use_gpu else batch_size) accuracy(tf.argmax(batch_train_logits, axis=1), tf.cast(labels, tf.int64)) if iterations % params.save_every == 0: checkpoint.save(os.path.join(model_dir, "ckpt")) if iterations % params.dev_every == 0: dev_loss, dev_frac_correct = _evaluate_on_dataset( val_dataset, trainer, use_gpu) #print(dev_log_template.format( #time.time() - start, #epoch, iterations, 1 + batch_idx, train_len, #100.0 * (1 + batch_idx) / train_len, #mean_loss.result(), dev_loss, #accuracy.result() * 100.0, dev_frac_correct * 100.0)) logging.info( dev_log_template.format( time.time() - start, epoch, iterations, 1 + batch_idx, train_len, 100.0 * (1 + batch_idx) / train_len, mean_loss.result(), dev_loss, accuracy.result() * 100.0, dev_frac_correct * 100.0)) tf.contrib.summary.scalar("dev/loss", dev_loss) tf.contrib.summary.scalar("dev/accuracy", dev_frac_correct) if dev_frac_correct >= best_eval_acc: #best_save_path = model_dir+'/'+'best_weights' logging.info( "- Found new best accuracy, saving in {}".format( best_save_path)) checkpoint.save( os.path.join(best_save_path, "ckpt_best")) best_eval_acc = dev_frac_correct elif iterations % params.log_every == 0: mean_loss_val = mean_loss.result() accuracy_val = accuracy.result() logging.info( log_template.format( time.time() - start, epoch, iterations, 1 + batch_idx, train_len, 100.0 * (1 + batch_idx) / train_len, mean_loss_val, " " * 8, accuracy_val * 100.0, " " * 12)) #print(log_template.format( #time.time() - start, #epoch, iterations, 1 + batch_idx, train_len, #100.0 * (1 + batch_idx) / train_len, #mean_loss_val, " " * 8, accuracy_val * 100.0, " " * 12)) tf.contrib.summary.scalar("train/loss", mean_loss_val) tf.contrib.summary.scalar("train/accuracy", accuracy_val) batch_idx += 1 # Reset metrics. mean_loss = tfe.metrics.Mean() accuracy = tfe.metrics.Accuracy() return trainer
outf += str(learning_rate) outf += '.' outf += time.strftime("%Y-%m-%d--%H-%M") if not os.path.exists(outf): os.makedirs(outf) else: print 'There exists a same model.' exit() logfile = os.path.join(outf, 'logfile.txt') shutil.copy(os.path.realpath(__file__), os.path.join(outf, filename_script)) # data train_dataset, validation_dataset, test_dataset = dataset.uci_binary_dataset( filename=setting[2]) train_dataset = train_dataset.shuffle(shuffle_buffer).batch(batch_size) dataset_iter = tfe.Iterator(train_dataset) ''' model and loss ''' rbm = RBM(vis_dim=vis_dim, hid_dim=hid_dim) if method is 'cd': def loss_fn(v_0): v_n = rbm.cd_step(v_0, train_mc_steps) return rbm.cd_loss(v_0, v_n) elif method is 'pcd': pass else: print 'unknown method' exit()