def __init__(self, batch_size=64, shuffle=True, is_training=True, is_vectorize=False, normalize_mode='tanh'): (train_x, train_y), (test_x, test_y) = mnist.load_data() if is_training: self.x = train_x self.y = train_y else: self.x = test_x self.y = test_y self.is_training = is_training self.normalize_mode = normalize_mode if is_vectorize: # x: (N, 28, 28) → (N, 784) self.x = self.x.reshape(self.x.shape[0], self.x.shape[1] * self.x.shape[2]) else: self.x = np.expand_dims(self.x, axis=-1) # one-hot vectorize y self.y = to_categorical(self.y, 10) super().__init__(len(self.x), batch_size, shuffle, None)
def test_image_blocks(tmp_path): num_instances = 10 (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = x_train[:num_instances] y_regression = utils.generate_data(num_instances=num_instances, shape=(1, )) input_node = ak.ImageInput() output = ak.Normalization()(input_node) output = ak.ImageAugmentation()(output) outputs1 = ak.ResNetBlock(version="v2")(output) outputs2 = ak.XceptionBlock()(output) output_node = ak.Merge()((outputs1, outputs2)) output_node = ak.ClassificationHead()(output_node) automodel = ak.AutoModel( inputs=input_node, outputs=output_node, directory=tmp_path, max_trials=1, seed=utils.SEED, ) automodel.fit(x_train, y_regression, validation_data=(x_train, y_regression), epochs=1)
def get_MNIST_train_test(n_images_per_label=None, test_size=0.2, standardize=True): """MNISTデータセットを読み込む""" # 訓練用のデータとテスト用のデータを読み込む ( keras の場合 ) (X_train, y_train), (X_test, y_test) = mnist.load_data() # 小規模なデータセットを利用する場合 if n_images_per_label: assert isinstance(n_images_per_label, int) # テスト用のデータ数 = 訓練用のサイズ x test_size ( 切り捨て ) tst_size = int(n_images_per_label * test_size) # 指定個数だけデータとラベルをランダムに抽出する (X_train, y_train), (X_test, y_test) = [ get_petit_MNIST(X, y, size) for X, y, size in ( (X_train, y_train, n_images_per_label), (X_test, y_test, tst_size), ) ] # 標準化を行う if standardize: X_train, X_test = [X / 255. for X in (X_train, X_test)] return X_train, y_train, X_test, y_test
def test_io_api(tmp_path): num_instances = 100 (image_x, train_y), (test_x, test_y) = mnist.load_data() (text_x, train_y), (test_x, test_y) = utils.imdb_raw(num_instances=num_instances) image_x = image_x[:num_instances] text_x = text_x[:num_instances] structured_data_x = utils.generate_structured_data( num_instances=num_instances) classification_y = utils.generate_one_hot_labels( num_instances=num_instances, num_classes=3) regression_y = utils.generate_data(num_instances=num_instances, shape=(1, )) # Build model and train. automodel = ak.AutoModel( inputs=[ak.ImageInput(), ak.TextInput(), ak.StructuredDataInput()], outputs=[ ak.RegressionHead(metrics=['mae']), ak.ClassificationHead(loss='categorical_crossentropy', metrics=['accuracy']) ], directory=tmp_path, max_trials=2, seed=utils.SEED) automodel.fit([image_x, text_x, structured_data_x], [regression_y, classification_y], epochs=1, validation_split=0.2)
def __init__(self, num_labeled): # Number labeled examples to use for training self.num_labeled = num_labeled # Load the MNIST dataset (self.x_train, self.y_train), (self.x_test, self.y_test) = mnist.load_data() def preprocess_imgs(input_images): """ As tin says """ # Rescale [0, 255] grayscale pixel values to [-1, 1] input_images = (input_images.astype(np.float32) - 127.5) / 127.5 # Expand image dimensions to width x height x channels input_images = np.expand_dims(input_images, axis=3) return input_images def preprocess_labels(input_labels): """ As tin says""" return input_labels.reshape(-1, 1) # Training data self.x_train = preprocess_imgs(self.x_train) self.y_train = preprocess_labels(self.y_train) # Testing data self.x_test = preprocess_imgs(self.x_test) self.y_test = preprocess_labels(self.y_test)
def get_data(dataset): """ Prepare the mnist or fashion-mnist data to feed to the model. Args: dataset (string): one of the possible_datasets Returns: A tuple with the clean training data, clean testing data, and noisy testing data. """ if dataset not in possible_datasets: datasets_output = ', '.join(possible_datasets) raise ValueError('dataset must be one of: {}'.format(datasets_output)) if dataset == 'mnist': (clean_train, __), (clean_test, __) = mnist.load_data() elif dataset == 'fashion-mnist': (clean_train, __), (clean_test, __) = fashion_mnist.load_data() clean_train = clean_train.astype('float32') / 255. clean_train = clean_train.reshape(data_shape) clean_test = clean_test.astype('float32') / 255. clean_test = clean_test.reshape(data_shape) noisy_test = add_gaussian_noise_np(clean_test, 0.0, 0.4) return clean_train, clean_test, noisy_test
def get_keras_data(dataname): """Get datasets using keras API and return as a Dataset object.""" if dataname == 'cifar10_keras': train, test = cifar10.load_data() elif dataname == 'cifar100_coarse_keras': train, test = cifar100.load_data('coarse') elif dataname == 'cifar100_keras': train, test = cifar100.load_data() elif dataname == 'mnist_keras': train, test = mnist.load_data() else: raise NotImplementedError('dataset not supported') X = np.concatenate((train[0], test[0])) y = np.concatenate((train[1], test[1])) if dataname == 'mnist_keras': # Add extra dimension for channel num_rows = X.shape[1] num_cols = X.shape[2] X = X.reshape(X.shape[0], 1, num_rows, num_cols) if K.image_data_format() == 'channels_last': X = X.transpose(0, 2, 3, 1) y = y.flatten() data = Dataset(X, y) return data
def test_io_api(tmp_path): num_instances = 20 (image_x, train_y), (test_x, test_y) = mnist.load_data() text_x = utils.generate_text_data(num_instances=num_instances) image_x = image_x[:num_instances] structured_data_x = (pd.read_csv(utils.TRAIN_CSV_PATH).to_numpy().astype( np.unicode)[:num_instances]) classification_y = utils.generate_one_hot_labels( num_instances=num_instances, num_classes=3) regression_y = utils.generate_data(num_instances=num_instances, shape=(1, )) # Build model and train. automodel = ak.AutoModel( inputs=[ak.ImageInput(), ak.TextInput(), ak.StructuredDataInput()], outputs=[ ak.RegressionHead(metrics=["mae"]), ak.ClassificationHead(loss="categorical_crossentropy", metrics=["accuracy"]), ], directory=tmp_path, max_trials=2, tuner=ak.RandomSearch, seed=utils.SEED, ) automodel.fit( [image_x, text_x, structured_data_x], [regression_y, classification_y], epochs=1, validation_split=0.2, batch_size=4, )
def __init__(self, train_from_idx, train_to_idx): # input image dimensions img_rows, img_cols = 28, 28 self.num_classes = 10 # the data, shuffled and split between train and test sets (x_train, y_train), (x_test, y_test) = mnist.load_data() if K.image_data_format() == 'channels_first': x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols) x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols) self.input_shape = (1, img_rows, img_cols) else: x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1) x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1) self.input_shape = (img_rows, img_cols, 1) x_train = x_train[train_from_idx:train_to_idx, :, :, :] y_train = y_train[train_from_idx:train_to_idx] x_train = x_train.astype('float32') x_test = x_test.astype('float32') self.x_train = x_train / 255 self.x_test = x_test / 255 # convert class vectors to binary class matrices self.y_train = to_categorical(y_train, self.num_classes) self.y_test = to_categorical(y_test, self.num_classes)
def load_data(phase='train', target_size=(32, 32), normalization=None, with_label=False): (x_train, y_train), (x_test, y_test) = mnist.load_data() assert phase in ['train', 'test'] x = x_train if phase == 'train' else x_test y = y_train if phase == 'train' else y_test if target_size is not None or target_size != (28, 28): x = np.array([imresize(arr, target_size) for arr in x]) if normalization is not None: x = x.astype('float32') if normalization == 'sigmoid': x /= 255 elif normalization == 'tanh': x = (x / 255 - 0.5) * 2 else: raise ValueError x = np.expand_dims(x, -1) if with_label: return x, y else: return x
def data_mnist(): # These values are specific to MNIST img_rows = 28 img_cols = 28 nb_classes = 10 # the data, shuffled and split between train and test sets (X_train, y_train), (X_test, y_test) = mnist.load_data() if keras.backend.image_data_format() == 'th': X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols) X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols) else: X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1) X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1) X_train = X_train.astype('float32') X_test = X_test.astype('float32') X_train /= 255 X_test /= 255 print('X_train shape:', X_train.shape) print(X_train.shape[0], 'train samples') print(X_test.shape[0], 'test samples') # convert class vectors to binary class matrices Y_train = np_utils.to_categorical(y_train, nb_classes) Y_test = np_utils.to_categorical(y_test, nb_classes) from sklearn.utils import shuffle X_train, Y_train = shuffle(X_train, Y_train) return X_train, Y_train, X_test, Y_test
def __init__(self, train_from_idx, train_to_idx, included_characters): # input image dimensions img_rows, img_cols = 28, 28 self.num_classes = 10 # the data, shuffled and split between train and test sets (x_train, y_train), (x_test, y_test) = mnist.load_data() if K.image_data_format() == 'channels_first': x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols) x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols) self.input_shape = (1, img_rows, img_cols) else: x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1) x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1) self.input_shape = (img_rows, img_cols, 1) x_train = x_train[train_from_idx:train_to_idx, :, :, :] y_train = y_train[train_from_idx:train_to_idx] if not included_characters is None: train_idx = np.isin(y_train, included_characters) test_idx = np.isin(y_test, included_characters) y_train = y_train[train_idx] x_train = x_train[train_idx] y_test = y_test[test_idx] x_test = x_test[test_idx] x_train = x_train.astype('float32') x_test = x_test.astype('float32') self.x_train = x_train / 255 self.x_test = x_test / 255 self.y_train = to_categorical(y_train, self.num_classes) self.y_test = to_categorical(y_test, self.num_classes)
def train(self, epochs, batch_size=128, sample_interval=50): (X_train, _), (_, _) = mnist.load_data() X_train = X_train / 127.5 - 1. X_train = np.expand_dims(X_train, axis=3) valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) # we will first train GAN(generator + discriminator) by inputting noise and give output as valid(1) and only generator weights # will be trained(the purpose is therefore to fool the discriminator into thinking these generated images are valid). # next we will train discriminator by inputting images obtained from generator with output - fake(0) and also by images from # mnist with output - valid(1)(basically train discriminator to label images generated by generator as fake). # therefore we set up a rivalry between generator and discriminator where purpose of generator is to fool the discriminator # and purpose of discriminator is to recognise images by generator and not get fooled. for epoch in range(epochs+1): idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] # real images - inputted to discriminator for training noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) # random noise - inputted to gan(generator part) g_loss = self.combined.train_on_batch(noise, valid) # training gan gen_imgs = self.generator.predict(noise) # output from generator d_loss_real = self.discriminator.train_on_batch(imgs, valid) # discriminator trained on real images from mnist d_loss_fake = self.discriminator.train_on_batch(gen_imgs, fake) # discriminator trained on fake images from generator d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) print("%d [Discriminator loss: %f, acc.: %.2f%%] [GAN loss: %f]" % (epoch, d_loss[0], 100 * d_loss[1], g_loss)) if epoch % sample_interval == 0: self.sample_images(epoch)
def train(iterations, batch_size, sample_interval): """ Alt. train discriminator/generator for specified iterations & batch size Generate sample images (save to /data) at specified intervals """ # Load the MNIST dataset (x_train, _), (_, _) = mnist.load_data() # Rescale [0, 255] grayscale pixel values to [-1, 1] # since the Generator is tanhing outputs x_train = x_train / 127.5 - 1. x_train = np.expand_dims(x_train, axis=3) # Labels for real images: all ones real = np.ones((batch_size, 1)) # Labels for fake images: all zeros fake = np.zeros((batch_size, 1)) for iteration in range(iterations): # Train the Discriminator # ---------------------------------- # Get a random batch of real images idx = np.random.randint(0, x_train.shape[0], batch_size) imgs = x_train[idx] # Generate batch of fake images noise_vector = np.random.normal(0, 1, (batch_size, 100)) gen_imgs = generator.predict(noise_vector) # Train Discriminator d_loss_real = discriminator.train_on_batch(imgs, real) d_loss_fake = discriminator.train_on_batch(gen_imgs, fake) d_loss, accuracy = 0.5 * np.add(d_loss_real, d_loss_fake) # Train Generator # ---------------------------------- # Generate a batch of fake images noise_vector = np.random.normal(0, 1, (batch_size, 100)) gen_imgs = generator.predict(noise_vector) # Train Generator # Note that we're labeling them as real g_loss = gan.train_on_batch(noise_vector, real) if (iteration + 1) % sample_interval == 0: # Save losses and accuracies so they can be plotted after training losses.append((d_loss, g_loss)) accuracies.append(100. * accuracy) iteration_checkpoints.append(iteration + 1) # Output training progress print('{} [D loss: {}, acc.: {}], [G loss: {}'.format( iteration + 1, d_loss, 100. * accuracy, g_loss)) # Output sample of generated image sample_images(generator, iteration + 1)
def train(iterations_, batch_size_, sample_interval_): """ Alt. train Discriminator and Generator for iterations/batch_size and generate sample images at specified intervals """ # Load the MNIST dataset (x_train, y_train), (_, _) = mnist.load_data() # Rescale [0, 255] grayscale pixel values to [-1, 1] # since the Generator is tanhing outputs x_train = x_train / 127.5 - 1. x_train = np.expand_dims(x_train, axis=3) # Labels for real images: all ones real = np.ones((batch_size_, 1)) # Labels for fake images: all zeros fake = np.zeros((batch_size_, 1)) for iteration in range(iterations_): # Train the Discriminator # ---------------------------------- # Get a random batch of real images and their labels idx = np.random.randint(0, x_train.shape[0], batch_size_) imgs, labels = x_train[idx], y_train[idx] # Generate batch of fake images noise_vector = np.random.normal(0, 1, (batch_size_, 100)) gen_imgs = generator.predict([noise_vector, labels]) # Train Discriminator d_loss_real = discriminator.train_on_batch([imgs, labels], real) d_loss_fake = discriminator.train_on_batch([gen_imgs, labels], fake) d_loss, accuracy = 0.5 * np.add(d_loss_real, d_loss_fake) # Train Generator # ---------------------------------- # Generate a batch of fake images noise_vector = np.random.normal(0, 1, (batch_size_, 100)) # Get batch of random labels labels = np.random.randint(0, NUM_CLASSES, batch_size_).reshape(-1, 1) # Train Generator # Note that we're labeling them as real g_loss = cgan.train_on_batch([noise_vector, labels], real) if (iteration + 1) % sample_interval_ == 0: # Save losses and accuracies so they can be plotted after training losses.append((d_loss, g_loss)) accuracies.append(100. * accuracy) # Output training progress print('{} [D loss: {}, acc.: {}], [G loss: {}]'.format( iteration + 1, d_loss, 100. * accuracy, g_loss)) # Output sample of generated image sample_images(iteration)
def load_npz(self): path = os.path.join(self.data_dir, 'mnist.npz') train_data, test_data = load_data(path) if self.__mode == 'train': return self.sc.parallelize(zip(*train_data)) else: return self.sc.parallelize(zip(*test_data))
def loadDataset(): from tensorflow.python.keras.datasets import mnist (X_train, y_train), (X_test, y_test) = mnist.load_data() X_train = X_train.reshape([-1,28,28,1])/255. X_test = X_test.reshape([-1,28,28,1])/255. print(X_train.shape) print(X_test.shape) return X_train, X_test
def main(): img_rows, img_cols = 28, 28 input_shape = (img_rows, img_cols, 1) num_classes = 10 keras_model = build_keras_model(input_shape, num_classes) model_dir = 'experiments/' if os.path.exists(model_dir): shutil.rmtree(model_dir) save_checkpoints_steps = 1000 max_checkpoint_without_decrease = 2 config = tf.estimator.RunConfig( save_checkpoints_steps=save_checkpoints_steps, save_summary_steps=100, keep_checkpoint_max=max_checkpoint_without_decrease + 2 ) estimator = keras.estimator.model_to_estimator( keras_model, model_dir=model_dir, config=config ) print(estimator.eval_dir()) (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = x_train[:1000, :, :] y_train = y_train[:1000] x_train = np.reshape(x_train, x_train.shape + (1,)) x_test = np.reshape(x_test, x_test.shape + (1,)) # convert class vectors to binary class matrices y_train = keras.utils.to_categorical(y_train, num_classes) y_test = keras.utils.to_categorical(y_test, num_classes) early_stopping = tf.contrib.estimator.stop_if_no_decrease_hook( estimator, metric_name='loss', max_steps_without_decrease=save_checkpoints_steps * max_checkpoint_without_decrease, run_every_secs=None, run_every_steps=save_checkpoints_steps, min_steps=0) train_spec = tf.estimator.TrainSpec( tf.estimator.inputs.numpy_input_fn(x_train, y_train, shuffle=True, num_epochs=100, batch_size=10), hooks=[early_stopping] ) eval_spec = tf.estimator.EvalSpec( tf.estimator.inputs.numpy_input_fn(x_test, y_test, shuffle=True, num_epochs=1), steps=save_checkpoints_steps, start_delay_secs=0, throttle_secs=0 ) tf.estimator.train_and_evaluate( estimator, train_spec, eval_spec )
def main(argv=None): # input image dimensions img_rows, img_cols = 28, 28 # the data, split between train and test sets (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1) x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1) input_shape = (img_rows, img_cols, 1) x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255 x_test /= 255 # convert class vectors to binary class matrices y_train = tf.keras.utils.to_categorical(y_train, 10) y_test = tf.keras.utils.to_categorical(y_test, 10) model = Sequential() model.add( Conv2D(20, (5, 5), activation='relu', padding='valid', input_shape=input_shape, name='x')) model.add(MaxPooling2D(pool_size=(2, 2), strides=2)) model.add(Conv2D(40, (5, 5), activation='relu', padding='valid')) model.add(MaxPooling2D(pool_size=(2, 2), strides=2)) model.add(Flatten()) model.add(Dense(256, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(10, activation='softmax', name='probabilities')) model.compile(loss=tf.keras.losses.categorical_crossentropy, optimizer=tf.keras.optimizers.Adam(0.002), metrics=['accuracy']) model_dir = "gs://mnist-estimator/train" est_mnist = model_to_estimator(keras_model=model, model_dir=model_dir) train_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x_input": x_train}, y=y_train, batch_size=128, num_epochs=None, shuffle=True) train_spec = tf.estimator.TrainSpec(train_input_fn, max_steps=500) eval_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x_input": x_test}, y=y_test, num_epochs=1, shuffle=False) exporter = tf.estimator.FinalExporter('mnist', serving_input_fn) eval_spec = tf.estimator.EvalSpec(eval_input_fn, steps=1, exporters=[exporter]) #eval_spec = tf.estimator.EvalSpec(eval_input_fn, steps=1) tf.estimator.train_and_evaluate(est_mnist, train_spec, eval_spec)
def load_mnist(): (train_x, train_y), (test_x, test_y) = mnist.load_data() train_x = train_x.reshape(train_x.shape[0], 784).astype('float32') / 255.0 test_x = test_x.reshape(test_x.shape[0], 784).astype('float32') / 255.0 train_y = to_categorical(train_y, 10) test_y = to_categorical(test_y, 10) return (train_x, train_y), (test_x, test_y)
def mnist_dataset(batch_size): with self.skip_fetch_failure_exception(): (x_train, y_train), _ = mnist.load_data() # The `x` arrays are in uint8 and have values in the range [0, 255]. # We need to convert them to float32 with values in the range [0, 1] x_train = x_train / np.float32(255) y_train = y_train.astype(np.int64) train_dataset = dataset_ops.DatasetV2.from_tensor_slices( (x_train, y_train)).shuffle(60000).repeat().batch(batch_size) return train_dataset
def load_real_samples(): # load mnist dataset (trainX, _), (_, _) = load_data() # expand to 3d, e.g. add channels dimension X = expand_dims(trainX, axis=-1) # convert from unsigned ints to floats X = X.astype('float32') # scale from [0,255] to [0,1] X = X / 255.0 return X
def get_mnist_dataset(): (X_train, y_train), (X_test, y_test) = mnist.load_data() X_train = X_train.astype('float32') / 255 X_test = X_test.astype('float32') / 255 X_train = X_train[..., None] X_test = X_test[..., None] Y_train = keras.utils.to_categorical(y_train, 10) Y_test = keras.utils.to_categorical(y_test, 10) return (X_train, Y_train), (X_test, Y_test)
def load_data(): """ Load train and test data. """ x_train = None y_train = None x_test = None y_test = None (x_train, y_train), (x_test, y_test) = mnist.load_data() return (x_train, y_train), (x_test, y_test)
def load_dataset(): # load dataset (trainX, trainY), (testX, testY) = mnist.load_data() # reshape dataset to have a single channel trainX = trainX.reshape((trainX.shape[0], 28, 28, 1)) testX = testX.reshape((testX.shape[0], 28, 28, 1)) # one hot encode target values trainY = to_categorical(trainY) testY = to_categorical(testY) return trainX, trainY, testX, testY
def test_functional_api(tmp_dir): # Prepare the data. num_instances = 20 (image_x, train_y), (test_x, test_y) = mnist.load_data() (text_x, train_y), (test_x, test_y) = common.imdb_raw() (structured_data_x, train_y), (test_x, test_y) = common.dataframe_numpy() image_x = image_x[:num_instances] text_x = text_x[:num_instances] structured_data_x = structured_data_x[:num_instances] classification_y = common.generate_one_hot_labels( num_instances=num_instances, num_classes=3) regression_y = common.generate_data(num_instances=num_instances, shape=(1, )) # Build model and train. image_input = ak.ImageInput() output = ak.Normalization()(image_input) output = ak.ImageAugmentation()(output) outputs1 = ak.ResNetBlock(version='next')(image_input) outputs2 = ak.XceptionBlock()(image_input) image_output = ak.Merge()((outputs1, outputs2)) structured_data_input = ak.StructuredDataInput( column_names=common.COLUMN_NAMES_FROM_CSV, column_types=common.COLUMN_TYPES_FROM_CSV) structured_data_output = ak.FeatureEngineering()(structured_data_input) structured_data_output = ak.DenseBlock()(structured_data_output) text_input = ak.TextInput() outputs1 = ak.TextToIntSequence()(text_input) outputs1 = ak.EmbeddingBlock()(outputs1) outputs1 = ak.ConvBlock(separable=True)(outputs1) outputs1 = ak.SpatialReduction()(outputs1) outputs2 = ak.TextToNgramVector()(text_input) outputs2 = ak.DenseBlock()(outputs2) text_output = ak.Merge()((outputs1, outputs2)) merged_outputs = ak.Merge()( (structured_data_output, image_output, text_output)) regression_outputs = ak.RegressionHead()(merged_outputs) classification_outputs = ak.ClassificationHead()(merged_outputs) automodel = ak.GraphAutoModel( inputs=[image_input, text_input, structured_data_input], directory=tmp_dir, outputs=[regression_outputs, classification_outputs], max_trials=2, seed=common.SEED) automodel.fit((image_x, text_x, structured_data_x), (regression_y, classification_y), validation_split=0.2, epochs=2)
def __init__(self, init_learning_rate=0.1): self._init_learning_rate = init_learning_rate self._is_training = tf.placeholder(dtype=tf.bool, shape=()) self._learning_rate = tf.placeholder(dtype=tf.float32, shape=()) self._batch_size = tf.placeholder(dtype=tf.int64, shape=()) (self.x_train, self.y_train), (self.x_test, self.y_test) = mnist.load_data() self._train_dataset() self._test_dataset()
def gen_data(): (x_train, y_train), (x_test, y_test) = mnist.load_data() #NHWC is tensorflow default x_train = x_train.reshape(x_train.shape[0], 28 * 28) x_test = x_test.reshape(x_test.shape[0], 28 * 28) y_train = y_train.reshape(y_train.shape[0], 1) y_test = y_test.reshape(y_test.shape[0], 1) train_data = np.concatenate((y_train, x_train), axis=1) test_data = np.concatenate((y_test, x_test), axis=1) print(train_data.shape) return train_data, test_data
def load_minst_data(): # load the data #x_train = np.load('Data/Data_Train.npy') #y_train = np.load('Data/Data_Labels.npy') (x_train, y_train), (x_test, y_test) = mnist.load_data() #y_train = np.squeeze(y_train); # activar solo para cifar 10 #*****IMPORTANTE*****# # normalize our inputs to be in the range[-1, 1] x_train = (x_train.astype(np.float32) - 127.5) / 127.5 # convert x_train with a shape of (60000, 28, 28) to (60000, 784) so we have # 784 columns per row return (x_train, y_train)
def test_functional_api(tmp_path): # Prepare the data. num_instances = 80 (image_x, train_y), (test_x, test_y) = mnist.load_data() (text_x, train_y), (test_x, test_y) = utils.imdb_raw() (structured_data_x, train_y), (test_x, test_y) = utils.dataframe_numpy() image_x = image_x[:num_instances] text_x = text_x[:num_instances] structured_data_x = structured_data_x[:num_instances] classification_y = utils.generate_one_hot_labels( num_instances=num_instances, num_classes=3) regression_y = utils.generate_data(num_instances=num_instances, shape=(1, )) # Build model and train. image_input = ak.ImageInput() output = ak.Normalization()(image_input) output = ak.ImageAugmentation()(output) outputs1 = ak.ResNetBlock(version='next')(output) outputs2 = ak.XceptionBlock()(output) image_output = ak.Merge()((outputs1, outputs2)) structured_data_input = ak.StructuredDataInput() structured_data_output = ak.CategoricalToNumerical()(structured_data_input) structured_data_output = ak.DenseBlock()(structured_data_output) text_input = ak.TextInput() outputs1 = ak.TextToIntSequence()(text_input) outputs1 = ak.Embedding()(outputs1) outputs1 = ak.ConvBlock(separable=True)(outputs1) outputs1 = ak.SpatialReduction()(outputs1) outputs2 = ak.TextToNgramVector()(text_input) outputs2 = ak.DenseBlock()(outputs2) text_output = ak.Merge()((outputs1, outputs2)) merged_outputs = ak.Merge()( (structured_data_output, image_output, text_output)) regression_outputs = ak.RegressionHead()(merged_outputs) classification_outputs = ak.ClassificationHead()(merged_outputs) automodel = ak.AutoModel( inputs=[image_input, text_input, structured_data_input], directory=tmp_path, outputs=[regression_outputs, classification_outputs], max_trials=2, tuner=ak.Hyperband, seed=utils.SEED) automodel.fit((image_x, text_x, structured_data_x), (regression_y, classification_y), validation_split=0.2, epochs=1)
def get_input_datasets(use_bfloat16=False): """Downloads the MNIST dataset and creates train and eval dataset objects. Args: use_bfloat16: Boolean to determine if input should be cast to bfloat16 Returns: Train dataset and eval dataset. The dataset doesn't include batch dim. """ cast_dtype = dtypes.bfloat16 if use_bfloat16 else dtypes.float32 # the data, split between train and test sets (x_train, y_train), (x_test, y_test) = mnist.load_data() train_data_shape = (x_train.shape[0],) + get_data_shape() test_data_shape = (x_test.shape[0],) + get_data_shape() if backend.image_data_format() == 'channels_first': x_train = x_train.reshape(train_data_shape) x_test = x_test.reshape(test_data_shape) else: x_train = x_train.reshape(train_data_shape) x_test = x_test.reshape(test_data_shape) x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255 x_test /= 255 # convert class vectors to binary class matrices y_train = utils.to_categorical(y_train, NUM_CLASSES) y_test = utils.to_categorical(y_test, NUM_CLASSES) # train dataset train_ds = dataset_ops.Dataset.from_tensor_slices((x_train, y_train)) # TODO(rchao): Remove maybe_shard_dataset() once auto-sharding is done. train_ds = maybe_shard_dataset(train_ds) train_ds = train_ds.repeat() train_ds = train_ds.map(lambda x, y: (math_ops.cast(x, cast_dtype), y)) train_ds = train_ds.batch(64, drop_remainder=True) # eval dataset eval_ds = dataset_ops.Dataset.from_tensor_slices((x_test, y_test)) # TODO(rchao): Remove maybe_shard_dataset() once auto-sharding is done. eval_ds = maybe_shard_dataset(eval_ds) eval_ds = eval_ds.repeat() eval_ds = eval_ds.map(lambda x, y: (math_ops.cast(x, cast_dtype), y)) eval_ds = eval_ds.batch(64, drop_remainder=True) return train_ds, eval_ds
def _make_dataset(self): (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = x_train.reshape(60000, 784) x_test = x_test.reshape(10000, 784) dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train)) dataset = dataset.repeat() dataset = dataset.shuffle(self.batch_size * 3) dataset = dataset.batch(self.batch_size) def _map_fn(image, label): image = tf.to_float(image) / 255. label.set_shape([self.batch_size]) label = tf.cast(label, dtype=tf.int32) label_onehot = tf.one_hot(label, 10) image = tf.reshape(image, [self.batch_size, 28, 28, 1]) return common.ImageLabelOnehot( image=image, label=label, label_onehot=label_onehot) self.dataset = dataset.map(_map_fn)
def _Run(self, is_training, use_trt, batch_size, num_epochs, model_dir): """Train or evaluate the model. Args: is_training: whether to train or evaluate the model. In training mode, quantization will be simulated where the quantize_and_dequantize_v2 are placed. use_trt: if true, use TRT INT8 mode for evaluation, which will perform real quantization. Otherwise use native TensorFlow which will perform simulated quantization. Ignored if is_training is True. batch_size: batch size. num_epochs: how many epochs to train. Ignored if is_training is False. model_dir: where to save or load checkpoint. Returns: The Estimator evaluation result. """ # Get dataset train_data, test_data = mnist.load_data() def _PreprocessFn(x, y): x = math_ops.cast(x, dtypes.float32) x = array_ops.expand_dims(x, axis=2) x = 2.0 * (x / 255.0) - 1.0 y = math_ops.cast(y, dtypes.int32) return x, y def _EvalInputFn(): mnist_x, mnist_y = test_data dataset = data.Dataset.from_tensor_slices((mnist_x, mnist_y)) dataset = dataset.apply( data.experimental.map_and_batch( map_func=_PreprocessFn, batch_size=batch_size, num_parallel_calls=8)) dataset = dataset.repeat(count=1) iterator = dataset.make_one_shot_iterator() features, labels = iterator.get_next() return features, labels def _TrainInputFn(): mnist_x, mnist_y = train_data dataset = data.Dataset.from_tensor_slices((mnist_x, mnist_y)) dataset = dataset.shuffle(2 * len(mnist_x)) dataset = dataset.apply( data.experimental.map_and_batch( map_func=_PreprocessFn, batch_size=batch_size, num_parallel_calls=8)) dataset = dataset.repeat(count=num_epochs) iterator = dataset.make_one_shot_iterator() features, labels = iterator.get_next() return features, labels def _ModelFn(features, labels, mode): if is_training: logits_out = self._BuildGraph(features) else: graph_def = self._GetGraphDef(use_trt, batch_size, model_dir) logits_out = importer.import_graph_def( graph_def, input_map={INPUT_NODE_NAME: features}, return_elements=[OUTPUT_NODE_NAME + ':0'], name='')[0] loss = losses.sparse_softmax_cross_entropy( labels=labels, logits=logits_out) summary.scalar('loss', loss) classes_out = math_ops.argmax(logits_out, axis=1, name='classes_out') accuracy = metrics.accuracy( labels=labels, predictions=classes_out, name='acc_op') summary.scalar('accuracy', accuracy[1]) if mode == ModeKeys.EVAL: return EstimatorSpec( mode, loss=loss, eval_metric_ops={'accuracy': accuracy}) elif mode == ModeKeys.TRAIN: optimizer = AdamOptimizer(learning_rate=1e-2) train_op = optimizer.minimize(loss, global_step=get_global_step()) return EstimatorSpec(mode, loss=loss, train_op=train_op) config_proto = config_pb2.ConfigProto() config_proto.gpu_options.allow_growth = True estimator = Estimator( model_fn=_ModelFn, model_dir=model_dir if is_training else None, config=RunConfig(session_config=config_proto)) if is_training: estimator.train(_TrainInputFn) results = estimator.evaluate(_EvalInputFn) logging.info('accuracy: %s', str(results['accuracy'])) return results
from tensorflow.python.keras.optimizers import RMSprop # import memory_saving_gradients from .. module_path=os.path.dirname(os.path.abspath(__file__)) sys.path.append(module_path+'/..') import memory_saving_gradients # monkey patch Keras gradients to point to our custom version, with automatic checkpoint selection K.__dict__["gradients"] = memory_saving_gradients.gradients_memory batch_size = 128 num_classes = 10 epochs = 10 # the data, shuffled and split between train and test sets (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = x_train.reshape(60000, 784) x_test = x_test.reshape(10000, 784) x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255 x_test /= 255 print(x_train.shape[0], 'train samples') print(x_test.shape[0], 'test samples') # convert class vectors to binary class matrices y_train = tf.keras.utils.to_categorical(y_train, num_classes) y_test = tf.keras.utils.to_categorical(y_test, num_classes) model = Sequential()