def get_cifar_gen(): # get dataset (x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar100.load_data( label_mode='fine') x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255. x_test /= 255. y_train = tf.keras.utils.to_categorical(y_train) y_test = tf.keras.utils.to_categorical(y_test) print('x_train shape:', x_train.shape) print('y_train shape:', y_train.shape) # preprocess data x_train, x_test = pre_processing(x_train, x_test) datagen = ImageDataGenerator(shear_range=0.2, zoom_range=0.2, horizontal_flip=True) cifar_gen = datagen.flow(x_train, y_train, batch_size=batch_size) testgen = ImageDataGenerator() cifar_test_gen = testgen.flow(x_test, y_test, batch_size=batch_size) return cifar_gen, cifar_test_gen
def __init__(self, batch_size=64, data_aug=True): self.batch_size = batch_size (self.x_train, self.y_train), (self.x_valid, self.y_valid) = tf.keras.datasets.mnist.load_data() self.x_train = self.x_train / 127.5 - 1 self.x_valid = self.x_valid / 127.5 - 1 self.train_size = len(self.x_train) self.valid_size = len(self.x_valid) self.img_shape = self.x_train[0].shape self.x_train = self.x_train.reshape(self.train_size, self.img_shape[0], self.img_shape[1], 1) self.x_valid = self.x_valid.reshape(self.valid_size, self.img_shape[0], self.img_shape[1], 1) if data_aug: _train_gen = ImageDataGenerator(rotation_range=20, width_shift_range=0.2, height_shift_range=0.2) else: _train_gen = ImageDataGenerator() _valid_gen = ImageDataGenerator() self.train_img_gen = _train_gen.flow(self.x_train, self.y_train, batch_size=self.batch_size) self.valid_img_gen = _valid_gen.flow(self.x_valid, self.y_valid, batch_size=self.batch_size)
def augmentation_viewer(): training_file = 'data/train.p' with open(training_file, mode='rb') as f: train = pickle.load(f) image = train['features'][3432] cv2.imwrite('project_report/before_image.png', cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) data_augmentor = ImageDataGenerator( rotation_range=15, zoom_range=.15, width_shift_range=.1, height_shift_range=.1, shear_range=.15, horizontal_flip=False, vertical_flip=False, fill_mode='nearest', brightness_range=[.2, 1], ) data_augmentor.flow(image.reshape([1, 32, 32, 3]), batch_size=1, save_to_dir='project_report', save_prefix="image", save_format="jpg") plt.show()
def create_custom_iterators(cls, data, labels, valid_split): #print(f"The labels are {labels} and their dims are {labels.size}") if data.shape == 0: data = tf.constant([], shape=(0, 0, 0, 0)) if labels.shape != 0: # the list of labels is not empty scalar_classes = tf.math.argmax(labels, axis=1) #print( # f"The scalar classes are {scalar_classes} and their dims are {scalar_classes.shape}") labels = to_categorical(scalar_classes, num_classes=cls.get_default_num_classes()) else: labels = tf.constant([], shape=(0, 0, 0, 0)) # print(f"The newly structured labels are {labels} and their dims are {labels.size}") train_datagen = ImageDataGenerator(width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True, validation_split=valid_split) valid_datagen = ImageDataGenerator(validation_split=valid_split) train_iter = train_datagen.flow(data, labels, batch_size=cls._batch_size, shuffle=True, seed=123, subset="training") valid_iter = valid_datagen.flow(data, labels, batch_size=cls._batch_size, shuffle=True, seed=123, subset="validation") return train_iter, valid_iter
def augment_data(x_train, y_train, x_val, y_val, batch_size=64): train_datagen = ImageDataGenerator(rescale=1. / 255, rotation_range=40, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.2, zoom_range=0.2, horizontal_flip=True, fill_mode='nearest') val_datagen = ImageDataGenerator(rescale=1. / 255, rotation_range=40, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.2, zoom_range=0.2, horizontal_flip=True, fill_mode='nearest') train_datagen.fit(x_train) train_generator = train_datagen.flow(x_train, y_train, batch_size=batch_size) val_datagen.fit(x_val) val_generator = val_datagen.flow(x_val, y_val, batch_size=batch_size) return train_generator, val_generator
def fit(model, X_train, y_train, X_test, y_test, epochs, batch_size): img_gen = ImageDataGenerator() return model.fit_generator( img_gen.flow(X_train, y_train), steps_per_epoch=len(X_train) // batch_size, epochs=epochs, validation_data=img_gen.flow(X_test, y_test), validation_steps=len(X_test) // batch_size, verbose=1, )
def tta_prediction(model, image, n_examples): """ make a prediction for one image using test-time augmentation @param model: model for prediction @param image: input image to be predicted (nrows* ncols * nchns) @param n_exampls: @return yhat: predicted label for image (scalar int) """ # configure image data augmentation datagen = ImageDataGenerator( horizontal_flip=True, vertical_flip=True, data_format='channels_last', # rotation_range=15, ) samples = np.expand_dims(image, 0) it = datagen.flow(samples, batch_size=n_examples) yhats = model.predict_generator(it, verbose=0) summed = np.sum(yhats, axis=0) yhat = np.argmax(summed) print(yhat) print(yhats) return yhat
def augment_image(self, filename, number_mult): """ applies ImageDataGenerator and generate given number of randomly created images from the base one, which has the filename path """ # extract the path to the folder folder = '/'.join(filename.split('/')[:-1]) # read the image into a numpy array image = np.expand_dims(imread(str(filename)), 0) # create datagenetator datagen = ImageDataGenerator( rotation_range=10, zoom_range=0.1, brightness_range=[0.1, 1], width_shift_range=0.1, height_shift_range=0.1 ) datagen.fit(image) for x, val in zip(datagen.flow(image, # image we chose save_to_dir=folder, # this is where we figure out where to save save_prefix='aug', # it will save the images as 'aug_0912' some number for every new augmented image save_format='png'), range( number_mult)): # here we define a range because we want 10 augmented images otherwise it will keep looping forever I think pass
def get_slice_generator(path: str = 'data/im_data.pickle', slice_per_file: int = 128, slice_shape=(224, 224, 1), val_split: float = 0.2, batch_size: int = 32) -> tuple: # Load dataset from .pickle file dataset = load_dataset(path) # Extract slice from 3D img X, Y = generate_slice_dataset(dataset, slice_per_file, slice_shape) # Shuffle arrays p = np.random.permutation(len(X)) X, Y = X[p], Y[p] # Extract validation data size_valid = int(len(X) * val_split) X_valid, Y_valid = X[0:size_valid], Y[0:size_valid] X_train, Y_train = X[size_valid:len(X)], Y[size_valid:len(Y)] # Create generator datagen = ImageDataGenerator(width_shift_range=0.05, height_shift_range=0.05, shear_range=0.05, zoom_range=0.05, dtype=np.float16) datagen.fit(X) data_generator = datagen.flow(X_train, Y_train, batch_size) data_validation = (X_valid, Y_valid) x_train_size = len(X_train) return data_generator, data_validation, x_train_size
def generator(self,fp,imgs,path): # 이미지 증식 idg = ImageDataGenerator( # rescale=1/255, # 스케일 변경 rotation_range=40.0, # 회전 각도 width_shift_range=0.2, # 수평 방향 이동 비율 height_shift_range=0.2, # 수직 방향 이동 비율 shear_range=0.2, # 반시계 방향의 전단 강도(radian) zoom_range=0.2, # 랜덤하게 확대할 사진의 범위 지정 horizontal_flip=True, # 수평 방향으로 입력 반전 fill_mode = 'nearest' # vertical_flip=True # 수직 방향으로 입력 반전 ) class_name = fp.split('\\')[-1] for img_fp in imgs: img = load_img(fp+'\\'+img_fp) ary = img_to_array(img) ary = ary.reshape((1,)+ary.shape) i = 0 for batch in idg.flow(ary, batch_size=1, save_to_dir=f'../../data/crl_image/{path}/{class_name}', save_prefix=class_name,save_format='jpeg'): i += 1 if i > 20: break print(fp,imgs)
def createAugmentedData(training_data, training_labels): """ This is creates the augmented data. Args: training_data(numpy arrays): This is the numpy array of the training data. training_labels(numpy arrays): This is the numpy array of the training labels. Returns: complete_training_data_set(numpy array): This is the numpy array of the total training data, which is has undergone augmentation. complete_training_labels_set(numpy array): This is the numpy array of the total training labels, which is has undergone augmentation. """ complete_training_data_set = [] complete_training_labels_set = [] for data in training_data: complete_training_data_set.append(data) print("Complete Training Data: " + str(len(complete_training_data_set))) for label in training_labels: complete_training_labels_set.append(label) print("Complete Training Label: " + str(len(complete_training_labels_set))) # create augmented data data_augmented = ImageDataGenerator(featurewise_center=True, featurewise_std_normalization=True, rotation_range=90, width_shift_range=0.2, height_shift_range=0.2, horizontal_flip=True, vertical_flip=True) data_augmented.fit(training_data) training_data_size = training_data.shape[0] aug_counter = 0 while aug_counter < (augmented_multiple - 1): iterator = data_augmented.flow(training_data, training_labels, batch_size=training_data_size) # iterator = data_augmented.flow(training_data, training_labels, batch_size=batch_size) augmented_data = iterator.next() for data in augmented_data[0]: complete_training_data_set.append(data) for label in augmented_data[1]: complete_training_labels_set.append(label) aug_counter += 1 print("Size of All Training Data: " + str(len(complete_training_data_set))) print("Size of All Training Labels: " + str(len(complete_training_labels_set))) array_training_data = np.array(complete_training_data_set) array_training_labels = np.array(complete_training_labels_set) print("Shape of complete training data: " + str(array_training_data.shape)) print("Shape of complete training labels: " + str(array_training_labels.shape)) return np.array(complete_training_data_set), np.array( complete_training_labels_set)
def _train(self): x_train, y_train = self.train_data x_test, y_test = self.test_data aug_gen = ImageDataGenerator( featurewise_center=False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization= False, # divide inputs by std of the dataset samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening rotation_range= 0, # randomly rotate images in the range (degrees, 0 to 180) width_shift_range= 0.1, # randomly shift images horizontally (fraction of total width) height_shift_range= 0.1, # randomly shift images vertically (fraction of total height) horizontal_flip=True, # randomly flip images vertical_flip=False, # randomly flip images ) aug_gen.fit(x_train) gen = aug_gen.flow(x_train, y_train, batch_size=self.config['batch_size']) self.model.fit_generator(generator=gen, steps_per_epoch=50000 // self.config['batch_size'], epochs=self.config['epochs'], validation_data=None) # loss, accuracy _, accuracy = self.model.evaluate(x_test, y_test, verbose=0) return TrainingResult(timesteps_this_iter=10, mean_accuracy=accuracy)
def main(net, epochs, batch_size): config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) set_session(sess) (x_train, y_train), (x_test, y_test) = cifar10.load_data() x_train, x_test = x_train.astype('float32') / 255, x_test.astype( 'float32') / 255 mean = np.mean(x_train, axis=0) x_train -= mean x_test -= mean datagen = ImageDataGenerator( width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True, ) datagen.fit(x_train) model = make_resnet(net) model.summary() model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size), validation_data=(x_test, y_test), epochs=epochs, callbacks=[ ReduceLROnPlateau(verbose=1, patience=20), TensorBoard(observer.dir) ])
def data_augmentation(in_dir, out_dir, result_images): images_list = [os.path.join(in_dir, f) for f in os.listdir(in_dir)] num_images_from_one = math.ceil(result_images / len(images_list)) aug = ImageDataGenerator(rotation_range=30, zoom_range=0.15, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.15, horizontal_flip=True, fill_mode="nearest") for (image_num, image_file) in enumerate(images_list): image = load_img(image_file) image = img_to_array(image) image = np.expand_dims(image, axis=0) imageGen = aug.flow(image, batch_size=1, save_to_dir=out_dir, save_prefix="image" + str(image_num), save_format="jpg") total = 0 for image in imageGen: total += 1 if total == num_images_from_one: break
def evaluate(model, X_train, y_train): img_gen = ImageDataGenerator() # evaluate the model scores = model.evaluate_generator(img_gen.flow(X_train, y_train), workers=4, verbose=1) print("%s: %.2f" % (model.metrics_names[1], scores[1]))
def evaluate_on_cifar10(): total_depth = 36 n_blocks = 3 basic_block_count = total_depth // n_blocks # region Model input_layer = Input(shape=[32, 32, 3]) layer = input_layer kernel_initializer = ResBlock2D.get_fixup_initializer(total_depth) for k in range(n_blocks): strides = 2 if k < (n_blocks - 1) else 1 layer = ResBlock2D(filters=16 * (2**k), basic_block_count=basic_block_count, strides=strides, kernel_initializer=kernel_initializer, use_residual_bias=True)(layer) if k == (n_blocks - 1): layer = AveragePooling2D(pool_size=8)(layer) layer = Flatten()(layer) layer = Dense(units=10, activation="softmax")(layer) model = Model(inputs=input_layer, outputs=layer) model.summary() model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["acc"]) # endregion # region Data (x_train, y_train), (x_test, y_test) = cifar10.load_data() x_train = x_train.astype(np.float32) / 255.0 x_test = x_test.astype(np.float32) / 255.0 y_train = to_categorical(y_train, num_classes=10) y_test = to_categorical(y_test, num_classes=10) generator = ImageDataGenerator(rotation_range=15, width_shift_range=5. / 32, height_shift_range=5. / 32, horizontal_flip=True) generator.fit(x_train, seed=0) # endregion log_dir = "../logs/tests/res_block_cifar10/{}".format(int(time())) log_dir = os.path.normpath(log_dir) tensorboard = TensorBoard(log_dir=log_dir, profile_batch=0) model.fit_generator(generator.flow(x_train, y_train, batch_size=64), steps_per_epoch=100, epochs=300, validation_data=(x_test, y_test), validation_steps=100, verbose=1, callbacks=[tensorboard])
def train_generator(x, y, batch_size, shift_fraction=0.): train_datagen = ImageDataGenerator( width_shift_range=shift_fraction, height_shift_range=shift_fraction) generator = train_datagen.flow(x, y, batch_size=batch_size) while 1: x_batch, y_batch = generator.next() yield ([x_batch, y_batch], [y_batch, x_batch])
def augment_data(x_train, y_train, x_val, y_val, batch_size=128): train_datagen = ImageDataGenerator(rescale=1. / 255, rotation_range=15, horizontal_flip=True) val_datagen = ImageDataGenerator(rescale=1. / 255, rotation_range=15, horizontal_flip=True) train_datagen.fit(x_train) train_generator = train_datagen.flow(x_train, y_train, batch_size=batch_size) val_datagen.fit(x_val) val_generator = val_datagen.flow(x_val, y_val, batch_size=batch_size) return train_generator, val_generator
def get_train_generator(self, batch_size): """ Returns a batch generator which transforms chunk of raw images into numpy matrices and then "yield" them for the classifier. Doing so allow to greatly optimize memory usage as the images are processed then deleted by chunks (defined by batch_size) instead of preprocessing them all at once and feeding them to the classifier. :param batch_size: int The batch size :return: generator The batch generator """ # Image Augmentation datagen = ImageDataGenerator( rescale=1. / 255, shear_range=0.2, horizontal_flip=True, vertical_flip=True) # randomly flip images horizontally loop_range = len(self.X_train) while True: for i in range(loop_range): start_offset = batch_size * i # The last remaining files could be smaller than the batch_size range_offset = min(batch_size, loop_range - start_offset) # If we reached the end of the list then we break the loop if range_offset <= 0: break batch_features = np.zeros((range_offset, *self.img_resize, 3)) batch_labels = np.zeros((range_offset, len(self.y_train[0]))) for j in range(range_offset): # Maybe shuffle the index? img = Image.open(self.X_train[start_offset + j]) img.thumbnail(self.img_resize) # Augment the image `img` here # Convert to RGB and normalize img_array = np.asarray(img.convert("RGB"), dtype=np.float32) img_array = img_array[:, :, ::-1] # Zero-center by mean pixel img_array[:, :, 0] -= 103.939 img_array[:, :, 1] -= 116.779 img_array[:, :, 2] -= 123.68 batch_features[j] = img_array batch_labels[j] = self.y_train[start_offset + j] # Augment the images (using Keras allow us to add randomization/shuffle to augmented images) # Here the next batch of the data generator (and only one for this iteration) # is taken and returned in the yield statement yield next( datagen.flow(batch_features, batch_labels, range_offset))
def prepare_training_data(self, images: ndarray, labels: ndarray) -> Tuple[Any, int]: train_datagen = ImageDataGenerator() steps = self.steps if isinstance( self.steps, int) else len(images) / self.batch_size return train_datagen.flow(images, labels, batch_size=self.batch_size, shuffle=True), steps
def get_gen(set_name, batch_size, translate, scale, shuffle=True): if set_name == 'train': (X, Y), _ = get_mnist_dataset() elif set_name == 'test': _, (X, Y) = get_mnist_dataset() image_gen = ImageDataGenerator(zoom_range=scale, width_shift_range=translate, height_shift_range=translate) gen = image_gen.flow(X, Y, batch_size=batch_size, shuffle=shuffle) return gen
def image_augmentation(img, label, augnum): #num means batch_size #label = label.reshape((1,)+label.shape) image_datagen = ImageDataGenerator(rotation_range=0.2, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.2, zoom_range=0.2, fill_mode='nearest') label_datagen = ImageDataGenerator(rotation_range=0.2, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.2, zoom_range=0.2, fill_mode='nearest') seed = random.randint(1, 10000) n = 0 seed_ = 1 image_datagen.fit(img, seed=seed_) label_datagen.fit(label, seed=seed_) for batch in image_datagen.flow(img, batch_size=1, save_to_dir=IMG_PATH, save_prefix='aug', save_format='png', seed=seed): n += 1 if n > augnum: break n = 0 for batch in label_datagen.flow(label, batch_size=1, save_to_dir=LABEL_PATH, save_prefix='aug_label', save_format='png', seed=seed): n += 1 if n > augnum: break return '''
def image_augmentation(img, label, augnum): #num means batch_size image_datagen = ImageDataGenerator(rotation_range=0.2, width_shift_range=0.1, height_shift_range=0.1, shear_range=0.1, zoom_range=0.1, fill_mode='nearest') label_datagen = ImageDataGenerator(rotation_range=0.2, width_shift_range=0.1, height_shift_range=0.1, shear_range=0.1, zoom_range=0.1, fill_mode='nearest') #random.seed(1) seed = random.randint(1, 100000) n = 0 for batch in image_datagen.flow(img, batch_size=1, save_to_dir=path, save_prefix='aug', save_format='png', seed=seed): n += 1 if n > augnum: break n = 0 for batch in image_datagen.flow(label, batch_size=1, save_to_dir=path_, save_prefix='aug_label', save_format='png', seed=seed): n += 1 if n > augnum: break return
def img_and_mask_generator(x, y, batch_size=1, shuffle=True): """ Create a generator of two combined ImageDataGenerators for input and ground truth images without any data augmentation except scaling. """ data_gen_args = dict(rescale=1. / 255) image_data_generator = ImageDataGenerator(**data_gen_args) mask_data_generator = ImageDataGenerator(**data_gen_args) seed = 1 if isinstance(x, np.ndarray): image_gen = image_data_generator.flow(x, batch_size=batch_size, seed=seed, shuffle=shuffle) mask_gen = mask_data_generator.flow(y, batch_size=batch_size, seed=seed, shuffle=shuffle) else: image_gen = image_data_generator.flow_from_directory( x, batch_size=batch_size, seed=seed, shuffle=shuffle, class_mode=None, color_mode="grayscale", target_size=(465, 381)) mask_gen = mask_data_generator.flow_from_directory( y, batch_size=batch_size, seed=seed, shuffle=shuffle, class_mode=None, color_mode="grayscale", target_size=(465, 381)) return zip(image_gen, mask_gen)
def get_generator(self): """get generators for training set and validation set""" generator = ImageDataGenerator(rotation_range=90, horizontal_flip=True, vertical_flip=True, fill_mode='reflect') # preprocessing self.binarize_mask_edge() self.imgs = self.add_axis(self.imgs, repeat=True) self.masks = self.add_axis(self.masks) self.edges = self.add_axis(self.edges) seed = 66 # split crops imgs_tr, imgs_val, masks_tr, masks_val, edges_tr, edges_val\ = train_test_split(self.imgs, self.masks, self.edges, test_size=self.SPLIT_RATE, random_state=seed) self.num_train = len(imgs_tr) self.num_val = len(imgs_val) # feed generator with the corresponding data gene_img = generator.flow(imgs_tr, batch_size=self.batch_size, seed=seed) gene_mask = generator.flow(masks_tr, batch_size=self.batch_size, seed=seed) gene_edge = generator.flow(edges_tr, batch_size=self.batch_size, seed=seed) out_gene = zip(gene_mask, gene_edge) train_generator = zip(gene_img, out_gene) # # Visualize augmented crops for debugging # for i in range(10): # img = gene_img.next()[i][:, :, 0] # mask = gene_mask.next()[i][:, :, 0] # f1 = plt.subplot(211) # plt.imshow(img, 'gray') # f2 = plt.subplot(212) # plt.imshow(mask, 'gray') # plt.show() gene_img = generator.flow(imgs_val, batch_size=self.batch_size, seed=seed) gene_mask = generator.flow(masks_val, batch_size=self.batch_size, seed=seed) gene_edge = generator.flow(edges_val, batch_size=self.batch_size, seed=seed) out_gene = zip(gene_mask, gene_edge) val_generator = zip(gene_img, out_gene) return train_generator, val_generator
def fit_and_evaluate(train_x, val_x, train_y, val_y): model = None gc.collect() model = createModel(train_x) batch_size = 32 epochs = 30 gc.collect() datagen = ImageDataGenerator(zoom_range = 0.2,horizontal_flip = False) datagen.fit(train_x) gc.collect() results = model.fit_generator(datagen.flow(train_x,train_y,batch_size=batch_size), epochs = epochs,callbacks=[early_stopping, model_checkpoint], verbose=1,validation_data = (val_x,val_y)) gc.collect() print("Val Score: ", model.evaluate(val_x, val_y)) return model, results
def shift(samples, imagePath, augPath, label, iImg, iAug): print("[INFO] Saving horizontal and vertical shifting images...") datagen = ImageDataGenerator(width_shift_range=[-200, 200]) # prepare iterator it = datagen.flow(samples, batch_size=1) # generate samples for i in range(6): # generate batch of images batch = it.next() # convert to unsigned integers for viewing newImage = batch[0].astype('uint8') saveImage(imagePath, newImage, augPath, label, iImg, iAug) iAug += 1 datagen = ImageDataGenerator(height_shift_range=0.5) # prepare iterator it = datagen.flow(samples, batch_size=1) # generate samples for i in range(6): # generate batch of images batch = it.next() # convert to unsigned integers for viewing newImage = batch[0].astype('uint8') saveImage(imagePath, newImage, augPath, l, iImg, iAug) iAug += 1 return iAug, iImg
def classify(model, img_dir): X, all_files = load_data_to_test(img_dir) total = len(all_files) ans = {} test_datagen = ImageDataGenerator(rescale=1. / 255) test_generator = test_datagen.flow(X, batch_size=IMG_BATCHS_NUMBER, shuffle=None, save_to_dir=None) pred = model.predict_generator(test_generator, steps=total // IMG_BATCHS_NUMBER) for i in range(0, pred.shape[0], 1): print(pred[i]) ans[all_files[i]] = np.argmax(pred[i]) print(ans) return ans
def fit_and_evaluate(X_train,y_train): model = None gc.collect() model = createModel(X_train) batch_size = 32 epochs = 30 gc.collect() datagen = ImageDataGenerator(zoom_range = 0.2,horizontal_flip = False) datagen.fit(X_train) gc.collect() train_x, val_x, train_y, val_y = train_test_split(X_train, y_train, test_size=0.1, random_state = np.random.randint(1,1000, 1)[0]) plotCategories(train_y,val_y) results = model.fit_generator(datagen.flow(train_x,train_y,batch_size=batch_size), epochs = epochs,steps_per_epoch = X_train.shape[0] // batch_size ,callbacks=[early_stopping, model_checkpoint], verbose=1,validation_data = (val_x,val_y)) gc.collect() print("Val Score: ", model.evaluate(val_x, val_y)) return
def train_model(model, x, y, task): date = datetime.now().strftime("%d-%m-%Y-%H-%M-%S") checkpoints_path = f'/opt/ml/model/checkpoints/{task}/{date}' checkpoints_file_format = 'checkpoints.{epoch:02d}-{val_loss:.2f}.hdf5' checkpoint_callback = ModelCheckpoint( posixpath.join(checkpoints_path, checkpoints_file_format)) L = len(x) split_ration = 0.8 train_indexes = random.sample(range(L), int(L * split_ration)) validation_indexes = list(set(range(L)) - set(train_indexes)) x_train = x[train_indexes] y_train = y[train_indexes] x_val = x[validation_indexes] y_val = y[validation_indexes] generator = ImageDataGenerator(rotation_range=30, zoom_range=0.20, fill_mode="nearest", shear_range=0.20, horizontal_flip=False, width_shift_range=0.2, height_shift_range=0.2) lr_schedule = ExponentialDecay(initial_learning_rate=0.01, decay_steps=100, decay_rate=0.9, staircase=False) model.compile(loss='categorical_crossentropy', optimizer=Adam(learning_rate=lr_schedule), metrics=['accuracy']) model.fit(generator.flow(x_train, y_train, batch_size=32), steps_per_epoch=len(x_train) // 32, verbose=2, epochs=500, shuffle=True, validation_data=(x_val, y_val), callbacks=[ WandbCallback(), checkpoint_callback, LearningRateCallback(model) ]) model.save('/opt/ml/model')
def _train(self): x_train, y_train = self.train_data x_test, y_test = self.test_data aug_gen = ImageDataGenerator( # set input mean to 0 over the dataset featurewise_center=False, # set each sample mean to 0 samplewise_center=False, # divide inputs by dataset std featurewise_std_normalization=False, # divide each input by its std samplewise_std_normalization=False, # apply ZCA whitening zca_whitening=False, # randomly rotate images in the range (degrees, 0 to 180) rotation_range=0, # randomly shift images horizontally (fraction of total width) width_shift_range=0.1, # randomly shift images vertically (fraction of total height) height_shift_range=0.1, # randomly flip images horizontal_flip=True, # randomly flip images vertical_flip=False, ) aug_gen.fit(x_train) gen = aug_gen.flow( x_train, y_train, batch_size=self.config["batch_size"]) self.model.fit_generator( generator=gen, steps_per_epoch=50000 // self.config["batch_size"], epochs=self.config["epochs"], validation_data=None) # loss, accuracy _, accuracy = self.model.evaluate(x_test, y_test, verbose=0) return {"mean_accuracy": accuracy}