def _train(self): x_train, y_train = self.train_data x_test, y_test = self.test_data aug_gen = ImageDataGenerator( # set input mean to 0 over the dataset featurewise_center=False, # set each sample mean to 0 samplewise_center=False, # divide inputs by dataset std featurewise_std_normalization=False, # divide each input by its std samplewise_std_normalization=False, # apply ZCA whitening zca_whitening=False, # randomly rotate images in the range (degrees, 0 to 180) rotation_range=0, # randomly shift images horizontally (fraction of total width) width_shift_range=0.1, # randomly shift images vertically (fraction of total height) height_shift_range=0.1, # randomly flip images horizontal_flip=True, # randomly flip images vertical_flip=False, ) aug_gen.fit(x_train) gen = aug_gen.flow( x_train, y_train, batch_size=self.config["batch_size"]) self.model.fit_generator( generator=gen, steps_per_epoch=50000 // self.config["batch_size"], epochs=self.config["epochs"], validation_data=None) # loss, accuracy _, accuracy = self.model.evaluate(x_test, y_test, verbose=0) return TrainingResult(timesteps_this_iter=10, mean_accuracy=accuracy)
def evaluate(self, path, transfer_model='Inception'): if transfer_model in ['Inception', 'Xception', 'Inception_Resnet']: target_size = (299, 299) else: target_size = (224, 224) generator_test = ImageDataGenerator().flow_from_directory( directory=path, target_size=target_size, shuffle=False) #results = model.predict_generator(generator=generator_test) self.test_results = self.model.evaluate_generator( generator=generator_test) print('Accuracy of', self.test_results[1] * 100, '%')
def _train(self): x_train, y_train = self.train_data x_test, y_test = self.test_data aug_gen = ImageDataGenerator( # set input mean to 0 over the dataset featurewise_center=False, # set each sample mean to 0 samplewise_center=False, # divide inputs by dataset std featurewise_std_normalization=False, # divide each input by its std samplewise_std_normalization=False, # apply ZCA whitening zca_whitening=False, # randomly rotate images in the range (degrees, 0 to 180) rotation_range=0, # randomly shift images horizontally (fraction of total width) width_shift_range=0.1, # randomly shift images vertically (fraction of total height) height_shift_range=0.1, # randomly flip images horizontal_flip=True, # randomly flip images vertical_flip=False, ) aug_gen.fit(x_train) gen = aug_gen.flow( x_train, y_train, batch_size=self.config["batch_size"]) self.model.fit_generator( generator=gen, steps_per_epoch=50000 // self.config["batch_size"], epochs=self.config["epochs"], validation_data=None) # loss, accuracy _, accuracy = self.model.evaluate(x_test, y_test, verbose=0) return {"mean_accuracy": accuracy}
def read_image(self, image_paths): training_path = os.path.join(image_paths, 'By_datasetMK', '201907', '20190724_training_data') data_generation = ImageDataGenerator(validation_split=0.3, rescale=1. / 255) train_generator = data_generation.flow_from_directory( directory=training_path, target_size=(self.params.width, self.params.height), batch_size=self.params.batch_size, class_mode='categorical', shuffle=True, subset='training') validation_generator = data_generation.flow_from_directory( directory=training_path, target_size=(self.params.width, self.params.height), batch_size=self.params.batch_size, class_mode='categorical', shuffle=True, subset='validation') return train_generator, validation_generator
def step(self): x_train, y_train = self.train_data x_train, y_train = x_train[:NUM_SAMPLES], y_train[:NUM_SAMPLES] x_test, y_test = self.test_data x_test, y_test = x_test[:NUM_SAMPLES], y_test[:NUM_SAMPLES] aug_gen = ImageDataGenerator( # set input mean to 0 over the dataset featurewise_center=False, # set each sample mean to 0 samplewise_center=False, # divide inputs by dataset std featurewise_std_normalization=False, # divide each input by its std samplewise_std_normalization=False, # apply ZCA whitening zca_whitening=False, # randomly rotate images in the range (degrees, 0 to 180) rotation_range=0, # randomly shift images horizontally (fraction of total width) width_shift_range=0.1, # randomly shift images vertically (fraction of total height) height_shift_range=0.1, # randomly flip images horizontal_flip=True, # randomly flip images vertical_flip=False, ) aug_gen.fit(x_train) batch_size = self.config.get("batch_size", 64) gen = aug_gen.flow(x_train, y_train, batch_size=batch_size) self.model.fit_generator(generator=gen, epochs=self.config.get("epochs", 1), validation_data=None) # loss, accuracy _, accuracy = self.model.evaluate(x_test, y_test, verbose=0) return {"mean_accuracy": accuracy}
def preprocess_images(self): self.training_datagen = ImageDataGenerator(rescale=1. / 255, shear_range=0.2, zoom_range=0.2, horizontal_flip=True) self.test_datagen = ImageDataGenerator(rescale=1. / 255) self.training_generator = self.training_datagen.flow_from_directory( os.path.join(sys.path[0], Train.TRAIN_DATA), target_size=(Train.HEIGHT, Train.WIDTH), batch_size=Train.BATCH_SIZE, class_mode='categorical') self.validation_generator = self.test_datagen.flow_from_directory( os.path.join(sys.path[0], Train.VALIDATION_DATA), target_size=(Train.HEIGHT, Train.WIDTH), batch_size=Train.BATCH_SIZE, class_mode='categorical') return (self.training_generator, self.validation_generator)
def preprocess(train_path, test_path): df = pd.read_csv(train_path + 'labels.csv', sep = '\t') df = df.drop(columns = [df.columns[0]]).drop(columns = [df.columns[2]]) df['eye_color'] = df['eye_color'].apply(str) df2 = pd.read_csv(test_path + 'labels.csv', sep = '\t') df2 = df2.drop(columns = [df2.columns[0]]).drop(columns = [df2.columns[2]]) df2['eye_color'] = df2['eye_color'].apply(str) training, testing = np.split(df.sample(frac=1), [int(0.9*len(df)),]) #splitting at n-array img = (train_path + 'img') img2 = (test_path + 'img') # set up data generator data_generator = ImageDataGenerator( rescale = 1./255., validation_split = 0.2, horizontal_flip=True, vertical_flip=True ) # Get batches of training dataset from the dataframe print("Training Dataset Preparation: ") train_generator = data_generator.flow_from_dataframe( dataframe = training, directory = img, x_col = "file_name", y_col = "eye_color", class_mode = 'categorical', target_size = (64,64), batch_size = 128, subset = 'training') # Get batches of validation dataset from the dataframe print("\nValidation Dataset Preparation: ") validation_generator = data_generator.flow_from_dataframe( dataframe = training, directory = img , x_col = "file_name", y_col = "eye_color", class_mode = 'categorical', target_size = (64,64), batch_size = 128, subset = 'validation') return train_generator, validation_generator, data_generator, df2, img2
def train(model_file, train_path, validation_path, target_size=(256, 256), num_classes=5, steps=32, num_epochs=28): if os.path.exists(model_file): print('\n*** existing model found at {}. Loading. ***\n\n'.format( model_file)) model = load_existing(model_file) else: print("\n*** Creating new model ***\n\n") model = create_model(num_classes=num_classes) check_point = ModelCheckpoint(model_file, period=1) model.compile(optimizer='rmsprop', loss='categorical_crossentropy') train_datagen = ImageDataGenerator( rescale=1. / 255, shear_range=0.3, zoom_range=0.3, # horizontal_flip=True, rotation_range=20, width_shift_range=0.2, height_shift_range=0.2, brightness_range=(0.8, 1.2)) test_datagen = ImageDataGenerator(rescale=1. / 255) train_generator = train_datagen.flow_from_directory( train_path, target_size=target_size, batch_size=32, class_mode='categorical') validation_generator = test_datagen.flow_from_directory( validation_path, target_size=target_size, batch_size=32, class_mode='categorical') model.fit_generator(train_generator, steps_per_epoch=steps, epochs=num_epochs, callbacks=[ check_point, ], validation_data=validation_generator, validation_steps=50) for layer in model.layers[:249]: layer.trainable = False for layer in model.layers[249:]: layer.trainable = True model.compile(optimizer=SGD(lr=0.00001, momentum=0.9), loss='categorical_crossentropy') model.fit_generator(train_generator, steps_per_epoch=steps, epochs=num_epochs, callbacks=[check_point], validation_data=validation_generator, validation_steps=50)
def save_bottleneck_features(): # build the Inception V3 network model = inception_v3.InceptionV3(include_top=False, weights='imagenet', input_tensor=None, input_shape=None, pooling='avg') # Save the bottleneck features for the training data set datagen = ImageDataGenerator( preprocessing_function=inception_v3.preprocess_input) generator = datagen.flow_from_directory(train_data_dir, target_size=(img_width, img_height), batch_size=batch_size, class_mode='sparse', shuffle=False) features = model.predict_generator(generator, nb_train_samples // batch_size) labels = np.eye(generator.num_classes, dtype='uint8')[generator.classes] labels = labels[0:(nb_train_samples // batch_size) * batch_size] np.save(open(output_dir + 'bottleneck_features_train.npy', 'wb'), features) np.save(open(output_dir + 'bottleneck_labels_train.npy', 'wb'), labels) # Save the bottleneck features for the validation data set generator = datagen.flow_from_directory(validation_data_dir, target_size=(img_width, img_height), batch_size=batch_size, class_mode=None, shuffle=False) features = model.predict_generator(generator, nb_validation_samples // batch_size) labels = np.eye(generator.num_classes, dtype='uint8')[generator.classes] labels = labels[0:(nb_validation_samples // batch_size) * batch_size] np.save(open(output_dir + 'bottleneck_features_validation.npy', 'wb'), features) np.save(open(output_dir + 'bottleneck_labels_validation.npy', 'wb'), labels)
def __init__(self, root_dir, dset_dir, image_format, batch_size, new_shape=128, res_shape=156, channels=3, num_classes=10, shuffle=True, statistics=None): self.root_dir = root_dir if not os.path.exists(self.root_dir): self.download_files() self.dset_dir = dset_dir self.image_format = image_format self.batch_size = batch_size self.res_shape = res_shape self.new_shape = new_shape self.channels = channels self.num_classes = num_classes self.shuffle = shuffle self.augmenter = ImageDataGenerator(horizontal_flip=True) self.image_filenames = [] self.class_mapping = {} self.labels = [] self.get_image_filenames() if statistics is None: X = self.retrieve_set() self.statistics = self.extract_statistics(X) else: self.statistics = statistics self.on_epoch_end()
def train_classifier(train_gt, train_img_dir, fast_train=False): X, y = load_data_to_train(train_gt, train_img_dir, fast_train) model = get_model() if fast_train: model.fit(X, y, batch_size=2, epochs=1) else: total = len(train_gt) train_datagen = ImageDataGenerator( rescale=1. / 255, rotation_range=15, # max угол поворота img width_shift_range=0.15, height_shift_range=0.15, zoom_range=0.15, horizontal_flip=True, fill_mode='nearest' ) # заполение пикселей за пределами(aaaaaaaa|abcd|dddddddd) X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.15, random_state=42) train_generator = train_datagen.flow( X_train, y_train, batch_size= IMG_BATCHS_NUMBER, # размер выборки(число прочитанных изображений за 1 раз) shuffle=True, save_to_dir=None) model.fit_generator( train_generator, steps_per_epoch=2500, # используем каждое изображение ровно один раз epochs=EPOCHS_NUMBER, verbose=1, # информация validation_data=(X_val, y_val)) # данные проверки model.save('birds_model.hdf5')
def get_datagen(data_dir=DATA_DIR): datagen = ImageDataGenerator(rescale=1. / 255, shear_range=0.2, zoom_range=0.2, horizontal_flip=True, vertical_flip=True, validation_split=0.3) # set validation split train_gen = datagen.flow_from_directory(data_dir, target_size=IMAGE_SIZE, color_mode=COLOR_MODE, class_mode=CLASS_MODE, batch_size=BATCH_SIZE, shuffle=True, seed=SEED, save_to_dir=None, save_prefix='', save_format='jpg', follow_links=False, subset='training', interpolation='nearest') val_gen = datagen.flow_from_directory( data_dir, # same directory as training data target_size=IMAGE_SIZE, color_mode=COLOR_MODE, class_mode=CLASS_MODE, batch_size=BATCH_SIZE, shuffle=True, seed=SEED, save_to_dir=None, save_prefix='', save_format='jpg', follow_links=False, subset='validation', interpolation='nearest') return (train_gen, val_gen), train_gen.class_indices
def main(): dataset = 'new' # Which dataset to use batch_size = 12 epochs = 50 model = net.create_model() current_dir = os.path.dirname(__file__) training_dir = os.path.join(current_dir, 'datasets/', dataset, 'training/') validation_dir = os.path.join(current_dir, 'datasets/', dataset, 'validation/') # Calculate training & validation steps num_images = 0 for folder in ['true/', 'false/']: dir = os.path.join(training_dir, folder) num_images += len(os.listdir(dir)) train_steps = num_images / batch_size num_images = 0 for folder in ['true/', 'false/']: dir = os.path.join(validation_dir, folder) num_images += len(os.listdir(dir)) val_steps = num_images / batch_size print('Dataset: ' + dataset) print('batch_size: ' + str(batch_size)) print('train_steps: ' + str(train_steps)) print('val_steps: ' + str(val_steps)) print('epochs: ' + str(epochs)) # All images will be rescaled by 1./255 train_datagen = ImageDataGenerator(rescale=1. / 255) valid_datagen = ImageDataGenerator(rescale=1. / 255) # Flow training images in batches of 20 using train_datagen generator train_generator = train_datagen.flow_from_directory( training_dir, # This is the source directory for training images target_size=(150, 150), # All images will be resized to 150x150 batch_size=batch_size, # Since we use binary_crossentropy loss, we need binary labels class_mode='binary') # Flow validation images in batches of 20 using test_datagen generator validation_generator = valid_datagen.flow_from_directory( validation_dir, target_size=(150, 150), batch_size=batch_size, class_mode='binary') # Train model history = model.fit_generator(train_generator, steps_per_epoch=train_steps, epochs=epochs, validation_data=validation_generator, validation_steps=val_steps, verbose=2) evaluate(history)
def read_image(self, image_paths): training_path = os.path.join(image_paths, 'LG_CNS_data', 'By_datasetMK', '20190523_training_NoAug_data_JK') data_generation = ImageDataGenerator(validation_split=0.2) train_generator = data_generation.flow_from_directory( directory=training_path, target_size=(256, 256), batch_size=self.params.batch_size, class_mode='categorical', shuffle=True, subset='training') validation_generator = data_generation.flow_from_directory( directory=training_path, target_size=(256, 256), batch_size=self.params.batch_size, class_mode='categorical', shuffle=True, subset='validation') return train_generator, validation_generator
def predict(files, model): model = load_model(model) test_datagen = ImageDataGenerator(rescale=1.0 / 255) i = 0 labels = [] plt.figure(figsize=(10, 10), dpi=80) plt.xlabel('xlabel', fontsize=18) plt.ylabel('ylabel', fontsize=16) plt.axis("off") for file in files: image = dataFromImage(file) img = img_to_array(image) img = np.expand_dims(image, axis=0) image = Image.fromarray(image, 'RGB') label = '' result = model.predict(test_datagen.flow(img, batch_size=1)) if result > 0.5: labels.append('dog') else: labels.append('cat') plt.subplot(2, 2, i + 1) plt.title('This is a ' + labels[i]) imgplot = plt.imshow(image) i += 1 if i % 10 == 0: break target = os.path.join(APP_ROOT, 'static') image_name = "plot" + str(time.time()) + ".jpeg" finalUrl = "/".join([target, image_name]) plt.savefig(finalUrl, bbox_inches="tight") return image_name
def imagenet_generator_multi(train_data_path, val_data_path, batch_size, do_augment, val_batch_size=VALIDATION_BATCH_SIZE): ''' For use with auxiliary classifiers or mutliple outputs train_data_path: Path for ImageNet Training Directory val_data_path: Path for ImageNet Validation Directory :return: Keras Data Generators for Training and Validation ''' if do_augment == True: rot_range = ROT_RANGE w_shift_r = WIDTH_SHIFT_RANGE h_shift_r = HEIGHT_SHIFT_RANGE z_range = ZOOM_RANGE shear_r = SHEAR_RANGE h_flip = True else: rot_range = 0 w_shift_r = 0.0 h_shift_r = 0.0 z_range = 0.0 shear_r = 0.0 h_flip = False print("Grabbing Training Dataset") train_datagen = ImageDataGenerator(samplewise_center=False, \ rotation_range=rot_range, \ width_shift_range=w_shift_r, \ height_shift_range=h_shift_r, \ zoom_range=z_range, \ shear_range=shear_r, \ horizontal_flip=h_flip, \ fill_mode='nearest', rescale=1. / 255) val_datagen = ImageDataGenerator(rescale=1. / 255) ''' Change follow_links to True when using symbolic links to training and validation data ''' train_generator = train_datagen.flow_from_directory( \ train_data_path, target_size=(IMAGE_SIZE, IMAGE_SIZE), \ batch_size=batch_size, shuffle=True, class_mode='categorical', \ follow_links=True) print("Grabbing Validation Dataset") validation_generator = val_datagen.flow_from_directory( \ val_data_path, target_size=(IMAGE_SIZE, IMAGE_SIZE), \ batch_size=val_batch_size, shuffle=True, \ class_mode='categorical', \ follow_links=True) multi_train_generator = create_multi_generator(train_generator) multi_validation_generator = create_multi_generator(validation_generator) train_size = train_generator.samples val_size = validation_generator.samples return multi_train_generator, multi_validation_generator, train_size, val_size
def buildDataGenerator(dataset_path): # Create a generator with data augmentation data_generator = ImageDataGenerator( preprocessing_function=preprocess_input, validation_split=0.25, horizontal_flip=True, width_shift_range=0.2, height_shift_range=0.2) # flow_from_dicrectory will auto label the image by the folder structure! train_gen = data_generator.flow_from_directory( dataset_path, subset="training", # Subset of data ("training" or "validation") if validation_split is set in ImageDataGenerator target_size=(IMAGE_SIZE, IMAGE_SIZE), shuffle=True, seed=SEED, batch_size=BATCH_SIZE, class_mode='categorical') eval_gen = data_generator.flow_from_directory(dataset_path, subset="validation", target_size=(IMAGE_SIZE, IMAGE_SIZE), shuffle=True, seed=SEED, batch_size=BATCH_SIZE, class_mode='categorical') global STEP_SIZE_TRAIN global STEP_SIZE_VALID # Total number of steps equal to the number of samples in your dataset divided by the batch size STEP_SIZE_TRAIN = train_gen.n / train_gen.batch_size STEP_SIZE_VALID = eval_gen.n / eval_gen.batch_size test_generator = ImageDataGenerator( preprocessing_function=preprocess_input) test_gen = test_generator.flow_from_directory( dataset_path, # Can be replace with custom data target_size=(IMAGE_SIZE, IMAGE_SIZE), shuffle=True, seed=SEED, batch_size=1, class_mode=None) # class_mode: If None, no labels are returned # (the generator will only yield batches of image data, # which is useful to use with model.predict_generator(), # model.evaluate_generator(), etc.). # Please note that in case of class_mode None, # the data still needs to reside in a subdirectory of directory for it to work correctly. return train_gen, eval_gen, test_gen
def data_generator_with_augmentation(): """ Create an ImageDataGenerator and do Image Augmentation """ return ImageDataGenerator( rescale=1.0 / 255, rotation_range=40, width_shift_range=0.1, height_shift_range=0.1, zoom_range=0.2, shear_range=0.2, horizontal_flip=True, fill_mode="nearest", )
def build(self): self._prepare_data() self._prepare_hyperparameters() model = CustomLeNet(len(self.output_classes), self.hyperparameters['optimizer'], self.hyperparameters['output_activation'], self.hyperparameters['loss']).model train_datagen = ImageDataGenerator( rescale=1. / 255, horizontal_flip=True, vertical_flip=True ) test_datagen = ImageDataGenerator( rescale=1. / 255, horizontal_flip=True, vertical_flip=True ) train_generator = train_datagen.flow_from_dataframe( dataframe=self.train, directory='./', x_col=self.filename_col_header, y_col=self.label_col_header, classmode='categorical', classes=self.output_classes, shuffle=True, target_size=(self.input_size[0], self.input_size[1]), batch_size=self.train_batch_size ) validation_generator = test_datagen.flow_from_dataframe( dataframe=self.test, directory='./', x_col=self.filename_col_header, y_col=self.label_col_header, classmode='categorical', classes=self.output_classes, shuffle=True, target_size=(self.input_size[0], self.input_size[1]), batch_size=self.test_batch_size ) tensorboard_callback = keras.callbacks.TensorBoard( log_dir=self.log_dir+'/scalars/') model.fit_generator( train_generator, steps_per_epoch=self.train_img_count // self.train_batch_size, epochs=self.hyperparameters['epochs'], validation_data=validation_generator, validation_steps=self.test_img_count // self.test_batch_size, callbacks=[tensorboard_callback] ) self.model = model self.label_map = train_generator.class_indices self.__save()
def train_model_with_generator(self): self.logger.info('Training model with image data generator') if not self.data_loaded: self.trainX, self.testX, self.trainY, self.testY = self.load_data() image_data_gen = ImageDataGenerator(rotation_range=15, width_shift_range=0.1, height_shift_range=0.1, shear_range=0.2, zoom_range=[0.8, 1.1], brightness_range=[0.5, 1.5], fill_mode='reflect') image_data_gen.fit(self.trainX) self.model.fit_generator( image_data_gen.flow(self.trainX, self.trainY, batch_size=constants.BATCH_SIZE), validation_data=(self.testX, self.testY), steps_per_epoch=len(self.trainX) // constants.BATCH_SIZE, epochs=constants.EPOCHS) self.__evaluate_model()
def main(args): w, h = args.model_input_size.split('x') input_shape = np.asarray([h, w, 3], dtype=int) checkpoint = ModelCheckpoint(str(WEIGHTS_OUTPUT_PATH), monitor='val_loss', verbose=1, save_weights_only=True, save_best_only=True) logging = TensorBoard(log_dir=str(LOGS_DIR)) reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=25, verbose=1, cooldown=0, min_lr=1e-7) early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=300, verbose=1) terminate_on_nan = TerminateOnNaN() callbacks = [ logging, checkpoint, reduce_lr, early_stopping, terminate_on_nan ] optimizer = Adam(lr=args.learning_rate) model = build_simple_conv_net(input_shape) model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) initial_epoch = args.init_epoch epochs = args.total_epochs - initial_epoch assert epochs >= 1 train_datagen = ImageDataGenerator(rescale=1 / 255) test_datagen = ImageDataGenerator(rescale=1 / 255) train_generator = train_datagen.flow_from_directory( './data/training/', target_size=input_shape[:2], batch_size=args.batch_size, class_mode='binary') validation_generator = test_datagen.flow_from_directory( './data/validation/', target_size=input_shape[:2], batch_size=args.batch_size, class_mode='binary') history = model.fit(train_generator, epochs=epochs, validation_data=validation_generator, callbacks=callbacks) model.save(MODEL_OUTPUT_PATH)
def train(model_file, train_path, validation_path, num_hidden=200, num_classes=5, steps=32, num_epochs=20, save_period=1): if os.path.exists(model_file): print("\n*** Existing model found at {}.loading.***\n\n".format( model_file)) model = load_existing(model_file) else: print("\n*** Create new model ***\n\n") model = create_model(num_hidden, num_classes) model.compile(optimizer='rmsprop', loss='categorical_crossentropy') checkpoint = ModelCheckpoint(model_file, period=save_period) train_datagen=ImageDataGenerator(\ rescale=1./255,\ shear_range=0.2,\ zoom_range=0.2,\ horizontal_flip=True) test_datagen = ImageDataGenerator(rescale=1. / 255) train_generator=train_datagen.flow_from_directory(\ train_path,\ target_size=(249,249), batch_size=256, class_mode="categorical") validation_generator=test_datagen.flow_from_directory(\ validation_path,\ target_size=(249,249), batch_size=256, class_mode='categorical') model.fit_generator(\ train_generator,\ steps_per_epoch=steps,\ epochs=num_epochs,\ callbacks=[checkpoint],\ validation_data=validation_generator,\ validation_steps=50) for layer in model.layers[:249]: layer.trainable = False for layer in model.layers[249:]: layer.trainable = True model.compile(optimizer=SGD(lr=0.00001, momentum=0.9), loss='categorical_crossentropy') model.fit_generator(\ train_generator,\ steps_per_epoch=steps,\ epochs=num_epochs,\ callbacks=[checkpoint],\ validation_data=validation_generator,\ validation_steps=50)
def getDataGen(datagenMode: str = None, dataType=None) -> ImageDataGenerator: """ Function to create data generator for training :param datagenMode: str, 'train' or 'test' :param dataType: str can be one of these standard datasets - cifar10 - cifar100 - mnist - fashion_mnist or 'custom' :return: ImageDataGenerator """ datagen = ImageDataGenerator( ImageDataGenerator( featurewise_center=datagen_featurewise_center, samplewise_center=datagen_samplewise_center, featurewise_std_normalization=datagen_featurewise_std_normalization, samplewise_std_normalization=datagen_samplewise_std_normalization, zca_whitening=datagen_zca_whitening, zca_epsilon=datagen_zca_epsilon, rotation_range=datagen_rotation_range, width_shift_range=datagen_width_shift_range, height_shift_range=datagen_height_shift_range, brightness_range=datagen_brightness_range, shear_range=datagen_shear_range, zoom_range=datagen_zoom_range, channel_shift_range=datagen_channel_shift_range, fill_mode=datagen_fill_mode, cval=datagen_cval, horizontal_flip=datagen_horizontal_flip, vertical_flip=datagen_vertical_flip, rescale=datagen_rescale, preprocessing_function=datagen_preprocessing_function, data_format=datagen_data_format, validation_split=datagen_validation_split, dtype=datagen_datagen_dtype)) if not dataType == 'custom': (x_train, y_train), _ = getattr(keras.datasets, dataType).load_data() datagen.fit(x_train, y_train) del x_train, y_train elif dataType == 'standard': datagen.mean = data_mu datagen.std = data_sigma return datagen
def get_datagen(): datagen = ImageDataGenerator( # set input mean to 0 over the dataset featurewise_center=False, # set each sample mean to 0 samplewise_center=False, # divide inputs by std of dataset featurewise_std_normalization=False, # divide each input by its std samplewise_std_normalization=False, # apply ZCA whitening zca_whitening=False, # epsilon for ZCA whitening zca_epsilon=1e-06, # randomly rotate images in the range (deg 0 to 180) rotation_range=45, # randomly shift images horizontally width_shift_range=0.2, # randomly shift images vertically height_shift_range=0.2, # set range for random shear shear_range=0.1, # set range for random zoom zoom_range=0.2, # set range for random channel shifts channel_shift_range=0., # set mode for filling points outside the input boundaries fill_mode='nearest', # value used for fill_mode = "constant" cval=0., # randomly flip images horizontal_flip=True, # randomly flip images vertical_flip=True, # set rescaling factor (applied before any other transformation) rescale=1. / 255, # set function that will be applied on each input preprocessing_function=None, # image data format, either "channels_first" or "channels_last" data_format=None, # fraction of images reserved for validation (strictly between 0 and 1) validation_split=0.0) return datagen
def train(model_file, train_path, validation_path, target_size=(227, 227), num_classes=5, steps=32, num_epochs=28): if os.path.exists(model_file): print('\n*** existing model found at {}. Loading. ***\n\n'.format( model_file)) model = load_existing(model_file) else: print("\n*** Creating new model ***\n\n") model = get_alnext_net(num_classes=num_classes) check_point = ModelCheckpoint(model_file, period=1) model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) train_datagen = ImageDataGenerator(rescale=1. / 255, shear_range=0.3, zoom_range=0.3, horizontal_flip=True, vertical_flip=True, rotation_range=20, width_shift_range=0.2, height_shift_range=0.2, brightness_range=(0.8, 1.2)) test_datagen = ImageDataGenerator(rescale=1. / 255) train_generator = train_datagen.flow_from_directory( train_path, target_size=target_size, batch_size=32, class_mode='categorical') validation_generator = test_datagen.flow_from_directory( validation_path, target_size=target_size, batch_size=32, class_mode='categorical') model.fit_generator( train_generator, steps_per_epoch=steps, epochs=num_epochs, callbacks=[ check_point, ], validation_data=validation_generator, validation_steps=50, shuffle=True, )
class ImageGenerator: def __init__(self): self.train_data_gen = ImageDataGenerator(rotation_range=40, width_shift_range=0.2, height_shift_range=0.2, rescale=1. / 255, shear_range=0.2, zoom_range=0.2, horizontal_flip=True, fill_mode='nearest') def generate_data(self, batch_size): data = self.train_data_gen.flow_from_directory( 'data/train', # this is the target directory target_size=(150, 150), # all images will be resized to 150x150 batch_size=batch_size, class_mode='binary') return data
def train(model_file, train_path, val_path, num_hidden=200, num_classes=4, steps=32, num_epochs=20, save_period=1): if os.path.exists(model_file): print ("\n*******existing model found at {}".format(model_file)) model = load_existing(model_file) else: print ("\n***creating a new model****\n") model = create_model(num_hidden,num_classes) #config = tf.ConfigProto() #config.gpu_options.allow_growth = True #K.set_session(tf.Session(config=config)) #for layer in model.layers[:249]: # layer.trainable = False # #layer.trainable = True #for layer in model.layers[249:]: # layer.trainable = True for index, layer in enumerate(model.layers): print(index, layer, layer.trainable) #model.compile(optimizer='rmsprop',loss='categorical_crossentropy',metrics=['accuracy']) #model.compile(optimizer=Adam(lr=1e-3, decay=1e-3),loss='categorical_crossentropy',metrics=['accuracy']) model.compile(optimizer=Adam(lr=1e-4, decay=1e-3),loss='categorical_crossentropy',metrics=['accuracy']) checkpoint = ModelCheckpoint(model_file, period=save_period) train_datagen = ImageDataGenerator(rotation_range=10, width_shift_range=0.05, zoom_range=[0.8, 1.25], channel_shift_range=10, height_shift_range=0.05, shear_range=0.05, horizontal_flip=True, rescale=1./255, brightness_range=[0.5, 1.5]) #train_datagen = ImageDataGenerator(rescale = 1./255, shear_range=0.2, zoom_range=0.2, horizontal_flip = True, brightness_range=[0.5, 1.5]) test_datagen = ImageDataGenerator(rescale=1./255) #target_size = (299, 299) target_size = (249, 249) train_gen = train_datagen.flow_from_directory(train_path, target_size=target_size, batch_size = 64, class_mode="categorical") #val_gen = test_datagen.flow_from_directory(val_path, target_size=target_size, batch_size= 64, class_mode='categorical') val_gen = test_datagen.flow_from_directory(val_path, target_size=target_size, batch_size=16, class_mode='categorical') step_size_train = train_gen.n // train_gen.batch_size step_size_valid = val_gen.n // val_gen.batch_size print('step_size_train:', step_size_train) print('step_size_valid:', step_size_valid) model.fit_generator(train_gen, steps_per_epoch=step_size_train, epochs=num_epochs, callbacks=[checkpoint], validation_data=val_gen, validation_steps=step_size_valid)
def train(model_file,train_path,validation_path,num_hidden=200,num_classes=4,steps=32,num_epochs=20,save_period=1): if os.path.exists(model_file): print ("\n*******existing model found at {}".format(model_file)) model = load_existing(model_file) else: print ("\n***creating a new model****\n") model = create_model(num_hidden,num_classes) model.compile(optimizer='rmsprop',loss='categorical_crossentropy',metrics=['accuracy']) checkpoint = ModelCheckpoint(model_file,period=save_period) #通过实时数据增强生成张量图像数据批次。数据将不断循环(按批次)。 train_datagen=ImageDataGenerator( rescale = 1./255, #重缩放因子。默认为 None。如果是 None 或 0,不进行缩放,否则将数据乘以所提供的值 shear_range=0.2, #浮点数。剪切强度(以弧度逆时针方向剪切角度 zoom_range=0.2, #浮点数 或 [lower, upper]。随机缩放范围。如果是浮点数,[lower, upper] = [1-zoom_range, 1+zoom_range] horizontal_flip = True) #布尔值。随机水平翻转 test_datagen = ImageDataGenerator(rescale=1./255) train_generator = train_datagen.flow_from_directory( train_path, #目标目录的路径。每个类应该包含一个子目录。任何在子目录树下的 PNG, JPG, BMP, PPM 或 TIF 图像,都将被包含在生成器中 target_size = (249,249), #整数元组 (height, width),默认:(256, 256)。所有的图像将被调整到的尺寸。 batch_size = 32, #一批数据的大小(默认 32) class_mode="categorical") #决定返回的标签数组的类型 validation_generator = test_datagen.flow_from_directory( validation_path, target_size=(249,249), batch_size=32, class_mode='categorical') #使用 Python 生成器(或 Sequence 实例)逐批生成的数据,按批次训练模型。 #生成器与模型并行运行,以提高效率。例如,这可以让你在 CPU 上对图像进行实时数据增强,以在 GPU 上训练模型。 model.fit_generator( train_generator, steps_per_epoch = steps, #在声明一个 epoch 完成并开始下一个 epoch 之前从 generator 产生的总步数(批次样本)。它通常应该等于你的数据集的样本数量除以批量大小 epochs= num_epochs, callbacks = [checkpoint], validation_data = validation_generator, validation_steps = 50) #仅当 validation_data 是一个生成器时才可用。在停止前 generator 生成的总步数(样本批数) for layer in model.layers[:249]: layer.trainable = False for layer in model.layers[249:]: layer.trainable = True model.compile(optimizer=SGD(lr=0.001,momentum=0.9),loss='categorical_crossentropy',metrics=['accuracy'])
def img_and_mask_generator(x, y, batch_size=1, shuffle=True): """ Create a generator of two combined ImageDataGenerators for input and ground truth images without any data augmentation except scaling. """ data_gen_args = dict(rescale=1. / 255) image_data_generator = ImageDataGenerator(**data_gen_args) mask_data_generator = ImageDataGenerator(**data_gen_args) seed = 1 if isinstance(x, np.ndarray): image_gen = image_data_generator.flow(x, batch_size=batch_size, seed=seed, shuffle=shuffle) mask_gen = mask_data_generator.flow(y, batch_size=batch_size, seed=seed, shuffle=shuffle) else: image_gen = image_data_generator.flow_from_directory( x, batch_size=batch_size, seed=seed, shuffle=shuffle, class_mode=None, color_mode="grayscale", target_size=(465, 381)) mask_gen = mask_data_generator.flow_from_directory( y, batch_size=batch_size, seed=seed, shuffle=shuffle, class_mode=None, color_mode="grayscale", target_size=(465, 381)) return zip(image_gen, mask_gen)
def transform_images(self): datagen = ImageDataGenerator( rotation_range=2.5, width_shift_range=0.05, height_shift_range=0.05, shear_range=0, zoom_range=0.1, horizontal_flip=True, # todo: testing false fill_mode='nearest') print( 'Randomly transforming and cropping input images, please wait...') for image_class in self.data_labels: photos_train = os.listdir('{}/.train_temp/{}'.format( self.proc_folder, image_class)) photos_valid = os.listdir('{}/.validation_temp/{}'.format( self.proc_folder, image_class)) photos_train = [ '{}/.train_temp/{}/{}'.format(self.proc_folder, image_class, img) for img in photos_train ] # adds path photos_valid = [ '{}/.validation_temp/{}/{}'.format(self.proc_folder, image_class, img) for img in photos_valid ] self.process_class(image_class, photos_train, datagen, True) # print('starting cropping of validation data') self.process_class( image_class, photos_valid, datagen, False) # no transformations, only crop for valid shutil.rmtree('{}/.train_temp'.format(self.proc_folder)) shutil.rmtree('{}/.validation_temp'.format(self.proc_folder)) open(self.finished_file, 'a').close( ) # create finished file so we know in the future not to process data print('All finished, moving on to training!')
def image(self): entrenamiento_datagen = ImageDataGenerator(rescale=1. / 255, shear_range=0.2, zoom_range=0.2, horizontal_flip=True) test_datagen = ImageDataGenerator(rescale=1. / 255) self.entrenamiento_generador = entrenamiento_datagen.flow_from_directory( self.directorio, target_size=(self.alturadelaimagen, self.longituddelaimagen), batch_size=self.numerodeimagenesamandar, class_mode='categorical') self.validacion_generador = test_datagen.flow_from_directory( self.directorio, target_size=(self.alturadelaimagen, self.longituddelaimagen), batch_size=self.numerodeimagenesamandar, class_mode='categorical')
while True: batch_x, batch_y = next(batches) batch_crops = np.zeros((batch_x.shape[0], crop_length, crop_length, 3)) for i in range(batch_x.shape[0]): batch_crops[i] = random_crop(batch_x[i], (crop_length, crop_length)) yield (batch_crops, batch_y) # In[6]: train_datagen = ImageDataGenerator(preprocessing_function=preprocess_input, rotation_range=40, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.2, zoom_range=0.2, channel_shift_range=10, horizontal_flip=True, fill_mode='nearest') train_batches = train_datagen.flow_from_directory(DATASET_PATH + '/train', target_size=IMAGE_SIZE, interpolation='bicubic', class_mode='categorical', shuffle=True, batch_size=BATCH_SIZE) # First check 1 batch of uncropped images # In[7]: