def create_data_gen(): augmented_data_gen = ra.realtime_augmented_data_gen( num_chunks=N_TRAIN / BATCH_SIZE * (EPOCHS + 1), chunk_size=BATCH_SIZE, augmentation_params=augmentation_params, ds_transforms=ds_transforms, target_sizes=input_sizes) post_augmented_data_gen = ra.post_augment_brightness_gen( augmented_data_gen, std=0.5) train_gen = load_data.buffered_gen_mp(post_augmented_data_gen, buffer_size=GEN_BUFFER_SIZE) input_gen = input_generator(train_gen) return input_gen
def dataLoader(data_indices, batch_size=100, shuffle=True): data_indices = np.array(data_indices) if shuffle: np.random.shuffle(data_indices) augmented_data_gen = ra.realtime_augmented_data_gen( data_indices=data_indices, batch_size=batch_size, augmentation_params=augmentation_params, ds_transforms=ds_transforms, target_sizes=input_sizes) post_augmented_data_gen = ra.post_augment_brightness_gen( augmented_data_gen, std=0.5) viewpoints_gen = create_viewpoints(post_augmented_data_gen, part_size=viewpoint_size) data_gen = load_data.buffered_gen_mp(viewpoints_gen, buffer_size=GEN_BUFFER_SIZE) return data_gen
ra.build_augmentation_transform(rotation=45) ] num_input_representations = len(ds_transforms) augmentation_params = { 'zoom_range': (1.0 / 1.3, 1.3), 'rotation_range': (0, 360), 'shear_range': (0, 0), 'translation_range': (-4, 4), 'do_flip': True, } augmented_data_gen = ra.realtime_augmented_data_gen( num_chunks=NUM_CHUNKS, chunk_size=CHUNK_SIZE, augmentation_params=augmentation_params, ds_transforms=ds_transforms, target_sizes=input_sizes) post_augmented_data_gen = ra.post_augment_brightness_gen(augmented_data_gen, std=0.5) train_gen = load_data.buffered_gen_mp(post_augmented_data_gen, buffer_size=GEN_BUFFER_SIZE) y_train = np.load("data/solutions_train.npy") train_ids = load_data.train_ids test_ids = load_data.test_ids # split training data into training + a small validation set num_train = len(train_ids)
ra.build_ds_transform(3.0, target_size=input_sizes[0]), ra.build_ds_transform(3.0, target_size=input_sizes[1]) + ra.build_augmentation_transform(rotation=45) ] num_input_representations = len(ds_transforms) augmentation_params = { 'zoom_range': (1.0 / 1.3, 1.3), 'rotation_range': (0, 360), 'shear_range': (0, 0), 'translation_range': (-4, 4), 'do_flip': True, } augmented_data_gen = ra.realtime_augmented_data_gen(num_chunks=NUM_CHUNKS, chunk_size=CHUNK_SIZE, augmentation_params=augmentation_params, ds_transforms=ds_transforms, target_sizes=input_sizes) post_augmented_data_gen = ra.post_augment_brightness_gen(augmented_data_gen, std=0.5) train_gen = load_data.buffered_gen_mp(post_augmented_data_gen, buffer_size=GEN_BUFFER_SIZE) y_train = np.load("data/solutions_train.npy") train_ids = load_data.train_ids test_ids = load_data.test_ids # split training data into training + a small validation set num_train = len(train_ids) num_test = len(test_ids)
] num_input_representations = len(ds_transforms) augmentation_params = { "zoom_range": (1.0 / 1.3, 1.3), "rotation_range": (0, 360), "shear_range": (0, 0), "translation_range": (-4, 4), "do_flip": True, } augmented_data_gen = ra.realtime_augmented_data_gen( num_chunks=NUM_CHUNKS, chunk_size=CHUNK_SIZE, augmentation_params=augmentation_params, ds_transforms=ds_transforms, target_sizes=input_sizes, processor_class=ra.LoadAndProcessPysexGen1CenteringRescaling, ) post_augmented_data_gen = ra.post_augment_brightness_gen(augmented_data_gen, std=0.5) train_gen = load_data.buffered_gen_mp(post_augmented_data_gen, buffer_size=GEN_BUFFER_SIZE) y_train = np.load("data/solutions_train.npy") train_ids = load_data.train_ids test_ids = load_data.test_ids # split training data into training + a small validation set num_train = len(train_ids)
] num_input_representations = len(ds_transforms) augmentation_params = { 'zoom_range': (1.0 / 1.3, 1.3), 'rotation_range': (0, 360), 'shear_range': (0, 0), 'translation_range': (-4, 4), 'do_flip': True, } augmented_data_gen = ra.realtime_augmented_data_gen( num_chunks=NUM_CHUNKS, chunk_size=CHUNK_SIZE, augmentation_params=augmentation_params, ds_transforms=ds_transforms, target_sizes=input_sizes, processor_class=ra.LoadAndProcessPysexGen1CenteringRescaling) post_augmented_data_gen = ra.post_augment_brightness_gen(augmented_data_gen, std=0.5) train_gen = load_data.buffered_gen_mp(post_augmented_data_gen, buffer_size=GEN_BUFFER_SIZE) y_train = np.load("data/solutions_train.npy") train_ids = load_data.train_ids test_ids = load_data.test_ids # split training data into training + a small validation set
ra.build_augmentation_transform(rotation=45) ] num_input_representations = len(ds_transforms) augmentation_params = { 'zoom_range': (1.0 / 1.3, 1.3), 'rotation_range': (0, 360), 'shear_range': (0, 0), 'translation_range': (-4, 4), 'do_flip': True, } augmented_data_gen = ra.realtime_augmented_data_gen( num_chunks=N_TRAIN / BATCH_SIZE * (EPOCHS + 1), chunk_size=BATCH_SIZE, augmentation_params=augmentation_params, ds_transforms=ds_transforms, target_sizes=input_sizes) post_augmented_data_gen = ra.post_augment_brightness_gen(augmented_data_gen, std=0.5) train_gen = load_data.buffered_gen_mp(post_augmented_data_gen, buffer_size=GEN_BUFFER_SIZE) input_gen = input_generator(train_gen) def create_valid_gen(): data_gen_valid = ra.realtime_fixed_augmented_data_gen( valid_indices,
ra.build_augmentation_transform(rotation=45) ] num_input_representations = len(ds_transforms) augmentation_params = { 'zoom_range': (1.0 / 1.3, 1.3), 'rotation_range': (0, 360), 'shear_range': (0, 0), 'translation_range': (-4, 4), 'do_flip': True, } augmented_data_gen = ra.realtime_augmented_data_gen( num_chunks=EPOCHS, chunk_size=N_TRAIN, augmentation_params=augmentation_params, ds_transforms=ds_transforms, target_sizes=input_sizes) post_augmented_data_gen = ra.post_augment_brightness_gen(augmented_data_gen, std=0.5) train_gen = post_augmented_data_gen #train_gen = load_data.buffered_gen_mp(post_augmented_data_gen, buffer_size=GEN_BUFFER_SIZE) augmentation buffering will not work with the keras .fit ''' def create_train_gen(): """ this generates the training data in order, for postprocessing. Do not use this for actual training. """ data_gen_train = ra.realtime_fixed_augmented_data_gen(train_indices, 'train', ds_transforms=ds_transforms, chunk_size=N_TRAIN, target_sizes=input_sizes)