def train_new_top_model(all_features_hdf5, all_labels_hdf5): #The Bottleneck or other image extracted features are stored in h5 files # Use HDF5MatrixCacheIterator to use them. By default HDF5MatrixCacheIterator doesnt transform and doesnt shuffle. # so we can use old ImageDataGenerator datagen = ImageDataGenerator(validation_split=0.2) f1_trainvalidation = h5.File(all_features_hdf5, 'r') shape = f1_trainvalidation['data'].shape f1_trainvalidation.close() #15 Gb is upper limit for cache memory. total_mem_usage, dividing_factor = calculateDividingFactor(shape) #print("Dividing factor {}:".format(dividing_factor)) print("Cached memory usage: {}".format(total_mem_usage / (1024**3) / dividing_factor)) chunk_shape = (1, max(shape[1] // 2, 1), max(shape[2] // 2, 1), max(shape[3] // 2, 1)) f1_trainvalidation = h5c.File(all_features_hdf5, 'r', chunk_cache_mem_size=total_mem_usage // dividing_factor) f1_label = h5.File(all_labels_hdf5, 'r') train_generator = datagen.flow_hdf5(f1_trainvalidation['data'], f1_label['data'], subset='training', batch_size=batch_size, shuffle=False) validation_generator = datagen.flow_hdf5(f1_trainvalidation['data'], f1_label['data'], subset='validation', batch_size=batch_size, shuffle=False) model = Sequential() model.add(Flatten(input_shape=shape[1:])) model.add(Dense(1024, activation='relu')) #model.add(Dense(256,input_shape=(train_data_shape[1:],),activation='relu')) model.add(Dropout(0.5)) model.add(Dense(85, activation='sigmoid')) #model.compile(optimizer='rmsprop',loss='binary_crossentropy', metrics=['accuracy']) model.compile(loss='binary_crossentropy', optimizer="adam", metrics=[f1]) model.fit_generator( generator=train_generator, steps_per_epoch=int(ceil(train_generator.samples / batch_size)), validation_data=validation_generator, max_queue_size= 10, # use a value which can fit batch_size * image_size * max_queue_size in your CPU memory workers= 4, # I don't see multi workers can have any performance benefit without multi threading use_multiprocessing=False, # HDF5Matrix cannot support multi-threads validation_steps=int(ceil(validation_generator.samples / batch_size)), shuffle=True, epochs=epochs) model.save_weights(top_model_weights_path)
f1_trainvalidation.close() #15 Gb is upper limit for cache memory. total_mem_usage, dividing_factor = calculateDividingFactor(shape) #print("Dividing factor {}:".format(dividing_factor)) print("Cached memory usage: {}".format(total_mem_usage / (1024**3) / dividing_factor)) chunk_shape = (1, 100, 100, 3) f1_trainvalidation = h5c.File(all_features_hdf5, 'r', chunk_cache_mem_size=total_mem_usage // dividing_factor) f1_label = h5.File(all_labels_hdf5, 'r') train_generator = datagen.flow_hdf5(f1_trainvalidation['data'], f1_label['data'], subset='training', batch_size=batch_size, shuffle=False) validation_generator = datagen.flow_hdf5(f1_trainvalidation['data'], f1_label['data'], subset='validation', batch_size=batch_size, shuffle=False) # fine-tune the model model.fit_generator( generator=train_generator, steps_per_epoch=int(ceil(train_generator.samples / batch_size)), validation_data=validation_generator, max_queue_size= 10, # use a value which can fit batch_size * image_size * max_queue_size in your CPU memory