def test(output_dir, weights_dir, test_dir):
"""
Tests the given model and generates a csv containing segmentation masks compressed using rle
Parameters:
output_dir -- directory path to store output csv
weights_dir -- directory path to load weights of given model
test_dir -- directory path containing test images
"""
batch_size = params.batch_size
model = params.model_factory()
# Copy file from gcs to local directory
with file_io.FileIO(weights_dir, mode='r') as input_f:
with file_io.FileIO('weights.hdf5', mode='w+') as output_f:
output_f.write(input_f.read())
model.load_weights(filepath='weights.hdf5')
graph = tf.get_default_graph()
with file_io.FileIO(test_dir + '/sample_submission.csv', mode='r') as f:
csv_bytes = f.read()
df_test = pd.read_csv(io.BytesIO(csv_bytes))
ids_test = df_test['img'].map(lambda s: s.split('.')[0])
names = []
for id in ids_test:
names.append('{}.jpg'.format(id))
q_size = 10
q = Queue.Queue(maxsize=q_size)
t1 = threading.Thread(target=data_loader, name='DataLoader', args=(q, ids_test, test_dir))
t2 = threading.Thread(target=predictor, name='Predictor', args=(q, len(ids_test), graph, model))
print('Predicting on {} samples with batch_size = {}...'.format(len(ids_test), batch_size))
t1.start()
t2.start()
# Wait for both threads to finish
t1.join()
t2.join()
print("Generating submission file...")
df = pd.DataFrame({'img': names, 'rle_mask': rles})
# Copy submission file to gcs
df.to_csv('submission.csv.gz', index=False, compression='gzip')
with file_io.FileIO('submission.csv.gz', mode='r') as input_f:
with file_io.FileIO(output_dir + '/submission.csv.gz', mode='w+') as output_f:
output_f.write(input_f.read())
def train_vggnet(train_dir, job_dir):
"""
Trains a fully convolutional VGG-16 CNN model.
Parameters:
train_dir -- directory containing training data
job_dir -- directory where output files (log and weight files) are stored
"""
# Get parameteers from params file
import params
input_size = params.input_size
epochs = params.max_epochs
batch_size = params.batch_size
model = params.model_factory()
# Get image names from csv file
with file_io.FileIO(train_dir + '/train_masks.csv', mode='r') as f:
csv_bytes = f.read()
df_train = pd.read_csv(io.BytesIO(csv_bytes))
ids_train = df_train['img'].map(lambda s: s.split('.')[0])
# Split images into training and validation sets
ids_train_split, ids_valid_split = train_test_split(ids_train, test_size=0.2, random_state=42)
print('Training on {} samples'.format(len(ids_train_split)))
print('Validating on {} samples'.format(len(ids_valid_split)))
# Initialize callbacks
callbacks = [EarlyStopping(monitor='val_loss',
patience=3,
verbose=1,
min_delta=1e-4),
ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=4,
verbose=1,
epsilon=1e-4),
ModelSave(monitor='val_loss',
filepath=job_dir,
save_best_only=True,
save_weights_only=True),
TensorBoard(log_dir=job_dir + '/logs')]
# Start training
model.fit_generator(generator=train_generator(train_dir, ids_train_split, batch_size, input_size),
steps_per_epoch=np.ceil(float(len(ids_train_split)) / float(batch_size)),
epochs=epochs,
verbose=2,
callbacks=callbacks,
validation_data=valid_generator(train_dir, ids_valid_split, batch_size, input_size),
validation_steps=np.ceil(float(len(ids_valid_split)) / float(batch_size)))
import os
import random
gpu_id = '0'
os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_id)
import params
DATA_PATH = '/media/Disk/yanpengxiang/dataset/carvana/'
input_size = params.input_size
orig_width = params.orig_width
orig_height = params.orig_height
epochs = params.max_epochs
batch_size = params.batch_size
model = params.model_factory()
#df_train = pd.read_csv(DATA_PATH + 'train_masks.csv')
#ids_train = df_train['img'].map(lambda s: s.split('.')[0])
ids_train = []
with open('input/train_id.txt') as f:
for line in f:
ids_train.append(line[:15])
ids_train_split, ids_valid_split = train_test_split(ids_train,
test_size=0.1,
random_state=42)
print('Training on {} samples'.format(len(ids_train_split)))
print('Validating on {} samples'.format(len(ids_valid_split)))
continue
single_layer_mem *= s
shapes_mem_count += single_layer_mem
trainable_count = np.sum(
[K.count_params(p) for p in set(model.trainable_weights)])
non_trainable_count = np.sum(
[K.count_params(p) for p in set(model.non_trainable_weights)])
total_memory = 4.0 * batch_size * (shapes_mem_count + trainable_count +
non_trainable_count)
gbytes = np.round(total_memory / (1024.0**3), 3)
return gbytes
model = params.model_factory(input_shape=params.input_shape)
model.compile(
optimizer=optimizers.RMSprop(lr=0.01),
loss='binary_crossentropy',
#loss=custom_losses.floss,
metrics=[
'accuracy', custom_losses.fmeasure, custom_losses.recall,
custom_losses.precision
])
labels = load_labels()
ids_train_split = glob.glob(params.folder_train + "*.*")
ids_valid_split = glob.glob(params.folder_valid + "*.*")
print('Memory needed estimation: {}GB'.format(get_model_memory_usage(model)))
save_best_only=True,
save_weights_only=True,
mode='min')
]
datagen = ImageDataGenerator(horizontal_flip=aug_horizontal_flip,
vertical_flip=aug_vertical_flip,
rotation_range=aug_rotation,
width_shift_range=aug_width_shift,
height_shift_range=aug_height_shift,
channel_shift_range=aug_channel_shift,
shear_range=aug_shear,
zoom_range=aug_zoom)
model_info = params.model_factory(input_shape=X_train.shape[1:],
inputs_meta=M_train.shape[1])
model_info.summary()
with open(best_model_path, "w") as json_file:
json_file.write(model_info.to_json())
def train_and_evaluate_model(model, X_tr, y_tr, X_cv, y_cv):
xtr, mtr = X_tr
xcv, mcv = X_cv
hist = model.fit_generator(
get_data_generator(datagen, xtr, mtr, ytr, batch_size=batch_size),
steps_per_epoch=np.ceil(float(len(xtr)) / float(batch_size)),
epochs=epochs,
verbose=2,
