def train_and_predict():
print('[{}] Loading and preprocessing train data...'.format(str(datetime.datetime.now())))
imgs_train, imgs_mask_train, train_ids = load_train_data()
print('[{}] Loading and preprocessing test data...'.format(str(datetime.datetime.now())))
imgs_test, imgs_id_test = load_test_data()
imgs_train = preprocess(imgs_train)
imgs_test = preprocess(imgs_test)
X = np.vstack([imgs_train, imgs_test])
print('[{}] Creating and compiling model...'.format(str(datetime.datetime.now())))
model = get_unet()
# model_checkpoint = ModelCheckpoint('unet.hdf5', monitor='loss', save_best_only=True)
print('[{}] Fitting model...'.format(str(datetime.datetime.now())))
model.fit(X, X,
batch_size=8,
nb_epoch=100,
verbose=1,
shuffle=True,
validation_split=0.2,
)
def train_and_predict():
print('[{}] Loading and preprocessing train data...'.format(str(datetime.datetime.now())))
imgs_train, imgs_mask_train, train_ids = load_train_data()
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
imgs_train = imgs_train.astype('float32')
X_train = imgs_train
y_train = imgs_mask_train
datagen = image_generator_xy.ImageDataGenerator(
# featurewise_center=False,
# featurewise_std_normalization=False,
rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
# zoom_range=0.1,
# channel_shift_range=5,
# shear_range=5,
# zca_whitening=True,
horizontal_flip=True,
vertical_flip=True
)
mean = np.mean(X_train) # mean for data centering
X_train -= mean
X_train /= 255 # We can probably live without this.
y_train = (y_train.astype(np.float32) / 255).astype(int).astype(float) # scale masks to [0, 1]
print
print '[{}] Num train non zero masks...'.format(np.mean((np.mean(y_train, (2, 3)) > 0).astype(int).flatten()))
# print '[{}] Num test non zero masks...'.format(np.mean((np.mean(y_test, (2, 3)) > 0).astype(int).flatten()))
print('[{}] Creating and compiling model...'.format(str(datetime.datetime.now())))
model = read_model('16_20_2016-08-14-21-36')
print('[{}] Fitting generator...'.format(str(datetime.datetime.now())))
history = History()
datagen.fit(X_train)
print('[{}] Fitting model...'.format(str(datetime.datetime.now())))
batch_size = 16
nb_epoch = 20
# sgd = SGD(lr=1e-5, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(optimizer=Nadam(lr=1e-6), loss=dice_coef_loss,
metrics=[dice_coef, dice_coef_spec, 'binary_crossentropy'])
model.fit_generator(datagen.flow(X_train,
y_train,
batch_size=batch_size,
shuffle=True),
nb_epoch=nb_epoch,
verbose=1,
# validation_data=(X_val, y_val),
samples_per_epoch=len(X_train),
callbacks=[history],
# shuffle=True
)
now = datetime.datetime.now()
suffix = str(now.strftime("%Y-%m-%d-%H-%M"))
save_model(model, "{batch}_{epoch}_{suffix}".format(batch=batch_size, epoch=nb_epoch, suffix=suffix))
print
print('[{}] Loading and preprocessing test data...'.format(str(datetime.datetime.now())))
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= 255
print('[{}] Predicting masks on test data...'.format(str(datetime.datetime.now())))
imgs_mask_test = model.predict(imgs_test, verbose=1)
np.save('masks/imgs_mask_test_{suffix}.npy'.format(suffix=suffix), imgs_mask_test)
print '[{}] Saving history'.format(str(datetime.datetime.now()))
save_history(history, suffix)
def train_and_predict():
print('[{}] Loading and preprocessing train data...'.format(
str(datetime.datetime.now())))
imgs_train, imgs_mask_train, train_ids = load_train_data()
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
imgs_train = imgs_train.astype('float32')
print('[{}] Creating validation set...'.format(str(
datetime.datetime.now())))
# X_train, X_test, X_val, y_train, y_test, y_val = _train_val_split(imgs_train, imgs_mask_train)
# print('[{}] Augmenting train...'.format(str(datetime.datetime.now())))
# X_train, y_train = _add_transformations(X_train, y_train)
# print('[{}] Augmenting val...'.format(str(datetime.datetime.now())))
# X_val, y_val = _add_transformations(X_train, y_train)
# print('[{}] Augmenting test...'.format(str(datetime.datetime.now())))
# X_test, y_test = _add_transformations(X_train, y_train)
X = imgs_train
y = (np.mean(imgs_mask_train,
(2, 3)) == 0).astype(int).flatten().astype(np.float32)
mean = np.mean(X) # mean for data centering
#
X -= mean
# # X_train /= std # We can probably live without this.
# X_train /= 255 # We can probably live without this.
# X_test -= mean
# # X_test /= std # We can probably live without this.
# X_test /= 255 # We can probably live without this.
# X_val -= mean
# # X_val /= std # We can probably live without this.
# X_val /= 255 # We can probably live without this.
# y_train = (y_train.astype(np.float32) / 255).astype(int).astype(float) # scale masks to [0, 1]
# y_test = (y_test.astype(np.float32) / 255).astype(int).astype(float) # scale masks to [0, 1]
# y_val = (y_val.astype(np.float32) / 255).astype(int).astype(float) # scale masks to [0, 1]
# print
# print '[{}] Num train non zero masks...'.format(np.mean((np.mean(y_train, (2, 3)) > 0).astype(int).flatten()))
# print '[{}] Num val non zero masks...'.format(np.mean((np.mean(y_val, (2, 3)) > 0).astype(int).flatten()))
# print '[{}] Num test non zero masks...'.format(np.mean((np.mean(y_test, (2, 3)) > 0).astype(int).flatten()))
print('[{}] Creating and compiling model...'.format(
str(datetime.datetime.now())))
model = get_unet()
# model_checkpoint = ModelCheckpoint('unet.hdf5', monitor='loss', save_best_only=True)
print('[{}] Fitting model...'.format(str(datetime.datetime.now())))
model.fit(X,
y,
batch_size=16,
nb_epoch=20,
verbose=1,
shuffle=True,
validation_split=0.2
# validation_data=(X_val, y_val),
# callbacks=[model_checkpoint]
)
def train_and_predict():
print('[{}] Loading and preprocessing train data...'.format(
str(datetime.datetime.now())))
imgs_train, imgs_mask_train, train_ids = load_train_data()
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
imgs_train = imgs_train.astype('float32')
print('[{}] Creating validation set...'.format(str(
datetime.datetime.now())))
X_train, X_test, X_val, y_train, y_test, y_val = _train_val_split(
imgs_train, imgs_mask_train)
# print('[{}] Augmenting train...'.format(str(datetime.datetime.now())))
# X_train, y_train = _add_transformations(X_train, y_train)
# print('[{}] Augmenting val...'.format(str(datetime.datetime.now())))
# X_val, y_val = _add_transformations(X_train, y_train)
# print('[{}] Augmenting test...'.format(str(datetime.datetime.now())))
# X_test, y_test = _add_transformations(X_train, y_train)
mean = np.mean(X_train) # mean for data centering
X_train -= mean
# X_train /= std # We can probably live without this.
X_train /= 255 # We can probably live without this.
X_test -= mean
# X_test /= std # We can probably live without this.
X_test /= 255 # We can probably live without this.
X_val -= mean
# X_val /= std # We can probably live without this.
X_val /= 255 # We can probably live without this.
y_train = (y_train.astype(np.float32) / 255).astype(int).astype(
float) # scale masks to [0, 1]
y_test = (y_test.astype(np.float32) / 255).astype(int).astype(
float) # scale masks to [0, 1]
y_val = (y_val.astype(np.float32) / 255).astype(int).astype(
float) # scale masks to [0, 1]
print
print '[{}] Num train non zero masks...'.format(
np.mean((np.mean(y_train, (2, 3)) > 0).astype(int).flatten()))
print '[{}] Num val non zero masks...'.format(
np.mean((np.mean(y_val, (2, 3)) > 0).astype(int).flatten()))
print '[{}] Num test non zero masks...'.format(
np.mean((np.mean(y_test, (2, 3)) > 0).astype(int).flatten()))
print('[{}] Creating and compiling model...'.format(
str(datetime.datetime.now())))
model = get_unet()
# model_checkpoint = ModelCheckpoint('unet.hdf5', monitor='loss', save_best_only=True)
print('[{}] Fitting model...'.format(str(datetime.datetime.now())))
model.fit(
X_train,
y_train,
batch_size=8,
nb_epoch=40,
verbose=1,
shuffle=True,
validation_data=(X_val, y_val),
# callbacks=[model_checkpoint]
)
y_pred = model.predict(X_test)
for a in np.arange(0, 1, 0.1):
score = dice_coef_np(y_test, y_pred, a)
print '[{date}] a = {a}. Score = {score}'.format(date=str(
datetime.datetime.now()),
score=score,
a=a)
print
print('[{}] Loading and preprocessing test data...'.format(
str(datetime.datetime.now())))
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= 255
#
# print('[{}] Loading saved weights...'.format(str(datetime.datetime.now())))
#
# model.load_weights('unet.hdf5')
print('[{}] Predicting masks on test data...'.format(
str(datetime.datetime.now())))
imgs_mask_test = model.predict(imgs_test, verbose=1)
np.save('imgs_mask_test.npy', imgs_mask_test)
def train_and_predict():
print('[{}] Loading and preprocessing train data...'.format(str(datetime.datetime.now())))
imgs_train, imgs_mask_train, train_ids = load_train_data()
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
imgs_train = imgs_train.astype('float32')
print('[{}] Creating validation set...'.format(str(datetime.datetime.now())))
# X_train = imgs_train
# y_train = imgs_mask_train
X_train, X_val, y_train, y_val = _train_val_split(imgs_train, imgs_mask_train)
# print('[{}] Getting extra data...'.format(str(datetime.datetime.now())))
# extra_x, extra_y = extra_data()
#
# X_train = np.vstack([X_train, extra_x[:4000, :, :, :]])
# X_val = np.vstack([X_val, extra_x[4000:, :, :, :]])
datagen = image_generator_xy.ImageDataGenerator(
featurewise_center=False,
featurewise_std_normalization=False,
rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
# zoom_range=0.1,
# # channel_shift_range=30,
# shear_range=5,
# # zca_whitening=True,
horizontal_flip=True,
vertical_flip=True
)
mean = np.mean(X_train) # mean for data centering
X_train -= mean
X_train /= 255 # We can probably live without this.
# X_test -= mean
# X_test /= 255 # We can probably live without this.
X_val -= mean
X_val /= 255 # We can probably live without this.
y_train = (y_train.astype(np.float32) / 255).astype(int).astype(float) # scale masks to [0, 1]
# y_test = (y_test.astype(np.float32) / 255).astype(int).astype(float) # scale masks to [0, 1]
y_val = (y_val.astype(np.float32) / 255).astype(int).astype(float) # scale masks to [0, 1]
# y_train = np.vstack([y_train, extra_y[:4000, :, :, :]]).astype(int).astype(float)
# y_val = np.vstack([y_val, extra_y[4000:, :, :, :]]).astype(int).astype(float)
print
print '[{}] Num train non zero masks...'.format(np.mean((np.mean(y_train, (2, 3)) > 0).astype(int).flatten()))
print '[{}] Num val non zero masks...'.format(np.mean((np.mean(y_val, (2, 3)) > 0).astype(int).flatten()))
# print '[{}] Num test non zero masks...'.format(np.mean((np.mean(y_test, (2, 3)) > 0).astype(int).flatten()))
print('[{}] Creating and compiling model...'.format(str(datetime.datetime.now())))
# model = get_unet4()
model = vgg_fcn()
# model_checkpoint = ModelCheckpoint('unet.hdf5', monitor='loss', save_best_only=True)
print('[{}] Fitting generator...'.format(str(datetime.datetime.now())))
history = History()
datagen.fit(X_train)
print('[{}] Fitting model...'.format(str(datetime.datetime.now())))
batch_size = 16
nb_epoch = 20
model.compile(optimizer=Nadam(lr=1e-3), loss=dice_coef_loss,
metrics=[dice_coef, dice_coef_spec, 'binary_crossentropy'])
model.fit_generator(datagen.flow(X_train,
y_train,
batch_size=batch_size,
shuffle=True),
nb_epoch=nb_epoch,
verbose=1,
validation_data=(X_val, y_val),
samples_per_epoch=len(X_train),
callbacks=[history],
# shuffle=True
)
now = datetime.datetime.now()
suffix = str(now.strftime("%Y-%m-%d-%H-%M"))
save_model(model, "{batch}_{epoch}_{suffix}".format(batch=batch_size, epoch=nb_epoch, suffix=suffix))
print('[{data}] Saving model. Suffix = {suffix}'.format(data=str(datetime.datetime.now()), suffix=suffix))
model.compile(optimizer=Nadam(lr=1e-4), loss=dice_coef_loss,
metrics=[dice_coef, dice_coef_spec, 'binary_crossentropy'])
model.fit_generator(datagen.flow(X_train,
y_train,
batch_size=batch_size,
shuffle=True),
nb_epoch=nb_epoch,
verbose=1,
validation_data=(X_val, y_val),
samples_per_epoch=len(X_train),
callbacks=[history],
# shuffle=True
)
now = datetime.datetime.now()
suffix = str(now.strftime("%Y-%m-%d-%H-%M"))
save_model(model, "{batch}_{epoch}_{suffix}".format(batch=batch_size, epoch=nb_epoch, suffix=suffix))
print('[{data}] Saving model. Suffix = {suffix}'.format(data=str(datetime.datetime.now()), suffix=suffix))
model.compile(optimizer=Nadam(lr=1e-5), loss=dice_coef_loss,
metrics=[dice_coef, dice_coef_spec, 'binary_crossentropy'])
model.fit_generator(datagen.flow(X_train,
y_train,
batch_size=batch_size,
shuffle=True),
nb_epoch=nb_epoch,
verbose=1,
validation_data=(X_val, y_val),
samples_per_epoch=len(X_train),
callbacks=[history],
# shuffle=True
)
now = datetime.datetime.now()
suffix = str(now.strftime("%Y-%m-%d-%H-%M"))
save_model(model, "{batch}_{epoch}_{suffix}".format(batch=batch_size, epoch=nb_epoch, suffix=suffix))
print('[{data}] Saving model. Suffix = {suffix}'.format(data=str(datetime.datetime.now()), suffix=suffix))
model.compile(optimizer=Nadam(lr=1e-6), loss=dice_coef_loss,
metrics=[dice_coef, dice_coef_spec, 'binary_crossentropy'])
model.fit_generator(datagen.flow(X_train,
y_train,
batch_size=batch_size,
shuffle=True),
nb_epoch=nb_epoch,
verbose=1,
validation_data=(X_val, y_val),
samples_per_epoch=len(X_train),
callbacks=[history],
# shuffle=True
)
now = datetime.datetime.now()
suffix = str(now.strftime("%Y-%m-%d-%H-%M"))
save_model(model, "{batch}_{epoch}_{suffix}".format(batch=batch_size, epoch=nb_epoch, suffix=suffix))
print
print('[{}] Loading and preprocessing test data...'.format(str(datetime.datetime.now())))
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= 255
print('[{}] Predicting masks on test data...'.format(str(datetime.datetime.now())))
imgs_mask_test = model.predict(imgs_test, verbose=1)
np.save('masks/imgs_mask_test_{suffix}.npy'.format(suffix=suffix), imgs_mask_test)
suffix = str(now.strftime("%Y-%m-%d-%H-%M"))
print '[{}] Saving history'.format(str(datetime.datetime.now()))
save_history(history, suffix)
val_prediction = model.predict(X_val).astype(int).astype(float)
print 'binarized_prediction on val = ', dice_coef_np(y_val, val_prediction)
