def train_and_predict():
print('[{}] Loading and preprocessing train data...'.format(str(datetime.datetime.now())))
imgs_train, imgs_mask_train, train_ids = load_train_data()
print('[{}] Loading and preprocessing test data...'.format(str(datetime.datetime.now())))
imgs_test, imgs_id_test = load_test_data()
imgs_train = preprocess(imgs_train)
imgs_test = preprocess(imgs_test)
X = np.vstack([imgs_train, imgs_test])
print('[{}] Creating and compiling model...'.format(str(datetime.datetime.now())))
model = get_unet()
# model_checkpoint = ModelCheckpoint('unet.hdf5', monitor='loss', save_best_only=True)
print('[{}] Fitting model...'.format(str(datetime.datetime.now())))
model.fit(X, X,
batch_size=8,
nb_epoch=100,
verbose=1,
shuffle=True,
validation_split=0.2,
)
def submission():
imgs_test, imgs_id_test = load_test_data()
# 'imgs_mask_test_2016-08-15-01-38.npy'
# 'imgs_mask_test_2016-08-15-08-44.npy'
# 'imgs_mask_test_2016-08-15-06-37.npy'
#
# imgs_test1 = np.load('split5/imgs_mask_test_2016-08-15-01-38.npy')
# imgs_test2 = np.load('split5/imgs_mask_test_2016-08-15-08-44.npy')
# imgs_test3 = np.load('split5/imgs_mask_test_2016-08-15-06-37.npy')
# #
# imgs_test_a = (imgs_test1 + imgs_test2 + imgs_test3) / 3
imgs_test_b = np.load('masks/imgs_mask_test_2016-08-14-22-41.npy')
imgs_test_c = np.load('masks/imgs_mask_test_2016-08-12-07-00.npy')
# imgs_test_d = np.load('masks/imgs_mask_test_2016-08-16-00-54.npy')
# imgs_test_e = np.load('masks/imgs_mask_test_2016-08-15-21-35.npy')
#
# imgs_test = (imgs_test_a + imgs_test_b + imgs_test_c + imgs_test_d + imgs_test_e) / 5
imgs_test = np.load('masks/imgs_mask_test_2016-08-17-00-53.npy')
argsort = np.argsort(imgs_id_test)
imgs_id_test = imgs_id_test[argsort]
imgs_test = imgs_test[argsort]
total = imgs_test.shape[0]
ids = []
rles = []
for i in tqdm(range(total)):
img = imgs_test[i, 0]
img = prep(img)
rle = run_length_enc(img)
rles.append(rle)
ids.append(imgs_id_test[i])
# file_name = 'submissions/blend5.csv'
file_name = 'submissions/submission_2016-08-17-00-53.csv'
df = pd.DataFrame()
df['img'] = imgs_id_test
df['pixels'] = rles
df.loc[df['img'].isin(black_list), 'pixels'] = ''
df.to_csv(file_name, index=False)
def train_and_predict():
print('[{}] Loading and preprocessing train data...'.format(str(datetime.datetime.now())))
imgs_train, imgs_mask_train, train_ids = load_train_data()
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
imgs_train = imgs_train.astype('float32')
X_train = imgs_train
y_train = imgs_mask_train
datagen = image_generator_xy.ImageDataGenerator(
# featurewise_center=False,
# featurewise_std_normalization=False,
rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
# zoom_range=0.1,
# channel_shift_range=5,
# shear_range=5,
# zca_whitening=True,
horizontal_flip=True,
vertical_flip=True
)
mean = np.mean(X_train) # mean for data centering
X_train -= mean
X_train /= 255 # We can probably live without this.
y_train = (y_train.astype(np.float32) / 255).astype(int).astype(float) # scale masks to [0, 1]
print
print '[{}] Num train non zero masks...'.format(np.mean((np.mean(y_train, (2, 3)) > 0).astype(int).flatten()))
# print '[{}] Num test non zero masks...'.format(np.mean((np.mean(y_test, (2, 3)) > 0).astype(int).flatten()))
print('[{}] Creating and compiling model...'.format(str(datetime.datetime.now())))
model = read_model('16_20_2016-08-14-21-36')
print('[{}] Fitting generator...'.format(str(datetime.datetime.now())))
history = History()
datagen.fit(X_train)
print('[{}] Fitting model...'.format(str(datetime.datetime.now())))
batch_size = 16
nb_epoch = 20
# sgd = SGD(lr=1e-5, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(optimizer=Nadam(lr=1e-6), loss=dice_coef_loss,
metrics=[dice_coef, dice_coef_spec, 'binary_crossentropy'])
model.fit_generator(datagen.flow(X_train,
y_train,
batch_size=batch_size,
shuffle=True),
nb_epoch=nb_epoch,
verbose=1,
# validation_data=(X_val, y_val),
samples_per_epoch=len(X_train),
callbacks=[history],
# shuffle=True
)
now = datetime.datetime.now()
suffix = str(now.strftime("%Y-%m-%d-%H-%M"))
save_model(model, "{batch}_{epoch}_{suffix}".format(batch=batch_size, epoch=nb_epoch, suffix=suffix))
print
print('[{}] Loading and preprocessing test data...'.format(str(datetime.datetime.now())))
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= 255
print('[{}] Predicting masks on test data...'.format(str(datetime.datetime.now())))
imgs_mask_test = model.predict(imgs_test, verbose=1)
np.save('masks/imgs_mask_test_{suffix}.npy'.format(suffix=suffix), imgs_mask_test)
print '[{}] Saving history'.format(str(datetime.datetime.now()))
save_history(history, suffix)
def train_and_predict():
print('[{}] Loading and preprocessing train data...'.format(
str(datetime.datetime.now())))
imgs_train, imgs_mask_train, train_ids = load_train_data()
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
imgs_train = imgs_train.astype('float32')
print('[{}] Creating validation set...'.format(str(
datetime.datetime.now())))
X_train, X_test, X_val, y_train, y_test, y_val = _train_val_split(
imgs_train, imgs_mask_train)
# print('[{}] Augmenting train...'.format(str(datetime.datetime.now())))
# X_train, y_train = _add_transformations(X_train, y_train)
# print('[{}] Augmenting val...'.format(str(datetime.datetime.now())))
# X_val, y_val = _add_transformations(X_train, y_train)
# print('[{}] Augmenting test...'.format(str(datetime.datetime.now())))
# X_test, y_test = _add_transformations(X_train, y_train)
mean = np.mean(X_train) # mean for data centering
X_train -= mean
# X_train /= std # We can probably live without this.
X_train /= 255 # We can probably live without this.
X_test -= mean
# X_test /= std # We can probably live without this.
X_test /= 255 # We can probably live without this.
X_val -= mean
# X_val /= std # We can probably live without this.
X_val /= 255 # We can probably live without this.
y_train = (y_train.astype(np.float32) / 255).astype(int).astype(
float) # scale masks to [0, 1]
y_test = (y_test.astype(np.float32) / 255).astype(int).astype(
float) # scale masks to [0, 1]
y_val = (y_val.astype(np.float32) / 255).astype(int).astype(
float) # scale masks to [0, 1]
print
print '[{}] Num train non zero masks...'.format(
np.mean((np.mean(y_train, (2, 3)) > 0).astype(int).flatten()))
print '[{}] Num val non zero masks...'.format(
np.mean((np.mean(y_val, (2, 3)) > 0).astype(int).flatten()))
print '[{}] Num test non zero masks...'.format(
np.mean((np.mean(y_test, (2, 3)) > 0).astype(int).flatten()))
print('[{}] Creating and compiling model...'.format(
str(datetime.datetime.now())))
model = get_unet()
# model_checkpoint = ModelCheckpoint('unet.hdf5', monitor='loss', save_best_only=True)
print('[{}] Fitting model...'.format(str(datetime.datetime.now())))
model.fit(
X_train,
y_train,
batch_size=8,
nb_epoch=40,
verbose=1,
shuffle=True,
validation_data=(X_val, y_val),
# callbacks=[model_checkpoint]
)
y_pred = model.predict(X_test)
for a in np.arange(0, 1, 0.1):
score = dice_coef_np(y_test, y_pred, a)
print '[{date}] a = {a}. Score = {score}'.format(date=str(
datetime.datetime.now()),
score=score,
a=a)
print
print('[{}] Loading and preprocessing test data...'.format(
str(datetime.datetime.now())))
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= 255
#
# print('[{}] Loading saved weights...'.format(str(datetime.datetime.now())))
#
# model.load_weights('unet.hdf5')
print('[{}] Predicting masks on test data...'.format(
str(datetime.datetime.now())))
imgs_mask_test = model.predict(imgs_test, verbose=1)
np.save('imgs_mask_test.npy', imgs_mask_test)
def train_and_predict():
print('[{}] Loading and preprocessing train data...'.format(str(datetime.datetime.now())))
imgs_train, imgs_mask_train, train_ids = load_train_data()
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
imgs_train = imgs_train.astype('float32')
print('[{}] Creating validation set...'.format(str(datetime.datetime.now())))
# X_train = imgs_train
# y_train = imgs_mask_train
X_train, X_val, y_train, y_val = _train_val_split(imgs_train, imgs_mask_train)
# print('[{}] Getting extra data...'.format(str(datetime.datetime.now())))
# extra_x, extra_y = extra_data()
#
# X_train = np.vstack([X_train, extra_x[:4000, :, :, :]])
# X_val = np.vstack([X_val, extra_x[4000:, :, :, :]])
datagen = image_generator_xy.ImageDataGenerator(
featurewise_center=False,
featurewise_std_normalization=False,
rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
# zoom_range=0.1,
# # channel_shift_range=30,
# shear_range=5,
# # zca_whitening=True,
horizontal_flip=True,
vertical_flip=True
)
mean = np.mean(X_train) # mean for data centering
X_train -= mean
X_train /= 255 # We can probably live without this.
# X_test -= mean
# X_test /= 255 # We can probably live without this.
X_val -= mean
X_val /= 255 # We can probably live without this.
y_train = (y_train.astype(np.float32) / 255).astype(int).astype(float) # scale masks to [0, 1]
# y_test = (y_test.astype(np.float32) / 255).astype(int).astype(float) # scale masks to [0, 1]
y_val = (y_val.astype(np.float32) / 255).astype(int).astype(float) # scale masks to [0, 1]
# y_train = np.vstack([y_train, extra_y[:4000, :, :, :]]).astype(int).astype(float)
# y_val = np.vstack([y_val, extra_y[4000:, :, :, :]]).astype(int).astype(float)
print
print '[{}] Num train non zero masks...'.format(np.mean((np.mean(y_train, (2, 3)) > 0).astype(int).flatten()))
print '[{}] Num val non zero masks...'.format(np.mean((np.mean(y_val, (2, 3)) > 0).astype(int).flatten()))
# print '[{}] Num test non zero masks...'.format(np.mean((np.mean(y_test, (2, 3)) > 0).astype(int).flatten()))
print('[{}] Creating and compiling model...'.format(str(datetime.datetime.now())))
# model = get_unet4()
model = vgg_fcn()
# model_checkpoint = ModelCheckpoint('unet.hdf5', monitor='loss', save_best_only=True)
print('[{}] Fitting generator...'.format(str(datetime.datetime.now())))
history = History()
datagen.fit(X_train)
print('[{}] Fitting model...'.format(str(datetime.datetime.now())))
batch_size = 16
nb_epoch = 20
model.compile(optimizer=Nadam(lr=1e-3), loss=dice_coef_loss,
metrics=[dice_coef, dice_coef_spec, 'binary_crossentropy'])
model.fit_generator(datagen.flow(X_train,
y_train,
batch_size=batch_size,
shuffle=True),
nb_epoch=nb_epoch,
verbose=1,
validation_data=(X_val, y_val),
samples_per_epoch=len(X_train),
callbacks=[history],
# shuffle=True
)
now = datetime.datetime.now()
suffix = str(now.strftime("%Y-%m-%d-%H-%M"))
save_model(model, "{batch}_{epoch}_{suffix}".format(batch=batch_size, epoch=nb_epoch, suffix=suffix))
print('[{data}] Saving model. Suffix = {suffix}'.format(data=str(datetime.datetime.now()), suffix=suffix))
model.compile(optimizer=Nadam(lr=1e-4), loss=dice_coef_loss,
metrics=[dice_coef, dice_coef_spec, 'binary_crossentropy'])
model.fit_generator(datagen.flow(X_train,
y_train,
batch_size=batch_size,
shuffle=True),
nb_epoch=nb_epoch,
verbose=1,
validation_data=(X_val, y_val),
samples_per_epoch=len(X_train),
callbacks=[history],
# shuffle=True
)
now = datetime.datetime.now()
suffix = str(now.strftime("%Y-%m-%d-%H-%M"))
save_model(model, "{batch}_{epoch}_{suffix}".format(batch=batch_size, epoch=nb_epoch, suffix=suffix))
print('[{data}] Saving model. Suffix = {suffix}'.format(data=str(datetime.datetime.now()), suffix=suffix))
model.compile(optimizer=Nadam(lr=1e-5), loss=dice_coef_loss,
metrics=[dice_coef, dice_coef_spec, 'binary_crossentropy'])
model.fit_generator(datagen.flow(X_train,
y_train,
batch_size=batch_size,
shuffle=True),
nb_epoch=nb_epoch,
verbose=1,
validation_data=(X_val, y_val),
samples_per_epoch=len(X_train),
callbacks=[history],
# shuffle=True
)
now = datetime.datetime.now()
suffix = str(now.strftime("%Y-%m-%d-%H-%M"))
save_model(model, "{batch}_{epoch}_{suffix}".format(batch=batch_size, epoch=nb_epoch, suffix=suffix))
print('[{data}] Saving model. Suffix = {suffix}'.format(data=str(datetime.datetime.now()), suffix=suffix))
model.compile(optimizer=Nadam(lr=1e-6), loss=dice_coef_loss,
metrics=[dice_coef, dice_coef_spec, 'binary_crossentropy'])
model.fit_generator(datagen.flow(X_train,
y_train,
batch_size=batch_size,
shuffle=True),
nb_epoch=nb_epoch,
verbose=1,
validation_data=(X_val, y_val),
samples_per_epoch=len(X_train),
callbacks=[history],
# shuffle=True
)
now = datetime.datetime.now()
suffix = str(now.strftime("%Y-%m-%d-%H-%M"))
save_model(model, "{batch}_{epoch}_{suffix}".format(batch=batch_size, epoch=nb_epoch, suffix=suffix))
print
print('[{}] Loading and preprocessing test data...'.format(str(datetime.datetime.now())))
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= 255
print('[{}] Predicting masks on test data...'.format(str(datetime.datetime.now())))
imgs_mask_test = model.predict(imgs_test, verbose=1)
np.save('masks/imgs_mask_test_{suffix}.npy'.format(suffix=suffix), imgs_mask_test)
suffix = str(now.strftime("%Y-%m-%d-%H-%M"))
print '[{}] Saving history'.format(str(datetime.datetime.now()))
save_history(history, suffix)
val_prediction = model.predict(X_val).astype(int).astype(float)
print 'binarized_prediction on val = ', dice_coef_np(y_val, val_prediction)
def extra_data():
imgs_test, imgs_id_test = load_test_data()
imgs_test_mask = np.load('masks/imgs_mask_test_2016-08-12-07-00.npy')
return preprocess(imgs_test), preprocess(imgs_test_mask)
def extra_data():
imgs_test, imgs_id_test = load_test_data()
imgs_test_mask = np.load('imgs_mask_test.npy')
return preprocess(imgs_test), preprocess(imgs_test_mask)
----------
m: a list containing the indices of the nodes in the 2nd layer that are
connecting to the i-th node's j-th membership function.
"""
m = []
for ind in range(len(indices)):
key = indices[ind]
if key[i] == j:
m.append(ind)
return m
if __name__ == "__main__":
import matplotlib.pyplot as plt
from prepare_data import load_test_data
training_data, validation_data, all_data = load_test_data()
# network initialization
net = ANFIS([4, 1])
net.initialization(training_data)
# train the net
evaluation_cost, evaluation_accuracy, \
training_cost, training_accuracy =net.stochastic_gradient_descent(training_data,
validation_data,
eta = 0.1,
mini_batch_size=10,
epoch = 30,
adapt_eta_mode = True,
evaluation_track_flag=True,
training_track_flag=True)
predictions = np.array([net.inference(x.reshape(-1,1))[0,0] for \
