def train_and_predict(): print('[{}] Loading and preprocessing train data...'.format(str(datetime.datetime.now()))) imgs_train, imgs_mask_train, train_ids = load_train_data() print('[{}] Loading and preprocessing test data...'.format(str(datetime.datetime.now()))) imgs_test, imgs_id_test = load_test_data() imgs_train = preprocess(imgs_train) imgs_test = preprocess(imgs_test) X = np.vstack([imgs_train, imgs_test]) print('[{}] Creating and compiling model...'.format(str(datetime.datetime.now()))) model = get_unet() # model_checkpoint = ModelCheckpoint('unet.hdf5', monitor='loss', save_best_only=True) print('[{}] Fitting model...'.format(str(datetime.datetime.now()))) model.fit(X, X, batch_size=8, nb_epoch=100, verbose=1, shuffle=True, validation_split=0.2, )
def submission(): imgs_test, imgs_id_test = load_test_data() # 'imgs_mask_test_2016-08-15-01-38.npy' # 'imgs_mask_test_2016-08-15-08-44.npy' # 'imgs_mask_test_2016-08-15-06-37.npy' # # imgs_test1 = np.load('split5/imgs_mask_test_2016-08-15-01-38.npy') # imgs_test2 = np.load('split5/imgs_mask_test_2016-08-15-08-44.npy') # imgs_test3 = np.load('split5/imgs_mask_test_2016-08-15-06-37.npy') # # # imgs_test_a = (imgs_test1 + imgs_test2 + imgs_test3) / 3 imgs_test_b = np.load('masks/imgs_mask_test_2016-08-14-22-41.npy') imgs_test_c = np.load('masks/imgs_mask_test_2016-08-12-07-00.npy') # imgs_test_d = np.load('masks/imgs_mask_test_2016-08-16-00-54.npy') # imgs_test_e = np.load('masks/imgs_mask_test_2016-08-15-21-35.npy') # # imgs_test = (imgs_test_a + imgs_test_b + imgs_test_c + imgs_test_d + imgs_test_e) / 5 imgs_test = np.load('masks/imgs_mask_test_2016-08-17-00-53.npy') argsort = np.argsort(imgs_id_test) imgs_id_test = imgs_id_test[argsort] imgs_test = imgs_test[argsort] total = imgs_test.shape[0] ids = [] rles = [] for i in tqdm(range(total)): img = imgs_test[i, 0] img = prep(img) rle = run_length_enc(img) rles.append(rle) ids.append(imgs_id_test[i]) # file_name = 'submissions/blend5.csv' file_name = 'submissions/submission_2016-08-17-00-53.csv' df = pd.DataFrame() df['img'] = imgs_id_test df['pixels'] = rles df.loc[df['img'].isin(black_list), 'pixels'] = '' df.to_csv(file_name, index=False)
def train_and_predict(): print('[{}] Loading and preprocessing train data...'.format(str(datetime.datetime.now()))) imgs_train, imgs_mask_train, train_ids = load_train_data() imgs_train = preprocess(imgs_train) imgs_mask_train = preprocess(imgs_mask_train) imgs_train = imgs_train.astype('float32') X_train = imgs_train y_train = imgs_mask_train datagen = image_generator_xy.ImageDataGenerator( # featurewise_center=False, # featurewise_std_normalization=False, rotation_range=10, width_shift_range=0.1, height_shift_range=0.1, # zoom_range=0.1, # channel_shift_range=5, # shear_range=5, # zca_whitening=True, horizontal_flip=True, vertical_flip=True ) mean = np.mean(X_train) # mean for data centering X_train -= mean X_train /= 255 # We can probably live without this. y_train = (y_train.astype(np.float32) / 255).astype(int).astype(float) # scale masks to [0, 1] print print '[{}] Num train non zero masks...'.format(np.mean((np.mean(y_train, (2, 3)) > 0).astype(int).flatten())) # print '[{}] Num test non zero masks...'.format(np.mean((np.mean(y_test, (2, 3)) > 0).astype(int).flatten())) print('[{}] Creating and compiling model...'.format(str(datetime.datetime.now()))) model = read_model('16_20_2016-08-14-21-36') print('[{}] Fitting generator...'.format(str(datetime.datetime.now()))) history = History() datagen.fit(X_train) print('[{}] Fitting model...'.format(str(datetime.datetime.now()))) batch_size = 16 nb_epoch = 20 # sgd = SGD(lr=1e-5, decay=1e-6, momentum=0.9, nesterov=True) model.compile(optimizer=Nadam(lr=1e-6), loss=dice_coef_loss, metrics=[dice_coef, dice_coef_spec, 'binary_crossentropy']) model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size, shuffle=True), nb_epoch=nb_epoch, verbose=1, # validation_data=(X_val, y_val), samples_per_epoch=len(X_train), callbacks=[history], # shuffle=True ) now = datetime.datetime.now() suffix = str(now.strftime("%Y-%m-%d-%H-%M")) save_model(model, "{batch}_{epoch}_{suffix}".format(batch=batch_size, epoch=nb_epoch, suffix=suffix)) print print('[{}] Loading and preprocessing test data...'.format(str(datetime.datetime.now()))) imgs_test, imgs_id_test = load_test_data() imgs_test = preprocess(imgs_test) imgs_test = imgs_test.astype('float32') imgs_test -= mean imgs_test /= 255 print('[{}] Predicting masks on test data...'.format(str(datetime.datetime.now()))) imgs_mask_test = model.predict(imgs_test, verbose=1) np.save('masks/imgs_mask_test_{suffix}.npy'.format(suffix=suffix), imgs_mask_test) print '[{}] Saving history'.format(str(datetime.datetime.now())) save_history(history, suffix)
def train_and_predict(): print('[{}] Loading and preprocessing train data...'.format( str(datetime.datetime.now()))) imgs_train, imgs_mask_train, train_ids = load_train_data() imgs_train = preprocess(imgs_train) imgs_mask_train = preprocess(imgs_mask_train) imgs_train = imgs_train.astype('float32') print('[{}] Creating validation set...'.format(str( datetime.datetime.now()))) X_train, X_test, X_val, y_train, y_test, y_val = _train_val_split( imgs_train, imgs_mask_train) # print('[{}] Augmenting train...'.format(str(datetime.datetime.now()))) # X_train, y_train = _add_transformations(X_train, y_train) # print('[{}] Augmenting val...'.format(str(datetime.datetime.now()))) # X_val, y_val = _add_transformations(X_train, y_train) # print('[{}] Augmenting test...'.format(str(datetime.datetime.now()))) # X_test, y_test = _add_transformations(X_train, y_train) mean = np.mean(X_train) # mean for data centering X_train -= mean # X_train /= std # We can probably live without this. X_train /= 255 # We can probably live without this. X_test -= mean # X_test /= std # We can probably live without this. X_test /= 255 # We can probably live without this. X_val -= mean # X_val /= std # We can probably live without this. X_val /= 255 # We can probably live without this. y_train = (y_train.astype(np.float32) / 255).astype(int).astype( float) # scale masks to [0, 1] y_test = (y_test.astype(np.float32) / 255).astype(int).astype( float) # scale masks to [0, 1] y_val = (y_val.astype(np.float32) / 255).astype(int).astype( float) # scale masks to [0, 1] print print '[{}] Num train non zero masks...'.format( np.mean((np.mean(y_train, (2, 3)) > 0).astype(int).flatten())) print '[{}] Num val non zero masks...'.format( np.mean((np.mean(y_val, (2, 3)) > 0).astype(int).flatten())) print '[{}] Num test non zero masks...'.format( np.mean((np.mean(y_test, (2, 3)) > 0).astype(int).flatten())) print('[{}] Creating and compiling model...'.format( str(datetime.datetime.now()))) model = get_unet() # model_checkpoint = ModelCheckpoint('unet.hdf5', monitor='loss', save_best_only=True) print('[{}] Fitting model...'.format(str(datetime.datetime.now()))) model.fit( X_train, y_train, batch_size=8, nb_epoch=40, verbose=1, shuffle=True, validation_data=(X_val, y_val), # callbacks=[model_checkpoint] ) y_pred = model.predict(X_test) for a in np.arange(0, 1, 0.1): score = dice_coef_np(y_test, y_pred, a) print '[{date}] a = {a}. Score = {score}'.format(date=str( datetime.datetime.now()), score=score, a=a) print print('[{}] Loading and preprocessing test data...'.format( str(datetime.datetime.now()))) imgs_test, imgs_id_test = load_test_data() imgs_test = preprocess(imgs_test) imgs_test = imgs_test.astype('float32') imgs_test -= mean imgs_test /= 255 # # print('[{}] Loading saved weights...'.format(str(datetime.datetime.now()))) # # model.load_weights('unet.hdf5') print('[{}] Predicting masks on test data...'.format( str(datetime.datetime.now()))) imgs_mask_test = model.predict(imgs_test, verbose=1) np.save('imgs_mask_test.npy', imgs_mask_test)
def train_and_predict(): print('[{}] Loading and preprocessing train data...'.format(str(datetime.datetime.now()))) imgs_train, imgs_mask_train, train_ids = load_train_data() imgs_train = preprocess(imgs_train) imgs_mask_train = preprocess(imgs_mask_train) imgs_train = imgs_train.astype('float32') print('[{}] Creating validation set...'.format(str(datetime.datetime.now()))) # X_train = imgs_train # y_train = imgs_mask_train X_train, X_val, y_train, y_val = _train_val_split(imgs_train, imgs_mask_train) # print('[{}] Getting extra data...'.format(str(datetime.datetime.now()))) # extra_x, extra_y = extra_data() # # X_train = np.vstack([X_train, extra_x[:4000, :, :, :]]) # X_val = np.vstack([X_val, extra_x[4000:, :, :, :]]) datagen = image_generator_xy.ImageDataGenerator( featurewise_center=False, featurewise_std_normalization=False, rotation_range=10, width_shift_range=0.1, height_shift_range=0.1, # zoom_range=0.1, # # channel_shift_range=30, # shear_range=5, # # zca_whitening=True, horizontal_flip=True, vertical_flip=True ) mean = np.mean(X_train) # mean for data centering X_train -= mean X_train /= 255 # We can probably live without this. # X_test -= mean # X_test /= 255 # We can probably live without this. X_val -= mean X_val /= 255 # We can probably live without this. y_train = (y_train.astype(np.float32) / 255).astype(int).astype(float) # scale masks to [0, 1] # y_test = (y_test.astype(np.float32) / 255).astype(int).astype(float) # scale masks to [0, 1] y_val = (y_val.astype(np.float32) / 255).astype(int).astype(float) # scale masks to [0, 1] # y_train = np.vstack([y_train, extra_y[:4000, :, :, :]]).astype(int).astype(float) # y_val = np.vstack([y_val, extra_y[4000:, :, :, :]]).astype(int).astype(float) print print '[{}] Num train non zero masks...'.format(np.mean((np.mean(y_train, (2, 3)) > 0).astype(int).flatten())) print '[{}] Num val non zero masks...'.format(np.mean((np.mean(y_val, (2, 3)) > 0).astype(int).flatten())) # print '[{}] Num test non zero masks...'.format(np.mean((np.mean(y_test, (2, 3)) > 0).astype(int).flatten())) print('[{}] Creating and compiling model...'.format(str(datetime.datetime.now()))) # model = get_unet4() model = vgg_fcn() # model_checkpoint = ModelCheckpoint('unet.hdf5', monitor='loss', save_best_only=True) print('[{}] Fitting generator...'.format(str(datetime.datetime.now()))) history = History() datagen.fit(X_train) print('[{}] Fitting model...'.format(str(datetime.datetime.now()))) batch_size = 16 nb_epoch = 20 model.compile(optimizer=Nadam(lr=1e-3), loss=dice_coef_loss, metrics=[dice_coef, dice_coef_spec, 'binary_crossentropy']) model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size, shuffle=True), nb_epoch=nb_epoch, verbose=1, validation_data=(X_val, y_val), samples_per_epoch=len(X_train), callbacks=[history], # shuffle=True ) now = datetime.datetime.now() suffix = str(now.strftime("%Y-%m-%d-%H-%M")) save_model(model, "{batch}_{epoch}_{suffix}".format(batch=batch_size, epoch=nb_epoch, suffix=suffix)) print('[{data}] Saving model. Suffix = {suffix}'.format(data=str(datetime.datetime.now()), suffix=suffix)) model.compile(optimizer=Nadam(lr=1e-4), loss=dice_coef_loss, metrics=[dice_coef, dice_coef_spec, 'binary_crossentropy']) model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size, shuffle=True), nb_epoch=nb_epoch, verbose=1, validation_data=(X_val, y_val), samples_per_epoch=len(X_train), callbacks=[history], # shuffle=True ) now = datetime.datetime.now() suffix = str(now.strftime("%Y-%m-%d-%H-%M")) save_model(model, "{batch}_{epoch}_{suffix}".format(batch=batch_size, epoch=nb_epoch, suffix=suffix)) print('[{data}] Saving model. Suffix = {suffix}'.format(data=str(datetime.datetime.now()), suffix=suffix)) model.compile(optimizer=Nadam(lr=1e-5), loss=dice_coef_loss, metrics=[dice_coef, dice_coef_spec, 'binary_crossentropy']) model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size, shuffle=True), nb_epoch=nb_epoch, verbose=1, validation_data=(X_val, y_val), samples_per_epoch=len(X_train), callbacks=[history], # shuffle=True ) now = datetime.datetime.now() suffix = str(now.strftime("%Y-%m-%d-%H-%M")) save_model(model, "{batch}_{epoch}_{suffix}".format(batch=batch_size, epoch=nb_epoch, suffix=suffix)) print('[{data}] Saving model. Suffix = {suffix}'.format(data=str(datetime.datetime.now()), suffix=suffix)) model.compile(optimizer=Nadam(lr=1e-6), loss=dice_coef_loss, metrics=[dice_coef, dice_coef_spec, 'binary_crossentropy']) model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size, shuffle=True), nb_epoch=nb_epoch, verbose=1, validation_data=(X_val, y_val), samples_per_epoch=len(X_train), callbacks=[history], # shuffle=True ) now = datetime.datetime.now() suffix = str(now.strftime("%Y-%m-%d-%H-%M")) save_model(model, "{batch}_{epoch}_{suffix}".format(batch=batch_size, epoch=nb_epoch, suffix=suffix)) print print('[{}] Loading and preprocessing test data...'.format(str(datetime.datetime.now()))) imgs_test, imgs_id_test = load_test_data() imgs_test = preprocess(imgs_test) imgs_test = imgs_test.astype('float32') imgs_test -= mean imgs_test /= 255 print('[{}] Predicting masks on test data...'.format(str(datetime.datetime.now()))) imgs_mask_test = model.predict(imgs_test, verbose=1) np.save('masks/imgs_mask_test_{suffix}.npy'.format(suffix=suffix), imgs_mask_test) suffix = str(now.strftime("%Y-%m-%d-%H-%M")) print '[{}] Saving history'.format(str(datetime.datetime.now())) save_history(history, suffix) val_prediction = model.predict(X_val).astype(int).astype(float) print 'binarized_prediction on val = ', dice_coef_np(y_val, val_prediction)
def extra_data(): imgs_test, imgs_id_test = load_test_data() imgs_test_mask = np.load('masks/imgs_mask_test_2016-08-12-07-00.npy') return preprocess(imgs_test), preprocess(imgs_test_mask)
def extra_data(): imgs_test, imgs_id_test = load_test_data() imgs_test_mask = np.load('imgs_mask_test.npy') return preprocess(imgs_test), preprocess(imgs_test_mask)
---------- m: a list containing the indices of the nodes in the 2nd layer that are connecting to the i-th node's j-th membership function. """ m = [] for ind in range(len(indices)): key = indices[ind] if key[i] == j: m.append(ind) return m if __name__ == "__main__": import matplotlib.pyplot as plt from prepare_data import load_test_data training_data, validation_data, all_data = load_test_data() # network initialization net = ANFIS([4, 1]) net.initialization(training_data) # train the net evaluation_cost, evaluation_accuracy, \ training_cost, training_accuracy =net.stochastic_gradient_descent(training_data, validation_data, eta = 0.1, mini_batch_size=10, epoch = 30, adapt_eta_mode = True, evaluation_track_flag=True, training_track_flag=True) predictions = np.array([net.inference(x.reshape(-1,1))[0,0] for \