def predict():
print('-' * 30)
print('Loading and preprocessing test data...')
print('-' * 30)
imgs_test, imgs_id_test = load_test_data()
imgs_test = imgs_test.astype('float32')
mean = np.mean(imgs_test) # mean for data centering
std = np.std(imgs_test) # std for data normalization
imgs_test -= mean
imgs_test /= std
print('-' * 30)
print('Loading saved weights...')
print('-' * 30)
model = build_model()
model.load_weights('model.hdf5')
print('-' * 30)
print('Predicting masks on test data...')
print('-' * 30)
imgs_testing_train = model.predict(imgs_test, verbose=1)
np.save('imgs_testing_train1.npy', imgs_testing_train)
def submission():
from data import load_test_data
imgs_test, imgs_id_test = load_test_data()
imgs_test = np.load('imgs_mask_test_nfold.npy')
argsort = np.argsort(imgs_id_test)
imgs_id_test = imgs_id_test[argsort]
imgs_test = imgs_test[argsort]
total = imgs_test.shape[0]
ids = []
rles = []
for i in range(total):
img = imgs_test[i, 0]
img = prep(img)
rle = run_length_enc(img)
rles.append(rle)
ids.append(imgs_id_test[i])
if i % 100 == 0:
print('{}/{}'.format(i, total))
first_row = 'img,pixels'
file_name = 'submission.csv'
with open(file_name, 'w+') as f:
f.write(first_row + '\n')
for i in range(total):
s = str(ids[i]) + ',' + rles[i]
f.write(s + '\n')
def predict_unet(data_path, save_path, basename="", weight_load = "", train_mean=0, train_std=1):
print('-'*30)
print('Loading and preprocessing test data...')
print('-'*30)
imgs_test, imgs_id_test = load_test_data(data_path)
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= train_mean
imgs_test /= train_std
print('-'*30)
print('Loading saved weights...')
print('-'*30)
model.load_weights(weight_load)
print('-'*30)
print('Predicting masks on test data...')
print('-'*30)
imgs_mask_test = model.predict(imgs_test, verbose=1)
mask_filename = save_path + 'imgs_mask_test' + basename + '.npy'
np.save(mask_filename, imgs_mask_test)
submission_filename = save_path + '/submission' + basename + '.csv'
submission(data_path, submission_filename, mask_filename, .6)
def check_data():
imgs_test, imgs_id_test = load_test_data()
imgs_mask_test = np.load('imgs_mask_test.npy')
print('-' * 30)
print('Saving predicted masks to files...')
print('-' * 30)
pred_dir = 'preds/'
if not os.path.exists(pred_dir):
os.mkdir(pred_dir)
print(imgs_mask_test.shape)
print(imgs_id_test.shape)
for image, image_id in zip(imgs_mask_test, imgs_id_test):
# print(image.shape)
image = (image[:, :, 0] * 255.).astype(np.uint8)
# print(image.shape)
#cv2.imshow('1', image)
#if cv2.waitKey(0) == 27:
# exit( 1 )
imsave('preds/{}_pred.png'.format(image_id), image)
def load_trained_model():
model = get_unet()
model.load_weights(weights_path)
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
mean = np.mean(imgs_test)
std = np.std(imgs_test) # std for data normalization
imgs_test -= mean
imgs_test /= std
imgs_mask_test = model.predict(imgs_test, verbose=1, batch_size=1)
np.save('imgs_mask_test.npy', imgs_mask_test)
pred_dir = 'preds'
if not os.path.exists(pred_dir):
os.mkdir(pred_dir)
for image, image_id in zip(imgs_mask_test, imgs_id_test):
image = (image[:, :, 0] * 255.).astype(np.uint8)
image_resized = resize(image, (1040, 1392), mode='constant')
imsave(os.path.join(pred_dir,
str(image_id) + '_pred.png'), image_resized)
def load_data(data_path):
print('-'*30)
print('Loading and preprocessing train data...')
print('-'*30)
imgs_train, imgs_mask_train, imgs_patient_train = load_train_data(data_path)
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
imgs_train = imgs_train.astype('float32')
mean = np.mean(imgs_train) # mean for data centering
std = np.std(imgs_train) # std for data normalization
imgs_train -= mean
imgs_train /= std
imgs_mask_train = imgs_mask_train.astype('float32')
imgs_mask_train /= 255. # scale masks to [0, 1]
print('-'*30)
print('Loading and preprocessing test data...')
print('-'*30)
imgs_test, imgs_id_test = load_test_data(data_path)
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
return imgs_train, imgs_mask_train, imgs_patient_train, imgs_test
def test():
print('-' * 30)
print('Loading and preprocessing test data...')
print('-' * 30)
# imgs_train, imgs_mask_train = load_train_data()
# imgs_train = imgs_train.astype('float32')
# mean = np.mean(imgs_train) # mean for data centering
#std = np.std(imgs_train) # std for data normalization
imgs_test = load_test_data()
mean = np.load('scaler.npz')['mean']
std = np.load('scaler.npz')['std']
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
print('-' * 30)
print('Creating and compiling model...')
print('-' * 30)
model = get_unet()
# print('Loading saved weights...')
# print('-'*30)
model.load_weights('final_weights.h5')
# print('-'*30)
print('Predicting masks on test data...')
# print('-'*30)
imgs_mask_test = model.predict(imgs_test, verbose=2)
np.savez_compressed('/imgs_test_mask', imgs_test_mask=imgs_mask_test)
print('Saving predicted masks to files...')
def load_model_network():
X_test, Y_test = load_test_data()
model = CNN('digitos.hdf5')
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
model.compile(optimizer=optimizer,
loss="categorical_crossentropy",
metrics=["accuracy"])
Y_pred = model.predict(X_test)
# Convert predictions classes to one hot vectors
Y_pred_classes = np.argmax(Y_pred, axis=1)
# Convert validation observations to one hot vectors
Y_true = np.argmax(Y_test, axis=1)
# compute the confusion matrix
confusion_mtx = confusion_matrix(Y_true, Y_pred_classes)
print(" ACURÀCIA = %.2f" % (accuracy_score(Y_true, Y_pred_classes) * 100),
"%")
# plot the confusion matrix
plot_confusion_matrix(confusion_mtx, classes=range(10))
def length_analyze():
# load data
#get_data = lambda d: [d["1"], d["2"], d["3"]] # character
#get_data = lambda d: [d["1_tokenized"], d["2_tokenized"], d["3_tokenized"]] # word
get_data = lambda d: [d["1_m_2000"], d["2_m_2000"], d["3_m_2000"]] # word
data = load_data("train.txt")
x_train = get_data(data)
data = load_data("dev.txt")
x_valid = get_data(data)
data = load_test_data("test.txt")
x_test = get_data(data)
l_train = np.array([len(row) for x in x_train for row in x])
l_valid = np.array([len(row) for x in x_valid for row in x])
l_test = np.array([len(row) for x in x_test for row in x])
for length, name in zip([l_train, l_valid, l_test], ["train", "valid", "test"]):
print("============================")
print("length max", np.max(length))
print("length min", np.min(length))
print("length mean", length.mean())
print("length std", length.std())
print("length median", np.median(length))
print("length percentile", np.percentile(length, 75))
def submission():
from data import load_test_data
imgs_test, imgs_id_test = load_test_data()
imgs_test = np.load('imgs_mask_test.npy')
argsort = np.argsort(imgs_id_test)
imgs_id_test = imgs_id_test[argsort]
imgs_test = imgs_test[argsort]
total = imgs_test.shape[0]
ids = []
rles = []
for i in range(total):
img = imgs_test[i, 0]
img = prep(img)
rle = run_length_enc(img)
rles.append(rle)
ids.append(imgs_id_test[i])
if i % 100 == 0:
print('{}/{}'.format(i, total))
first_row = 'img,pixels'
file_name = 'submission.csv'
with open(file_name, 'w+') as f:
f.write(first_row + '\n')
for i in range(total):
s = str(ids[i]) + ',' + rles[i]
f.write(s + '\n')
def train_and_predict():
print('-' * 30)
print('Loading and preprocessing train data...')
print('-' * 30)
imgs_train, imgs_mask_train = load_train_data()
imgs_train, imgs_mask_train = preprocess(imgs_train, imgs_mask_train)
imgs_train = imgs_train.astype('float32')
mean = np.mean(imgs_train) # mean for data centering
std = np.std(imgs_train) # std for data normalization
imgs_train -= mean
imgs_train /= std
imgs_mask_train = imgs_mask_train.astype('float32')
imgs_mask_train /= 255. # scale masks to [0, 1]
print('-' * 30)
print('Creating and compiling model...')
print('-' * 30)
model = get_unet()
model_checkpoint = ModelCheckpoint('unet.hdf5',
monitor='val_loss',
save_best_only=True)
plot(model, to_file='model.png')
print('-' * 30)
print('Fitting model...')
print('-' * 30)
early_stopping = EarlyStopping(monitor='val_loss', patience=5, verbose=1)
model.fit(imgs_train,
imgs_mask_train,
batch_size=1,
nb_epoch=500,
verbose=1,
shuffle=True,
callbacks=[model_checkpoint, early_stopping],
validation_split=0.10)
print('-' * 30)
print('Loading and preprocessing test data...')
print('-' * 30)
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
print('-' * 30)
print('Loading saved weights...')
print('-' * 30)
model.load_weights('unet.hdf5')
print('-' * 30)
print('Predicting masks on test data...')
print('-' * 30)
imgs_mask_test = model.predict(imgs_test, verbose=1)
np.save('imgs_mask_test.npy', imgs_mask_test)
def test_dataloader(self):
# Load data and create a DataLoader
X_test, y_test = load_test_data()
test_dataloader = to_dataloader(X_test,
y_test,
batch_size=cfg.BATCH_SIZE)
return test_dataloader
def train_and_predict():
print('-' * 30)
print('Loading and preprocessing train data...')
print('-' * 30)
imgs_train, imgs_mask_train = load_train_data()
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
imgs_train = imgs_train.astype('float32')
mean = np.mean(imgs_train) # mean for data centering
std = np.std(imgs_train) # std for data normalization
imgs_train -= mean
imgs_train /= std
imgs_mask_train = imgs_mask_train.astype('float32')
imgs_mask_train /= 255. # scale masks to [0, 1]
print('-' * 30)
print('Creating and compiling model...')
print('-' * 30)
model = get_unet()
# model.save_weights('/unet.hdf5')
model_checkpoint = ModelCheckpoint(filepath='unet.hdf5',
monitor='loss',
save_best_only=True)
print('-' * 30)
print('Fitting model...')
print('-' * 30)
model.fit(imgs_train,
imgs_mask_train,
batch_size=32,
nb_epoch=nb_epoch,
verbose=1,
shuffle=True,
callbacks=[model_checkpoint])
print('-' * 30)
print('Loading and preprocessing test data...')
print('-' * 30)
#imgs_test, imgs_id_test = load_test_data()
imgs_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
print('-' * 30)
print('Loading saved weights...')
print('-' * 30)
model.load_weights('unet.hdf5')
print('-' * 30)
print('Predicting masks on test data...')
print('-' * 30)
imgs_mask_test = model.predict(imgs_test, verbose=1)
np.save('imgs_mask_test.npy', imgs_mask_test)
def word_distribution():
# load data
#get_data = lambda d: [d["1"], d["2"], d["3"]] # character
#get_data = lambda d: [d["1_tokenized"], d["2_tokenized"], d["3_tokenized"]] # word
get_data = lambda d: [d["1_m_2000"], d["2_m_2000"], d["3_m_2000"]] # word
data = load_data("train.txt")
x_train = get_data(data)
data = load_data("dev.txt")
x_valid = get_data(data)
data = load_test_data("test.txt")
x_test = get_data(data)
# freq
freq_train = Counter(
w
for x in x_train
for row in x
for w in row
)
freq_valid = Counter(
w
for x in x_valid
for row in x
for w in row
)
freq_test = Counter(
w
for x in x_test
for row in x
for w in row
)
# statistic
print("Train All", sum(freq_train.values()))
print("Valid All", sum(freq_valid.values()))
# overlap
print("=====================================")
for min_freq in [0, 5, 10]:
print("min_freq = ", min_freq)
available_chars = {k:v for k, v in freq_train.items() if v >= min_freq}
not_in_valid = {k:v for k, v in freq_valid.items() if k not in available_chars}
not_in_test = {k:v for k, v in freq_test.items() if k not in available_chars}
print("Train # unique chars = {}, # chars = {}".format(len(available_chars), sum(available_chars.values())))
print("Valid Not in Train -- # unique chars = {}, # chars = {} | # available chars = {}".format(
len(not_in_valid),
sum(not_in_valid.values()),
sum(freq_valid.values()) - sum(not_in_valid.values()),
))
print("Test Not in Train -- # unique chars = {}, # chars = {} | # available chars = {}".format(
len(not_in_test),
sum(not_in_test.values()),
sum(freq_test.values()) - sum(not_in_test.values()),
))
print()
def val_dataloader(self):
# Load data and create a DataLoader
X_val, y_val = load_test_data()
val_dataloader = to_dataloader(X_val,
y_val[:, :, :, 0:self.output_shape[2]],
batch_size=cfg.BATCH_SIZE,
shuffle=False)
return val_dataloader
def train_and_predict():
print('-'*30)
print('Loading and preprocessing train data...')
print('-'*30)
imgs_train, imgs_mask_train = load_train_data()
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
imgs_train = imgs_train.astype('float32')
mean = np.mean(imgs_train) # mean for data centering
std = np.std(imgs_train) # std for data normalization
imgs_train -= mean
imgs_train /= std
imgs_mask_train = imgs_mask_train.astype('float32')
imgs_mask_train /= 255. # scale masks to [0, 1]
print('-'*30)
print('Creating and compiling model...')
print('-'*30)
model = get_unet()
plot(model, to_file='model.png', show_shapes=True)
model_checkpoint = ModelCheckpoint('unet.hdf5', monitor='loss', save_best_only=True)
print('-'*30)
print('Fitting model...')
print('-'*30)
datagen = ImageDataGenerator(
rotation_range=5,
vertical_flip=True,
horizontal_flip=True,
)
model.fit_generator(datagen.flow(imgs_train, imgs_mask_train, batch_size=32, shuffle=True),
samples_per_epoch=len(imgs_train), nb_epoch=120, verbose=1, callbacks=[model_checkpoint])
print('-'*30)
print('Loading and preprocessing test data...')
print('-'*30)
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
print('-'*30)
print('Loading saved weights...')
print('-'*30)
model.load_weights('unet.hdf5')
print('-'*30)
print('Predicting masks on test data...')
print('-'*30)
imgs_mask_test = model.predict(imgs_test, verbose=1)
np.save('imgs_mask_test.npy', imgs_mask_test)
def predict():
print('-'*30)
print('Loading and preprocessing train data...')
print('-'*30)
imgs_train, imgs_mask_train, imgs_label_train = load_train_data()
# imgs_train = imgs_train[:32]
# imgs_label_train = imgs_label_train[:32]
imgs_train = preprocess(imgs_train)
imgs_label_train = preprocess(imgs_label_train)
imgs_label_train = np_utils.to_categorical(imgs_label_train)
imgs_label_train = imgs_label_train.reshape((280,96,96,4))
mean = np.mean(imgs_train) # mean for data centering
std = np.std(imgs_train) # std for data normalization
imgs_train -= mean
imgs_train /= std
print('-'*30)
print('Loading and preprocessing test data...')
print('-'*30)
imgs_test, imgs_mask_test, imgs_label_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_label_test = preprocess(imgs_label_test)
imgs_label_test = np_utils.to_categorical(imgs_label_test)
imgs_label_test = imgs_label_test.reshape((70,96,96,4))
imgs_test -= mean
imgs_test /= std
print('-'*30)
print('Creating and compiling model...')
print('-'*30)
model = get_unet()
# model_checkpoint = ModelCheckpoint('weights.h5', monitor='val_loss', save_best_only=True)
print('-'*30)
print('Loading saved weights...')
print('-'*30)
model.load_weights('weights.h5')
print('-'*30)
print('Predicting labels on test data...')
print('-'*30)
predicted = model.predict(imgs_test, verbose=1)
np.save('predicted.npy', predicted)
print('-' * 30)
print('Saving predicted labels to files...')
print('-' * 30)
pred_dir = 'preds'
if not os.path.exists(pred_dir):
os.mkdir(pred_dir)
for image, image_id in zip(predicted.argmax(axis=3)[...,np.newaxis], imgs_id_test):
image = (image[:, :, 0]).astype(np.uint8)
scipy.misc.imsave(os.path.join(pred_dir, str(image_id) + '_pred.png'), image)
def load_and_predict():
print('-'*30)
print('Loading and preprocessing train data...')
print('-'*30)
imgs_train, imgs_mask_train = load_train_data()
## Image Resizing
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
## Network Preprocessing
imgs_train = imgs_train.astype('float32')
mean = np.mean(imgs_train) # mean for data centering
std = np.std(imgs_train) # std for data normalization
imgs_train -= mean
imgs_train /= std
imgs_mask_train = imgs_mask_train.astype('float32')
imgs_mask_train /= 255. # scale masks to [0, 1]
print('-'*30)
print('Creating and compiling model...')
print('-'*30)
model = get_unet()
print('-'*30)
print('Loading and preprocessing test data... NOT LOADING LABELS')
print('-'*30)
imgs_test, imgs_id_test , _ = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
print('-'*30)
print('Loading saved weights...')
print('-'*30)
model.load_weights('weights.h5')
print('-'*30)
print('Predicting masks on test data...')
print('-'*30)
imgs_mask_test = model.predict(imgs_test, verbose=1)
np.save('imgs_mask_test_PRED.npy', imgs_mask_test)
print('-' * 30)
print('Saving predicted masks to files...')
print('-' * 30)
pred_dir = 'preds'
if not os.path.exists(pred_dir):
os.mkdir(pred_dir)
for image, image_id in zip(imgs_mask_test, imgs_id_test):
image = (image[:, :, 0] * 255.).astype(np.uint8)
imsave(os.path.join(pred_dir, image_id), image)
def main(hps):
#pre_train
if args.pre_train:
word2idx, idx2word, verb2idx, idx2verb = make_vocab(hps)
# word2idx, idx2word, verb2idx, idx2verb = load_vocab(hps)
# mapping, vectors = load_glove(hps)
# weights_matrix = make_pre_trained_word_embedding(mapping, vectors, word2idx.keys(), hps)
hps = hps._replace(vocab_size=len(word2idx))
hps = hps._replace(verb_vocab_size=len(verb2idx))
hps = hps._replace(pre_train=True)
print('parameters:')
print(hps)
train_loader, valid_loader, char_weights, action_weights = load_train_data(word2idx, verb2idx, hps)
model = Model(char_weights, action_weights, hps)
# model.load_state_dict(torch.load(hps.test_path + 'models/pre_train.model'))
model = model.cuda()
optimizer = optim.Adam(model.parameters(), lr=hps.lr, weight_decay=hps.weight_decay)
print('pre_training', flush=True)
pre_train(model, optimizer, train_loader, valid_loader, idx2word, hps)
#train
elif args.train:
# word2idx, idx2word = make_vocab(hps)
word2idx, idx2word = load_vocab(hps)
hps = hps._replace(vocab_size=len(word2idx))
print('parameters:')
print(hps)
train_loader, valid_loader, char_weights, action_weights = load_train_data(word2idx, hps)
model = Model(char_weights, action_weights, hps)
# model.load_state_dict(torch.load(hps.test_path + 'models/best.model'))
if args.reload:
model.load_state_dict(torch.load(hps.test_path + hps.save_path.format(args.reload_epoch)))
model.cuda()
# model = nn.DataParallel(model,device_ids=[0])
optimizer = optim.Adam(model.parameters(), lr=hps.lr, weight_decay=hps.weight_decay)
print('training', flush=True)
train(model, optimizer, train_loader, valid_loader, idx2word, hps)
#test
if args.test:
print('testing', flush=True)
word2idx, idx2word = load_vocab(hps)
hps = hps._replace(vocab_size=len(word2idx))
hps = hps._replace(test=True)
model = Model([0] * hps.max_num_char, [0] * hps.vocab_size, hps)
model.load_state_dict(torch.load(hps.test_path + hps.save_path))
model.cuda()
test_loader, anony2names = load_test_data(word2idx, hps)
test(model, test_loader, idx2word, anony2names, hps)
def load_data():
imgs_train, imgs_mask_train = load_train_data()
imgs_train = imgs_train.astype('float32')
imgs_mask_train = imgs_mask_train.astype('float32')
imgs_mask_train /= 255. # scale masks to [0, 1]
print('We have ',imgs_train.shape[0],' training samples')
imgs_test, imgs_id_test = load_test_data()
imgs_test = imgs_test.astype('float32')
print('We have ',imgs_test.shape[0],' test samples')
return imgs_train, imgs_mask_train, imgs_test, imgs_id_test
def train_and_predict():
print("-"*30)
print("Loading train data...")
print('-'*30)
train_images,train_masks = load_train_data()
# train_images = preprocess(train_images)
# train_masks = preprocess(train_masks)
# np.save("./generated_data/train_images_64*80.npy",train_images)
# np.save("./generated_data/train_masks_64*80.npy",train_masks)
train_images = train_images.astype('float32')
mean = np.mean(train_images)
std = np.std(train_images)
train_images -= mean
train_images /= std
train_masks = np.load("./generated_data/train_class.npy")
print('-'*30)
print('Creating and compile model...')
print('-'*30)
model = create_model()
model_checkpoint = ModelCheckpoint('./generated_data/model_2nd_007.hdf5',monitor='loss',save_best_only=True)
print('-'*30)
print('fitting model...')
print('-'*30)
model.fit(train_images,train_masks,batch_size=120,nb_epoch=60,verbose=1,shuffle=True,callbacks=[model_checkpoint])
print('-'*30)
print('Loading test data...')
print('-'*30)
test_images,test_ids = load_test_data()
# test_images = preprocess(test_images)
# np.save("./generated_data/test_images_64*80.npy",test_images)
test_images = test_images.astype('float32')
test_images -= mean
test_images /= std
print('-'*30)
print('Loading saved weights...')
print('-'*30)
test_class = model.predict(test_images,verbose=1)
np.save('./generated_data/test_class_full_model007.npy',test_class)
def train_and_predict():
print('-' * 30)
print('Loading and preprocessing train data...')
print('-' * 30)
imgs_train, imgs_train_label = load_train_data()
imgs_train = imgs_train.astype('float32')
mean = np.mean(imgs_train) # mean for data centering
std = np.std(imgs_train) # std for data normalization
imgs_train -= mean
imgs_train /= std
print('-' * 30)
print('Creating and compiling model...')
print('-' * 30)
model = get_unet()
model_checkpoint = ModelCheckpoint('unet.hdf5',
monitor='loss',
save_best_only=True)
print('-' * 30)
print('Fitting model...')
print('-' * 30)
model.fit(imgs_train,
imgs_train_label,
batch_size=32,
nb_epoch=200,
verbose=1,
shuffle=True,
callbacks=[model_checkpoint])
print('-' * 30)
print('Loading and preprocessing test data...')
print('-' * 30)
imgs_test = load_test_data()
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
print('-' * 30)
print('Loading saved weights...')
print('-' * 30)
model.load_weights('unet.hdf5')
print('-' * 30)
print('Predicting results on test data...')
print('-' * 30)
imgs_test_result = model.predict(imgs_test, verbose=1)
np.save('imgs_test_result.npy', imgs_test_result)
test_result = np.load('imgs_test_result.npy')
np.savetxt("test_result.csv", test_result, delimiter=",")
def train_and_predict():
print('-'*30)
print('Loading and preprocessing train data...')
print('-'*30)
imgs_train, imgs_mask_train = load_train_data()
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
imgs_train = imgs_train.astype('float32')
mean = np.mean(imgs_train) # mean for data centering
std = np.std(imgs_train) # std for data normalization
imgs_train -= mean
imgs_train /= std
imgs_mask_train = imgs_mask_train.astype('float32')
imgs_mask_train /= 255. # scale masks to [0, 1]
imgs_has_mask_train = (imgs_mask_train.reshape(imgs_mask_train.shape[0], -1).max(axis=1) > 0) * 1
print('-'*30)
print('Creating and compiling model...')
print('-'*30)
model = get_unet()
model_checkpoint = ModelCheckpoint('unet.hdf5', monitor='loss', save_best_only=True)
remote_monitor = RemoteMonitor(root='http://localhost:9000')
print('-'*30)
print('Fitting model...')
print('-'*30)
model.fit(imgs_train, [imgs_mask_train, imgs_has_mask_train], batch_size=32, nb_epoch=20, verbose=1, shuffle=True,
callbacks=[model_checkpoint, remote_monitor])
print('-'*30)
print('Loading and preprocessing test data...')
print('-'*30)
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
print('-'*30)
print('Loading saved weights...')
print('-'*30)
model.load_weights('unet.hdf5')
print('-'*30)
print('Predicting masks on test data...')
print('-'*30)
imgs_mask_test = model.predict(imgs_test, verbose=1)[0] # Get output only for first output
np.save('imgs_mask_test.npy', imgs_mask_test)
def train_and_predict():
print('-' * 30)
print('Loading and preprocessing train data...')
print('-' * 30)
"""
imgs_train, imgs_mask_train = load_train_data()
"""
imgs_train = np.load("save/data.npy")
imgs_mask_train = np.load("save/mask.npy")
imgs_train = imgs_train.astype('float32')
imgs_mask_train = imgs_mask_train.astype('float32')
total = imgs_train.shape[0]
print('-' * 30)
print('Creating and compiling model...')
print('-' * 30)
model = get_unet()
""" START COMMENTING """
model_checkpoint = ModelCheckpoint('unet.hdf5',
monitor='loss',
verbose=1,
save_best_only=True)
print('-' * 30)
print('Fitting model...')
print('-' * 30)
model.fit(imgs_train,
imgs_mask_train,
batch_size=1,
nb_epoch=3,
verbose=1,
shuffle=True,
callbacks=[model_checkpoint])
print('-' * 30)
print('Loading and preprocessing test data...')
print('-' * 30)
imgs_test, imgs_id_test = load_test_data()
imgs_test = imgs_test.astype('float32')
imgs_test -= np.load('save/mean.npy')
imgs_test /= np.load('save/std.npy')
print('-' * 30)
print('Loading saved weights...')
print('-' * 30)
model.load_weights('unet.hdf5')
print('-' * 30)
print('Predicting masks on test data...')
print('-' * 30)
imgs_mask_test = model.predict(imgs_test, verbose=1)
np.save('imgs_mask_test.npy', imgs_mask_test)
def train_and_predict():
print('-'*30)
print('Loading and preprocessing train data...')
print('-'*30)
imgs_train, imgs_mask_train = load_train_data()
imgs_train, imgs_mask_train = preprocess(imgs_train, imgs_mask_train)
imgs_train = imgs_train.astype('float32')
mean = np.mean(imgs_train) # mean for data centering
std = np.std(imgs_train) # std for data normalization
imgs_train -= mean
imgs_train /= std
imgs_mask_train = imgs_mask_train.astype('float32')
imgs_mask_train /= 255. # scale masks to [0, 1]
imgs_train = imgs_train[:30]
imgs_mask_train = imgs_mask_train[:30]
print('-'*30)
print('Creating and compiling model...')
print('-'*30)
model = get_model(img_rows, img_cols)
model_checkpoint = ModelCheckpoint('unet.hdf5', monitor='loss', save_best_only=True)
print('-'*30)
print('Fitting model...')
print('-'*30)
#lrreducer = SimpleLrReducer(5, .94)
model.fit(imgs_train, imgs_mask_train, batch_size=5, nb_epoch=250, verbose=1, shuffle=True,
callbacks=[model_checkpoint], validation_split=0.15)
print('-'*30)
print('Loading and preprocessing test data...')
print('-'*30)
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
print('-'*30)
print('Loading saved weights...')
print('-'*30)
model.load_weights('unet.hdf5')
print('-'*30)
print('Predicting masks on test data...')
print('-'*30)
imgs_mask_test = model.predict(imgs_test, verbose=1)
np.save('imgs_mask_test.npy', imgs_mask_test)
def train_and_predict():
print('-'*30)
print('Loading and preprocessing train data...')
print('-'*30)
imgs_train, imgs_mask_train = load_train_data()
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
imgs_train = imgs_train.astype('float32')
mean = np.mean(imgs_train) # mean for data centering
std = np.std(imgs_train) # std for data normalization
imgs_train -= mean
imgs_train /= std
imgs_mask_train = imgs_mask_train.astype('float32')
imgs_mask_train /= 255. # scale masks to [0, 1]
print('-'*30)
print('Creating and compiling model...')
print('-'*30)
model = get_unet()
model_checkpoint = ModelCheckpoint('unet.hdf5', monitor='loss', save_best_only=True)
print('-'*30)
print('Fitting model...')
print('-'*30)
model.fit(imgs_train, imgs_mask_train, batch_size=32, nb_epoch=50, verbose=1, shuffle=True,
callbacks=[model_checkpoint])
print('-'*30)
print('Loading and preprocessing test data...')
print('-'*30)
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
tst_mean = np.mean(imgs_test)
tst_std = np.std(imgs_test)
imgs_test -= tst_mean
imgs_test /= tst_std
print('-'*30)
print('Loading saved weights...')
print('-'*30)
model.load_weights('unet.hdf5')
print('-'*30)
print('Predicting masks on test data...')
print('-'*30)
imgs_mask_test = model.predict(imgs_test, verbose=1)
np.save('imgs_mask_test.npy', imgs_mask_test)
def run_predict(): texts = data.load_test_data() input, word_index = embed.sequence_vectorize(texts) # Create a new model instance model = SavedModel() model.build(MODEL_PATH) # Re-evaluate the model predictions = model.predict(input)
def run_test():
BS = 128
print('Loading and preprocessing test data...')
mean, std = Learner.load_meanstd()
imgs_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
print('Loading saved weights...')
model = get_unet(Adam(0.001))
print ('Loading weights from %s' % Learner.best_weight_path)
model.load_weights(Learner.best_weight_path)
print ('Augment')
alen, dlen = len(transforms), len(imgs_test)
test_x = np.ndarray((alen, dlen, 1, img_rows, img_cols), dtype=np.float32)
for i in range(dlen):
for j, transform in enumerate(transforms):
test_x[j,i] = transform['do'](imgs_test[i].copy())
#
print('Predicting masks on test data...')
outputs = []
asis_res = model.predict(imgs_test, batch_size=BS, verbose=1)
outputs.append(asis_res)
for j, transform in enumerate(transforms):
t_y = model.predict(test_x[j], batch_size=BS, verbose=1)
outputs.append(t_y)
#
print('Analyzing')
test_masks = np.ndarray((dlen, 1, img_rows, img_cols), dtype=np.float32)
test_probs = np.ndarray((dlen, ), dtype=np.float32)
for i in range(dlen):
masks = np.ndarray((alen+1, 1, img_rows, img_cols), dtype=np.float32)
probs = np.ndarray((alen+1, ), dtype=np.float32)
for j, t_y in enumerate(outputs):
mask, prob = t_y[0][i], t_y[1][i]
if j:
mask = transforms[j-1]['undo'](mask)
masks[j] = mask
probs[j] = prob
#
test_masks[i] = np.mean(masks, 0)
test_probs[i] = np.mean(probs)
print('Saving ')
np.save(Learner.test_mask_res, test_masks)
np.save(Learner.test_mask_exist_res, test_probs)
def test(self, model, batch_size=256):
print('Loading and pre-processing test data...')
imgs_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = self.standartize(imgs_test, to_float=True)
imgs_test = imgs_test.transpose((0, 2, 3, 1))
print('Loading best saved weights...')
model.load_weights(self.best_weight_path)
print('Predicting masks on test data and saving...')
imgs_mask_test = model.predict(imgs_test, batch_size=batch_size, verbose=1)
np.save(self.test_mask_res, imgs_mask_test[0])
np.save(self.test_mask_exist_res, imgs_mask_test[1])
def train_and_predict():
print('-' * 30)
print('Loading and preprocessing train data...')
print('-' * 30)
imgs_train, imgs_mask_train = load_train_data()
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
imgs_train = imgs_train.astype('float32')
mean = np.mean(imgs_train) # mean for data centering
std = np.std(imgs_train) # std for data normalization
imgs_train -= mean
imgs_train /= std
imgs_mask_train = imgs_mask_train.astype('float32')
imgs_mask_train /= 255. # scale masks to [0, 1]
print('-' * 30)
print('Fitting model...')
print('-' * 30)
n_fold = 5
k_fold_train(imgs_train, imgs_mask_train, n_fold=n_fold)
print('-' * 30)
print('Loading and preprocessing test data...')
print('-' * 30)
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
print('-' * 30)
print('Loading saved weights...')
print('-' * 30)
model = get_unet()
results = []
for i in range(n_fold):
model.load_weights('unet_fold%s.hdf5' % i)
print('-' * 30)
print('%s Predicting masks on test data...' % i)
print('-' * 30)
imgs_mask_test = model.predict(imgs_test, verbose=1)
results.append(imgs_mask_test)
imgs_mask_test = reduce(lambda x, y: x + y, results) / n_fold
np.save('imgs_mask_test_nfold.npy', imgs_mask_test)
def train_and_predict():
print('-'*30)
print('Loading and preprocessing train data...')
print('-'*30)
imgs_train, imgs_mask_train = load_train_data()
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
imgs_train = imgs_train.astype('float32')
mean = np.mean(imgs_train) # mean for data centering
std = np.std(imgs_train) # std for data normalization
imgs_train -= mean
imgs_train /= std
imgs_mask_train = imgs_mask_train.astype('float32')
imgs_mask_train /= 255. # scale masks to [0, 1]
print('-'*30)
print('Fitting model...')
print('-'*30)
n_fold = 5
k_fold_train(imgs_train, imgs_mask_train, n_fold=n_fold)
print('-'*30)
print('Loading and preprocessing test data...')
print('-'*30)
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
print('-' * 30)
print('Loading saved weights...')
print('-' * 30)
model = get_unet()
results = []
for i in range(n_fold):
model.load_weights('unet_fold%s.hdf5' % i)
print('-' * 30)
print('%s Predicting masks on test data...' % i)
print('-' * 30)
imgs_mask_test = model.predict(imgs_test, verbose=1)
results.append(imgs_mask_test)
imgs_mask_test = reduce(lambda x, y: x + y, results)/n_fold
np.save('imgs_mask_test_nfold.npy', imgs_mask_test)
def apply_mask(mode):
print_heading('Apply mask to test images ...')
pred_dir = os.path.join(root_path, 'preds')
imgs_test, imgs_id_test = load_test_data(os.path.join(root_path, mode))
for image, image_id in zip(imgs_test, imgs_id_test):
pred_image_name = os.path.join(pred_dir, str(image_id) + '.pred.png')
pred_image_mask = imread(pred_image_name)
extracted = extract_image(pred_image_mask, image)
if extracted is None:
print("miss", image_id)
else:
imsave(os.path.join(pred_dir, image_id + '.result.jpg'), extracted)
def evaluate():
model = Model()
img_tf = tf.placeholder(tf.float32)
with tf.Graph().as_default():
images, labels = data.load_test_data()
print('size', images.shape)
x = tf.placeholder(shape=[None, data.IMAGE_SIZE, data.IMAGE_SIZE, 3],
dtype=tf.float32,
name='x')
y = tf.placeholder(shape=[None, data.IMAGE_SIZE, data.IMAGE_SIZE],
dtype=tf.float32,
name='y')
infered = model.inference(x,
keep_prob=1.0,
train=False,
avg_ch_array=avg_channel_color)
loss = model.loss(inference=infered, labels=y)
init = tf.global_variables_initializer()
#accuracy = model.accuracy(logits, y)
img_tf = infered
saver = tf.train.Saver()
results = []
loss_array = []
with tf.Session() as sess:
sess.run(init)
saver.restore(sess, FLAGS.checkpoint_file_path)
# total_accuracy = sess.run([accuracy])
# print('Test Error: {}'.format(total_accuracy))
#For every image
for i in range(len(images)):
offset = (i * FLAGS.batch_size) % (len(images) -
FLAGS.batch_size)
batch_x, batch_y = images[offset:(
offset +
FLAGS.batch_size), :], labels[offset:(offset +
FLAGS.batch_size), :]
infered_image = sess.run(infered, feed_dict={x: batch_x})
results.append(infered_image)
cur_loss = sess.run(loss, feed_dict={x: batch_x, y: batch_y})
print(i, np.round(cur_loss, decimals=1))
loss_array.append(cur_loss)
np.save('./robe_face/results/results_robe.npy', np.array(results))
np.save('./robe_face/results/loss_results_robe.npy',
np.array(loss_array))
def train():
imgs_train, imgs_mask_train = load_train_data()
imgs_train = imgs_train.astype('float32')
mean = np.mean(imgs_train) # mean for data centering
std = np.std(imgs_train) # std for data normalization
imgs_train -= mean
imgs_train /= std
imgs_mask_train = imgs_mask_train.astype('float32')
imgs_mask_train /= 255. # scale masks to [0, 1]
model = get_unet()
model_checkpoint = ModelCheckpoint('weights.h5',
monitor='val_loss',
save_best_only=True)
model.fit(imgs_train,
imgs_mask_train,
batch_size=32,
nb_epoch=20,
verbose=1,
shuffle=True,
validation_split=0.2,
callbacks=[model_checkpoint])
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
model.load_weights('weights.h5')
imgs_mask_test = model.predict(imgs_test, verbose=1)
np.save('imgs_mask_test.npy', imgs_mask_test)
pred_dir = 'preds'
if not os.path.exists(pred_dir):
os.mkdir(pred_dir)
for image, image_id in zip(imgs_mask_test, imgs_id_test):
image = (image[:, :, 0] * 255.).astype(np.uint8)
imsave(os.path.join(pred_dir, str(image_id) + '_pred.png'), image)
def just_predict():
imgs_train, imgs_mask_train = load_train_data()
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
print('imgs_train = ',len(imgs_train))
print('imgs_mask_train = ',len(imgs_mask_train))
shuffle = np.random.permutation(len(imgs_train))
imgs_train = imgs_train[shuffle]
imgs_mask_train = imgs_mask_train[shuffle]
imgs_train = imgs_train.astype('float32')
mean = np.mean(imgs_train) # mean for data centering
std = np.std(imgs_train) # std for data normalization
print('-'*30)
print('Loading and preprocessing test data...')
print('-'*30)
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
model = get_unet()
print('-'*30)
print('Loading saved weights...')
print('-'*30)
model.load_weights('weights.h5')
print('-'*30)
print('Predicting masks on test data...')
print('-'*30)
imgs_mask_test = model.predict(imgs_test, verbose=1)
imgs_mask_test = postprocess(imgs_mask_test)
np.save('imgs_mask_test.npy', imgs_mask_test)
def main():
"""
The main function.
Arguments:
1. Takes no arguments.
"""
train_data = data.load_training_data()
# function call to load training data
test_data = data.load_test_data()
# function call to load test data
count = CountVectorizer()
# initialize the count vector
tfidf_transformer = TfidfTransformer()
# initialize a tfidf transformer
models_dict = {}
# empty dict
train_tfidf = features.feature_extraction(train_data, count,
tfidf_transformer)
# function call for feature extraction
bayes = naive_bayes(train_data, train_tfidf)
# function call to fit the Naive Bayes classifier
models_dict['Naive Bayes'] = bayes
# add models to dictionary
svm = svm_classifier(train_data, train_tfidf)
# function call to fit SVM Classifier
models_dict['SVM'] = svm
# add models to a dictionary
rand_forest = random_forest_classifier(train_data, train_tfidf)
# function to build random forest classifier
models_dict['Random Forest'] = rand_forest
# add models to dictionary
logistic = logistic_regression_classifier(train_data, train_tfidf)
# function call to build logistic regression
models_dict['Logistic Regression'] = logistic
# add models to dictionary
decision_tree = decision_tree_classifier(train_data, train_tfidf)
# function call for decision tree classifier
models_dict['Decision Tree'] = decision_tree
# add model to the dictionary
predict_test_data(train_data, test_data, models_dict, count,
tfidf_transformer, train_tfidf)
def test(network, submission_file):
network.to(config.device)
test_dataloader = data.load_test_data()
network.eval()
outputs = []
with torch.no_grad():
for image, image_id in test_dataloader:
output = network(image)
outputs.append((image_id[0], output[0]))
csv_file = open(submission_file, 'w')
csv_writer = csv.writer(csv_file)
csv_writer.writerow(['Image_ID', 'x', 'y', 'w', 'h'])
for image_id, output in outputs:
csv_writer.writerow([
image_id, output[0].item(), output[1].item(), output[2].item(),
output[3].item()
])
csv_file.close()
def submission(): # load the data v_data = load_valid_data() t_data = load_test_data() # calculate the average loss print "===== #1 =====" # ====================== # Task number 1 # ====================== start = 0 end = 100 mistakes = [] # Train from steps 0 - 1000 for i in range(0, 10): # Train using the first kernel mistakes = train(v_data[start:end], kernel_one, 1, mistakes) avg = calculate_loss(end, mistakes) print " Average on step ", start, "-", end , ": ", avg # next start = end end += 100 print print "===== #2 =====" # =================== # Task number 2 # =================== steps = 1000 d = [1, 3, 5, 7, 10, 15, 20] # Try over all d for p in d: mistakes = [] # Train using the second kernel mistakes = train(v_data[:steps], kernel_two, p, mistakes) avg = calculate_loss(steps, mistakes) print " For d ", p , " steps=", steps, ": ", avg print print "===== #3 =====" # ===================== # Task number 3 # ===================== # choosen d = 5 # do kernel two first d = 5 start = 0 end = 100 mistakes = [] # Iterate from steps 0 - 1000 for i in range(0, 10): # train using second kernel with d = 5 mistakes = train(t_data[start:end], kernel_two, d, mistakes) avg = calculate_loss(end, mistakes) print " d=", d, " k2 average on step ", start, "-", end , ": ", avg start = end end += 100 print # do kernel three now sigma = 10 start = 0 end = 100 mistakes = [] # Iterate from steps 0 - 1000 for i in range(0, 10): # Train using the third kernel with sigma = 10 mistakes = train(t_data[start:end], kernel_three, sigma, mistakes) avg = calculate_loss(end, mistakes) print " sigma=", sigma, " k3 average on step ", start, "-", end , ": ", avg start = end end += 100
verbose=1,
max_epochs=150,
on_epoch_finished=[
AdjustVariable('update_learning_rate',
epochs=[50,100],rates=[2e-3,2e-4])],
regularization_rate=1e-5,
batch_iterator_train=BatchIterator(batch_size=128)
)
return net0
np.random.seed(1)
X,y,encoder,scaler,ids_val= data.load_train_data('data/train.csv')
X_test,ids=data.load_test_data('data/test.csv',scaler)
num_classes=len(encoder.classes_)
num_features=X.shape[1]
scores=[]
p=None
folds=5
eps=1e-4
delta=1
num=0
scores=np.zeros((y.shape[0],9))
prev_loss=10
while (delta>eps):
def problem3():
data = load_test_data()
lp, le = solve3(data)
graph_p3(lp, le)
p_keep_conv = tf.placeholder("float")
p_keep_hidden = tf.placeholder("float")
py_x = model(X, w, w2, w3, w4, w_o, p_keep_conv, p_keep_hidden)
# Define loss and optimizer
cost = dice_coef_loss(py_x, Y)
train_op = tf.train.AdamOptimizer(0.0001).minimize(cost)
predict_op = tf.argmax(py_x, 1)
print('-'*30)
print('Loading and preprocessing test data...')
print('-'*30)
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
# Launch the graph in a session
with tf.Session() as sess:
# you need to initialize all variables
tf.initialize_all_variables().run()
for i in range(100):
training_batch = zip(range(0, len(trX), batch_size),
range(batch_size, len(trX), batch_size))
for start, end in training_batch:
def train_and_predict():
print('-'*30)
print('Loading and preprocessing train data...')
print('-'*30)
imgs_train, imgs_mask_train = load_train_data()
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
imgs_train = imgs_train.astype('float32')
mean = np.mean(imgs_train) # mean for data centering
std = np.std(imgs_train) # std for data normalization
imgs_train -= mean
imgs_train /= std
imgs_mask_train = imgs_mask_train.astype('float32')
imgs_mask_train /= 255. # scale masks to [0, 1]
y_bin = np.array([mask_not_blank(mask) for mask in imgs_mask_train ])
# X_train,X_test,y_train,y_test = train_test_split(imgs_train,imgs_mask_train,test_size=0.2,random_state=seed)
skf = StratifiedKFold(y_bin, n_folds=10, shuffle=True, random_state=seed)
for ind_tr, ind_te in skf:
X_train = imgs_train[ind_tr]
X_test = imgs_train[ind_te]
y_train = imgs_mask_train[ind_tr]
y_test = imgs_mask_train[ind_te]
break
X_train_flip = X_train[:,:,:,::-1]
y_train_flip = y_train[:,:,:,::-1]
X_train = np.concatenate((X_train,X_train_flip),axis=0)
y_train = np.concatenate((y_train,y_train_flip),axis=0)
X_train_flip = X_train[:,:,::-1,:]
y_train_flip = y_train[:,:,::-1,:]
X_train = np.concatenate((X_train,X_train_flip),axis=0)
y_train = np.concatenate((y_train,y_train_flip),axis=0)
imgs_train = X_train
imgs_valid = X_test
imgs_mask_train = y_train
imgs_mask_valid = y_test
print('-'*30)
print('Creating and compiling model...')
print('-'*30)
model = get_unet()
model_name = 'unet_seed_1024_epoch_20_aug_64_80_shiftbn_sgd_srelu_plus10.hdf5'
model_checkpoint = ModelCheckpoint('E:\\UltrasoundNerve\\'+model_name, monitor='loss', save_best_only=True)
plot(model, to_file='E:\\UltrasoundNerve\\%s.png'%model_name.replace('.hdf5',''),show_shapes=True)
print('-'*30)
print('Fitting model...')
print('-'*30)
augmentation=False
batch_size=128
nb_epoch=10
load_model=True
use_all_data = False
if use_all_data:
imgs_train = np.concatenate((imgs_train,imgs_valid),axis=0)
imgs_mask_train = np.concatenate((imgs_mask_train,imgs_mask_valid),axis=0)
if load_model:
model.load_weights('E:\\UltrasoundNerve\\'+model_name)
if not augmentation:
# model.fit(imgs_train, imgs_mask_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1, shuffle=True,
# callbacks=[model_checkpoint],
# validation_data=[imgs_valid,imgs_mask_valid]
# )
pass
else:
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=45, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=0.0, # randomly shift images horizontally (fraction of total width)
height_shift_range=0.0, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=False) # randomly flip images
# compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied)
datagen.fit(imgs_train)
# fit the model on the batches generated by datagen.flow()
model.fit_generator(datagen.flow(imgs_train, imgs_mask_train,
batch_size=batch_size),
samples_per_epoch=imgs_train.shape[0],
nb_epoch=nb_epoch,
callbacks=[model_checkpoint],
validation_data=(imgs_valid,imgs_mask_valid))
print('-'*30)
print('Loading and preprocessing test data...')
print('-'*30)
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
print('-'*30)
print('Loading saved weights...')
print('-'*30)
model.load_weights('E:\\UltrasoundNerve\\'+model_name)
print('-'*30)
print('Predicting masks on test data...')
print('-'*30)
imgs_mask_test = model.predict(imgs_test, verbose=1)
np.save('imgs_mask_test.npy', imgs_mask_test)
def train_and_predict():
print('-' * 30)
print('Loading and preprocessing train data...')
print('-' * 30)
imgs_train, imgs_mask_train = load_train_data()
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
imgs_train = imgs_train.astype('float32')
mean = np.mean(imgs_train) # mean for data centering
std = np.std(imgs_train) # std for data normalization
imgs_train -= mean
imgs_train /= std
imgs_mask_train = imgs_mask_train.astype('float32')
imgs_mask_train /= 255. # scale masks to [0, 1]
y_hat_train = np.max(imgs_mask_train, axis=(1, 2, 3))
print("total:", y_hat_train.shape, np.unique(y_hat_train), "label-1:", np.sum(y_hat_train))
y_hat_train_sums = np.sum(imgs_mask_train, axis=(1, 2, 3))
print(y_hat_train_sums)
y_hat_train_sums_nonzero_ids = np.nonzero(y_hat_train_sums)[0]
y_hat_train_sums = y_hat_train_sums[y_hat_train_sums_nonzero_ids]
print(y_hat_train_sums.shape, np.min(y_hat_train_sums), np.max(y_hat_train_sums), np.mean(y_hat_train_sums))
# y = np.bincount(y_hat_train_sums.astype('int32'))
# ii = np.nonzero(y)[0]
# count = y[ii]
# from matplotlib import pyplot as plt
# plt.plot(ii, count)
# plt.show()
# raw_input("pause, input any to continue")
print('-' * 30)
print('Fitting model...')
print('-' * 30)
n_fold = 5
k_fold_train(imgs_train, y_hat_train, n_fold=n_fold)
print('-' * 30)
print('Loading and preprocessing test data...')
print('-' * 30)
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
print('-' * 30)
print('Loading saved weights...')
print('-' * 30)
# model = get_unet()
model = get_keras_example_net()
results = []
for i in range(n_fold):
model.load_weights('unet_fold%s.hdf5' % i)
print('-' * 30)
print('%s Predicting masks on test data...' % i)
print('-' * 30)
imgs_mask_test = model.predict(imgs_test, verbose=1)
results.append(imgs_mask_test)
imgs_mask_test = reduce(lambda x, y: x + y, results) / n_fold
np.save('imgs_mask_test_nfold.npy', imgs_mask_test)
def train_and_predict():
print('-'*30)
print('Loading and preprocessing train data...')
print('-'*30)
imgs_train, coeffs_train = load_train_data()
masked = np.ma.masked_values(coeffs_train[:,0],0.0)
imgs_train = imgs_train[~masked.mask,...]
coeffs_train = coeffs_train[~masked.mask,...]
imgs_train = preprocess(imgs_train)
imgs_train = imgs_train.astype('float32')
mean = np.mean(imgs_train) # mean for data centering
std = np.std(imgs_train) # std for data normalization
imgs_train -= mean
imgs_train /= std
coeffs_train = coeffs_train[:,0:2]
coeffs_train = coeffs_train.astype('float32')
print('-'*30)
print('Creating and compiling model...')
print('-'*30)
model = resnet(img_rows,img_cols)
model.compile(optimizer=Adam(lr=1e-5), loss='mean_squared_error')
model_checkpoint = ModelCheckpoint('resnet.hdf5', monitor='loss', save_best_only=True)
print('-'*30)
print('Fitting model...')
print('-'*30)
#print (coeffs_train)
prog = ProgbarLogger()
model.fit(imgs_train, coeffs_train, batch_size=64, nb_epoch=20, verbose=1, shuffle=True,
callbacks=[prog,model_checkpoint],validation_split = 0.2)
coeffs_train = model.predict(imgs_train, verbose=1)
np.save('coeffs_train_predicted.npy',coeffs_train)
print('-'*30)
print('Loading and preprocessing valid data...')
print('-'*30)
imgs_valid, coeffs_valid = load_valid_data()
imgs_valid = preprocess(imgs_valid)
imgs_valid = imgs_valid.astype('float32')
imgs_valid -= mean
imgs_valid /= std
print('-'*30)
print('Loading saved weights...')
print('-'*30)
model.load_weights('resnet.hdf5')
print('-'*30)
print('Predicting on valid data...')
print('-'*30)
coeffs_valid_predicted = model.predict(imgs_valid, verbose=1)
np.save('coeffs_valid_predicted.npy', coeffs_valid_predicted)
print('-'*30)
print('Loading and preprocessing valid data...')
print('-'*30)
imgs_valid, coeffs_valid = load_valid_data()
imgs_valid = preprocess(imgs_valid)
imgs_valid = imgs_valid.astype('float32')
imgs_valid -= mean
imgs_valid /= std
print('-'*30)
print('Loading saved weights...')
print('-'*30)
model.load_weights('unet.hdf5')
print('-'*30)
print('Predicting on valid data...')
print('-'*30)
coeffs_valid_predicted = model.predict(imgs_valid, verbose=1)
np.save('coeffs_valid_predicted.npy', coeffs_valid_predicted)
print('-'*30)
print('Loading and preprocessing test data...')
print('-'*30)
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
print('-'*30)
print('Loading saved weights...')
print('-'*30)
model.load_weights('resnet.hdf5')
print('-'*30)
print('Predicting on test data...')
print('-'*30)
coeffs_test = model.predict(imgs_test, verbose=1)
np.save('coeffs_test.npy', coeffs_test)
dif = coeffs_valid[:,0:2] - coeffs_valid_predicted
print(dif.shape)
dif = dif[coeffs_valid[:,0]>0,:]
print(dif.shape)
check = np.mean(np.sum(dif*dif,axis=1))
print('check',check)
def train_and_predict():
stats = {}
pp_head(config)
open('config.txt', 'w').write(str(config))
if True or config['FIT']:
pp_head('Loading and preprocessing train data...')
imgs_train, imgs_mask_train = load_train_data()
imgs_train = preprocess(imgs_train)
imgs_mask_train = preprocess(imgs_mask_train)
imgs_train = imgs_train.astype('float32')
stats['mean'] = np.mean(imgs_train) # mean for data centering
stats['std'] = np.std(imgs_train) # std for data normalization
imgs_train -= stats['mean']
imgs_train /= stats['std']
imgs_mask_train = imgs_mask_train.astype('float32')
imgs_mask_train /= 255. # scale masks to [0, 1]
open('stats.txt', 'w').write(str(stats))
else:
stats = eval(open('stats.txt', 'r').read()) # Read previously saved values from a file, needed to transform test images
pp_head('Creating and compiling model...')
if config['LOAD_MODEL']:
model = model_from_json(open('my_model_architecture.json').read())
else:
model = get_unet()
json_string = model.to_json()
open('my_model_architecture.json', 'w').write(json_string)
if config['LOAD_LAST']:
model.load_weights('unet.hdf5')
if config['FIT']:
pp_head('Fitting model...')
model_checkpoint = ModelCheckpoint('unet.hdf5', monitor='loss', save_best_only=True)
model_checkpoint2 = ModelCheckpoint("weights.{epoch:02d}-{loss:.2f}.hdf5", monitor='loss', save_best_only=True)
model.fit(imgs_train, imgs_mask_train,validation_split=config['VALIDATION'], batch_size=config['BATCH'], nb_epoch=config['EPOCH'], verbose=1, shuffle=True,
callbacks=[model_checkpoint,model_checkpoint2]) # batch size originally 32
#else:
# model.test_on_batch(imgs_train, imgs_mask_train)
pp_head(str(model.summary()))
pp_head('Loading and preprocessing test data...')
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test, True)
imgs_test = imgs_test.astype('float32')
imgs_test -= stats['mean']
imgs_test /= stats['std']
pp_head('Loading saved weights...')
model.load_weights('unet.hdf5')
pp_head('Predicting masks on test data...')
imgs_mask_test = model.predict(imgs_test, verbose=1) # USe batch to speed up on large picture
#imgs_mask_test = model.predict(imgs_test,1, verbose=1) # USe batch to speed up on large picture
np.save('imgs_mask_test.npy', imgs_mask_test)
x_train, y_train_mask = load_train_data()
x_train = preprocess(x_train)
y_train_mask = preprocess(y_train_mask)
x_train = x_train.astype('float32')
mean = np.mean(x_train)
std = np.std(x_train)
x_train -= mean
x_train /= std
y_train_mask = y_train_mask.astype('float32')
y_train_mask /= 255.
unet.fit(x_train, y_train_mask, batch_size=32, epochs=20, verbose=1, shuffle=True,
validation_split=0.2)
x_test, y_test_mask = load_test_data()
x_test = preprocess(x_test)
x_test = x_test.astype('float32')
x_test -= mean
x_test /= std
y_test_pred = unet.predict(x_test, verbose=1)
for image, image_id in zip(y_test_pred, y_test_mask):
image = (image[:, :, 0] * 255.).astype(np.uint8)
imsave(os.path.join(work_dir, str(image_id) + '.png'), image)
def setUp(self):
self.train_data = extract_features(load_train_data())
self.test_data = extract_features(load_test_data())
def train_and_predict():
print('-'*30)
print('Loading and preprocessing train data...')
print('-'*30)
# imgs_train, imgs_mask_train = load_train_data()
imgs_train=np.load("/mnt/data1/yihuihe/mnc/data.npy")
imgs_mask_train=np.load("/mnt/data1/yihuihe/mnc/mask.npy")
imgs_train = imgs_train.astype('float32')
imgs_mask_train = imgs_mask_train.astype('float32')
# imgs_train = preprocess(imgs_train)
# imgs_mask_train = preprocess(imgs_mask_train)
# print(np.histogram(imgs_train))
# print(np.histogram(imgs_mask_train))
total=imgs_train.shape[0]
# imgs_train/=255.
# mean = imgs_train.mean()# (0)[np.newaxis,:] # mean for data centering
# std = np.std(imgs_train) # std for data normalization
# imgs_train -= mean
# imgs_train /= std
# imgs_mask_train /= 255. # scale masks to [0, 1]
print('-'*30)
print('Creating and compiling model...')
print('-'*30)
model = get_unet()
# model_checkpoint = ModelCheckpoint('unet.hdf5', monitor='loss',verbose=1, save_best_only=True)
# print('-'*30)
# print('Fitting model...')
# print('-'*30)
# model.fit(imgs_train, imgs_mask_train, batch_size=32, nb_epoch=20, verbose=1, shuffle=True,callbacks=[model_checkpoint])
# batch_size=32
# max_iters=10000
# for i in range(max_iters):
# data_batch=np.ndarray((batch_size,1,img_rows,img_cols))
# mask_batch=np.ndarray((batch_size,1,img_rows,img_cols))
# for img in range(batch_size):
# idx=np.random.randint(total)
# data_batch[img,0],mask_batch[img,0]=augmentation(imgs_train[idx],imgs_mask_train[idx])
# # plt.subplot(121)
# # plt.imshow(data_batch[img,0])
# # plt.subplot(122)
# # plt.imshow(mask_batch[img,0])
# # plt.show()
# data_batch-=mean
# data_batch/=std
# print(np.histogram(data_batch))
# print(np.histogram(mask_batch))
# model.train_on_batch(data_batch,mask_batch)
print('-'*30)
print('Loading and preprocessing test data...')
print('-'*30)
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test) # TODO: bug
imgs_test = imgs_test.astype('float32')
imgs_test -= np.load('/mnt/data1/yihuihe/mnc/mean.npy')
imgs_test /=np.load('/mnt/data1/yihuihe/mnc/std.npy')
print('-'*30)
print('Loading saved weights...')
print('-'*30)
model.load_weights('unet.hdf5')
print('-'*30)
print('Predicting masks on test data...')
print('-'*30)
imgs_mask_test = model.predict(imgs_test, verbose=1)
np.save('imgs_mask_test.npy', imgs_mask_test)
def main():
# import time
# start = time.time()
# model = resnet()
# duration = time.time() - start
# print("{} s to make model".format(duration))
# start = time.time()
# model.output
# duration = time.time() - start
# print("{} s to get output".format(duration))
# start = time.time()
# model.compile(loss="categorical_crossentropy", optimizer="sgd")
# duration = time.time() - start
# print("{} s to get compile".format(duration))
# current_dir = os.path.dirname(os.path.realpath(__file__))
# model_path = os.path.join(current_dir, "resnet_50.png")
# plot(model, to_file=model_path, show_shapes=True)
# exit()
# -----------------------------------------------------------------------------
print('-'*30)
print('Loading and preprocessing train data...')
print('-'*30)
imgs_train, imgs_mask_train = load_train_data()
imgs_train = preprocess(imgs_train, rows,cols)
imgs_mask_train = preprocess(imgs_mask_train, rows/2,cols/2)
imgs_train = imgs_train.astype('float32')
mean = imgs_train.mean(0)[np.newaxis,:] # mean for data centering
std = np.std(imgs_train) # std for data normalization
imgs_train -= mean
imgs_train /= std
imgs_mask_train = imgs_mask_train.astype('float32')
imgs_mask_train /= 255. # scale masks to [0, 1]
print('-'*30)
print('Creating and compiling model...')
print('-'*30)
model = resnet()
# model.load_weights('resnet.hdf5')
model_checkpoint = ModelCheckpoint('resnet.hdf5', monitor='loss',verbose=1, save_best_only=True)
# -----------------------------------------------------------------------
print('-'*30)
print('Fitting model...')
print('-'*30)
model.fit(imgs_train, imgs_mask_train, batch_size=32, nb_epoch=20, verbose=1, shuffle=True, callbacks=[model_checkpoint])
# for i in range(3):
# model.train(imgs_train[:3],imgs_mask_train[:3])
print('-'*30)
print('Loading and preprocessing test data...')
print('-'*30)
imgs_test, imgs_id_test = load_test_data()
imgs_test = preprocess(imgs_test, rows,cols)
imgs_test = imgs_test.astype('float32')
imgs_test -= mean
imgs_test /= std
print('-'*30)
print('Loading saved weights...')
print('-'*30)
model.load_weights('resnet.hdf5')
print('-'*30)
print('Predicting masks on test data...')
print('-'*30)
imgs_mask_test = model.predict(imgs_test, verbose=1)
np.save('imgs_mask_test.npy', imgs_mask_test)
print('-'*30)
print('Predicting masks on train data...')
print('-'*30)
imgs_mask_test = model.predict(imgs_train, verbose=1)
np.save('imgs_train_pred.npy', imgs_mask_test)