def make_prediction_movie(image_dir, tissue=True):
model = get_model()
if tissue:
model.load_weights('model_tissue.h5')
vid_name = "tissue.mp4"
if image_dir:
bg = batch_generator(image_dir=image_dir, label_dir=None)
else:
bg = batch_generator(
image_dir=r"..\data\optical\062A\original\raw_image\*.png",
label_dir=None)
else:
model.load_weights('model_aperture.h5')
vid_name = "aperture.mp4"
if image_dir:
bg = batch_generator(image_dir=image_dir, label_dir=None)
else:
bg = batch_generator(
image_dir=r"..\data\optical\062A\original\raw_image\*.png",
label_dir=None)
vid_fourcc = cv2.VideoWriter_fourcc(*'XVID')
vid_out = cv2.VideoWriter(vid_name, vid_fourcc, 15.0, (640, 480))
font = cv2.FONT_HERSHEY_SIMPLEX
for index in range(bg.image_count):
img, mask8 = bg.get_image_and_mask(index, source='all', augment=True)
mask = np.zeros_like(img)
# mask[:,:,2] = mask8
y_pred = model.predict(img[None, ...].astype(np.float32))[0]
y_pred = y_pred.reshape((IMAGE_H, IMAGE_W, NUMBER_OF_CLASSES))
y_pred8 = y_pred[:, :, 0] * 255
y_pred8 = y_pred8.astype(np.uint8)
mask[:, :, 0] = y_pred8
alpha = 0.5
# cv2.addWeighted(mask, alpha, img, 1 - alpha, 0, img)
cv2.putText(img, bg.images[index], (21, 21), font, 0.25, (0, 0, 0), 1,
cv2.LINE_AA)
cv2.putText(img, bg.images[index], (20, 20), font, 0.25,
(255, 255, 255), 1, cv2.LINE_AA)
# outline contour
ret, thresh = cv2.threshold(y_pred8, 128, 255, 0)
im2, contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
cv2.drawContours(img, contours, -1, (0, 255, 0), 2)
vid_out.write(img)
#cv2.imshow('img',img)
# key = cv2.waitKey(0)
# if key == 27: # esc
# break
#cv2.destroyAllWindows()
vid_out.release()
def experiment(estimator,
train_df,
valid_df,
bg_paths,
batch_size,
sample_size,
version_path=None,
csv_log_path=None):
label_num = len(config.POSSIBLE_LABELS)
train_generator = generator.batch_generator(train_df,
batch_size,
label_num,
bg_paths,
sampling_size=sample_size)
valid_generator = generator.batch_generator(valid_df,
batch_size,
label_num,
bg_paths,
mode='valid',
sampling_size=sample_size)
valid_steps = int(np.ceil(valid_df.shape[0] / batch_size))
steps_per_epoch = int(np.ceil(sample_size * label_num / batch_size))
learn = learner.Learner(estimator, version_path, csv_log_path)
result = learn.learn(train_generator,
valid_generator,
valid_steps,
steps_per_epoch=steps_per_epoch)
return result
def main(args):
# Load pre-processed data
topic_transfer_values_train, feature_transfer_values_train, captions_train = load_data(
'train', args.data)
topic_transfer_values_val, feature_transfer_values_val, captions_val = load_data(
'val', args.data)
print("topic shape:", topic_transfer_values_train.shape)
print("feature shape:", feature_transfer_values_train.shape)
# process captions
mark_start = 'startseq'
mark_end = 'endseq'
captions_train_marked = mark_captions(captions_train, mark_start,
mark_end) # training
captions_val_marked = mark_captions(captions_val, mark_start,
mark_end) # validation
tokenizer, vocab_size = create_tokenizer(captions_train_marked)
# save the word_idx and idx_word dictionaries in a file
# this will be required during evaluation
word_idx_path = os.path.join(args.data, 'word_idx.pkl')
idx_word_path = os.path.join(args.data, 'idx_word.pkl')
with open(word_idx_path, mode='wb') as f:
pickle.dump(tokenizer.word_index, f)
with open(idx_word_path, mode='wb') as f:
pickle.dump(tokenizer.index_word, f)
num_classes = topic_transfer_values_train.shape[1]
# training-dataset generator
generator_train = batch_generator(topic_transfer_values_train,
feature_transfer_values_train,
captions_train_marked, tokenizer,
len(captions_train), args.batch_size,
args.max_tokens, vocab_size)
# validation-dataset generator
generator_val = batch_generator(topic_transfer_values_val,
feature_transfer_values_val,
captions_val_marked, tokenizer,
len(captions_val), args.batch_size,
args.max_tokens, vocab_size)
# Create Model
model = create_model(args.image_weights, args.state_size, args.dropout,
tokenizer.word_index, args.glove, mark_start,
mark_end, vocab_size, args.max_tokens)
# train the model
model = train(model, generator_train, generator_val, captions_train_marked,
captions_val_marked, args)
# save the model with the best weights
save_model(model, args.weights)
def calc_mean(image_dir,
label_dir,
weights=None,
fraction_cracks=None,
loops=100):
''' calc mean and stddev for the dataset.'''
bg = batch_generator(
image_dir,
label_dir,
batch_size=1,
validation_batch_size=1,
output_size=(1920, 1920), # (H, W)
scale_size=None, # (H, W)
include_only_crops_with_mask=False,
augment=True)
stats = RunningStats()
for loop in range(loops):
for index in range(bg.image_count):
img, mask8 = bg.get_image_and_mask(index,
source='all',
augment=False)
mean = np.mean(img)
stats.push(mean)
print(stats.mean(), stats.standard_deviation())
def visualy_inspect_result(image_dir, label_dir):
bg = batch_generator(image_dir,
label_dir,
batch_size=1,
validation_batch_size=1,
output_size=OUTPUT_SIZE)
model = models.get_model(BATCH_SIZE, width=bg.width, height=bg.height)
model.load_weights('model.h5')
img, mask = bg.get_random_validation()
y_pred = model.predict(img[None, ...].astype(np.float32))[0]
print('y_pred.shape', y_pred.shape)
y_pred = y_pred.reshape((bg.height, bg.width, NUMBER_OF_CLASSES))
print('np.min(y_pred)', np.min(y_pred))
print('np.max(y_pred)', np.max(y_pred))
cv2.imshow('img', img)
cv2.imshow('mask 1', mask)
cv2.imshow('mask object 1', y_pred[:, :, 0])
cv2.waitKey(0)
def train_model_batch_generator(image_dir=None,
label_dir=None,
job_dir='./tmp/semantic_segmenter',
model_out_name="model.h5",
tissue=True,
**args):
# set the logging path for ML Engine logging to Storage bucket
logs_path = job_dir + '/logs/' + datetime.now().isoformat()
print('Using logs_path located at {}'.format(logs_path))
if image_dir is None:
if tissue:
# tissue
image_dir = r'..\data\optical\062A\640x480\images\images\*.png'
label_dir = r'..\data\optical\062A\640x480\masks_tissue\masks\*.png'
else:
# apperture
image_dir = r'..\data\optical\062A\640x480\images\images\*.png'
label_dir = r'..\data\optical\062A\640x480\masks_aperture\masks\*.png'
if tissue:
model_out_name = "model_tissue.h5"
else:
model_out_name = "model_aperture.h5"
bg = batch_generator(image_dir, label_dir, training_split=0.8)
model = get_model()
checkpoint_name = 'model_weights_' + loss_name + '.h5'
callbacks = [
EarlyStopping(monitor='val_loss', patience=patience, verbose=0),
#ModelCheckpoint(checkpoint_name, monitor='val_loss', save_best_only=True, verbose=0),
]
history = model.fit_generator(bg.training_batch(batch_size),
validation_data=bg.validation_batch(10),
epochs=epochs,
steps_per_epoch=bg.steps_per_epoch,
validation_steps=10,
verbose=1,
shuffle=False,
callbacks=callbacks)
#tensorboard = TensorBoard(log_dir="logs/{}".format(time()))
# history = model.fit_generator(
# train_generator,
# validation_data=validate_generator,
# epochs=epochs,
# steps_per_epoch=48 / batch_size,
# validation_steps=8 / batch_size,
# verbose=1,
# shuffle=False,
# callbacks=callbacks)
#callbacks=[tensorboard])
# Save the model locally
model.save(model_out_name)
def predict(test_paths, silence_paths, estimator):
batch_size = 64
test_generator = generator.batch_generator(test_paths,
batch_size,
len(config.POSSIBLE_LABELS),
silence_paths,
mode='test')
steps = int(np.ceil(len(test_paths) / batch_size))
predict_probs = estimator.model.predict_generator(test_generator, steps)
return predict_probs
def train_model(model, X, Y):
split = int(validation_split * len(X))
X_train, Y_train = X[split:], Y[split:]
X_valid, Y_valid = X[:split], Y[:split]
checkpoint = ModelCheckpoint('model-{epoch:03d}.h5',
monitor='val_loss',
verbose=0,
save_best_only=True,
mode='auto')
model.compile(loss='mean_squared_error', optimizer='adam')
model.fit_generator(batch_generator(X_train, Y_train, batch_size, True),
steps,
epochs,
verbose=1,
validation_data=batch_generator(
X_valid, Y_valid, batch_size, False),
validation_steps=split,
callbacks=[checkpoint])
def visualy_inspect_result():
model = get_model()
model.load_weights('model.h5')
bg = batch_generator()
img, mask = bg.get_random_validation()
y_pred = model.predict(img[None, ...].astype(np.float32))[0]
print('y_pred.shape', y_pred.shape)
y_pred = y_pred.reshape((IMAGE_H, IMAGE_W, NUMBER_OF_CLASSES))
print('np.min(y_pred)', np.min(y_pred))
print('np.max(y_pred)', np.max(y_pred))
cv2.imshow('img', img)
cv2.imshow('mask 1', mask)
cv2.imshow('mask object 1', y_pred[:, :, 0])
cv2.waitKey(0)
def train_model_batch_generator(image_dir=None,
label_dir=None,
model_out_name="model.h5",
**args):
bg = batch_generator(
image_dir,
label_dir,
batch_size=BATCH_SIZE,
validation_batch_size=VALIDATION_BATCH_SIZE,
training_split=0.8,
output_size=OUTPUT_SIZE, # (H, W)
scale_size=SCALE_SIZE, # (H, W)
include_only_crops_with_mask=True,
augment=True)
model = models.get_model(BATCH_SIZE,
width=OUTPUT_SIZE[1],
height=OUTPUT_SIZE[0])
# frozen_layer = model.get_layer('block1_conv0')
checkpoint_name = NAME + '.h5'
# class LearningRateTracker(keras.callbacks.Callback):
# def on_epoch_end(self, epoch, logs={}):
# optimizer = self.model.optimizer
# lr = K.eval(optimizer.lr * (1. / (1. + optimizer.decay * optimizer.iterations)))
# print('\nLR: {:.6f}\n'.format(lr))
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=20,
min_lr=0.0001)
def unfreeze(model, epoch):
if epoch == 20:
model.layers[2].trainable = True
optimizer = Adam(lr=1e-4)
model.compile(optimizer=optimizer,
loss='binary_crossentropy',
metrics=['accuracy'])
unfreeze_layers = keras.callbacks.LambdaCallback(
on_epoch_end=lambda epoch, logs: unfreeze(model, epoch))
class LRTensorBoard(TensorBoard):
''' add the learning rate to tensorboard '''
def __init__(self,
log_dir,
histogram_freq=1,
write_grads=True,
write_images=True
): # add other arguments to __init__ if you need
super().__init__(log_dir=log_dir,
histogram_freq=1,
write_grads=True,
write_images=True)
def on_epoch_end(self, epoch, logs=None):
logs.update({'lr': K.eval(self.model.optimizer.lr)})
super().on_epoch_end(epoch, logs)
tensorboard = TensorBoard(log_dir="logs\\{}".format(NAME),
histogram_freq=1,
write_grads=True,
write_images=False)
callbacks = [
tensorboard,
reduce_lr,
#unfreeze_layers,
# LearningRateTracker(),
EarlyStopping(monitor='val_loss', patience=PATIENCE, verbose=0),
ModelCheckpoint(checkpoint_name,
monitor='val_loss',
save_best_only=True,
verbose=0),
]
X_test, Y_test = bg.validation_batch()
history = model.fit_generator(
bg.training_batch(),
validation_data=(X_test, Y_test),
epochs=EPOCHS,
steps_per_epoch=bg.steps_per_epoch,
validation_steps=bg.validation_steps,
verbose=1,
shuffle=False,
# class_weight=CLASS_WEIGHT,
callbacks=callbacks)
# Save the model locally
model.save(model_out_name)
def make_predition_movie(image_dir,
label_dir,
weights=None,
fraction_cracks=None):
''' make a movie of predictions.
if label_dir != None, draw ground truth as blue,
if min_crack_fraction != None, only add frames to movie if cracks > fraction.
'''
# bg = batch_generator(
# image_dir,
# label_dir,
# batch_size=1,
# validation_batch_size=1,
# output_size=OUTPUT_SIZE, # (H, W)
# scale_size=SCALE_SIZE, # (H, W)
# include_only_crops_with_mask = True,
# augment=True)
bg = batch_generator(
image_dir,
label_dir,
batch_size=1,
validation_batch_size=1,
output_size=(1920, 1920), # (H, W)
scale_size=None, # (H, W)
include_only_crops_with_mask=False,
augment=False)
model = models.get_model(1, width=bg.width, height=bg.height)
if weights:
model.load_weights(weights)
else:
model.load_weights(NAME + ".h5")
vid_name = NAME + ".mp4"
video_scale = 4
vid_scaled = (int(bg.width / video_scale), int(bg.height / video_scale))
vid_fourcc = cv2.VideoWriter_fourcc(*'mp4v')
vid_out = cv2.VideoWriter(vid_name, vid_fourcc, 2.0, vid_scaled)
font = cv2.FONT_HERSHEY_SIMPLEX
for loops in range(1):
for index in range(bg.image_count):
img, mask8 = bg.get_image_and_mask(index,
source='all',
augment=False)
img = img.reshape((bg.height, bg.width, NUMBER_OF_CLASSES))
y_pred = model.predict(img[None, ...].astype(np.float32))[0]
y_pred = y_pred.reshape(
(bg.height, bg.width)) # , NUMBER_OF_CLASSES))
#print (y_pred.min(), y_pred.max(), y_pred.sum(), y_pred.shape)
if fraction_cracks is not None:
fraction = np.count_nonzero(y_pred > 0.5)
fraction /= (bg.height * bg.width)
if fraction < fraction_cracks:
continue
y_pred8 = y_pred * 255
y_pred8 = y_pred8.astype(np.uint8)
#print (y_pred8.min(), y_pred8.max(), y_pred8.sum(), y_pred8.shape)
img = img[:, :, 0].astype(np.uint8)
img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
# mask = np.zeros_like(img)
# mask[:,:,0] = y_pred8
# alpha = 0.25
# img = cv2.addWeighted(mask, alpha, img, 1 - alpha, 0, img)
# fill with the ground truth
if mask8 is not None:
mask = np.zeros_like(img)
mask[:, :, 2] = mask8
alpha = 0.25
cv2.addWeighted(mask, alpha, img, 1 - alpha, 0, img)
# notmask8 = cv2.bitwise_not(mask8)
# img[:,:,0] = cv2.bitwise_or(img[:,:,0], mask8)
# img[:,:,1] = cv2.bitwise_and(img[:,:,1], notmask8)
# img[:,:,2] = cv2.bitwise_and(img[:,:,2], notmask8)
# draw the prediction contour
ret, thresh = cv2.threshold(y_pred8, 127, 255, 0)
im2, contours, hierarchy = cv2.findContours(
thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
cv2.drawContours(img, contours, -1, (0, 255, 0), 8)
font_scale = 1.6
fname = os.path.basename(bg.images[index])
cv2.putText(img, fname, (20, 44), font, font_scale, (0, 0, 0), 10,
cv2.LINE_AA)
cv2.putText(img, fname, (20, 40), font, font_scale,
(255, 255, 255), 6, cv2.LINE_AA)
if video_scale != 1:
img = cv2.resize(img, vid_scaled)
vid_out.write(img)
if index % 10 == 0:
print(index)
# cv2.imshow('img',img)
# key = cv2.waitKey(0)
# if key == 27: # esc
# break
#cv2.destroyAllWindows()
vid_out.release()