def main():
model_deploy_config_path = os.path.join(opt.data_path,
opt.model_deploy_config_filename)
with open(model_deploy_config_path, 'r') as f:
model_deploy_config = json.load(f)
deployable_checkpoint_dict = json.loads(opt.deployable_checkpoint_info)
model_deploy_config['checkpoint_path'] = deployable_checkpoint_dict[
'checkpoint_path']
model_deploy_config['metrics'] = deployable_checkpoint_dict['metrics']
model_deploy_config['update_time'] = datetime.datetime.now().strftime(
"%Y%m%d-%H%M%S")
with open(model_deploy_config_path, 'w') as f:
f.write(json.dumps(model_deploy_config))
model_serve_path = os.path.join(opt.data_path, opt.seldon_model_path,
'nwd')
if not os.path.exists(model_serve_path):
os.makedirs(model_serve_path)
saved_model_paths = glob.glob(model_serve_path + '/*')
if len(saved_model_paths) > 0:
latest_model_version = max([
int(os.path.relpath(g, model_serve_path))
for g in glob.glob(model_serve_path + '/*')
])
else:
latest_model_version = 0
model_save_path = os.path.join(model_serve_path,
str(latest_model_version + 1))
print(f'msp: {model_save_path}')
load_weight_path = os.path.join(
opt.data_path, deployable_checkpoint_dict['checkpoint_path'] + '.h5')
print(f'lwp: {load_weight_path}')
model = UNet().create_model(img_shape=[256, 256, 3], num_class=2, rate=.0)
model.load_weights(load_weight_path)
model.save(model_save_path)
metrics=[
'accuracy', categorical_accuracy_without_ambiguous,
categorical_accuracy_only_valid_classes
])
model.summary()
current_dir = os.getcwd()
save_path = os.path.join(current_dir, 'output/unet')
if not os.path.exists(save_path):
os.mkdir(save_path)
checkpoint_path = os.path.join(save_path, 'checkpoint_weights.hdf5')
if RESUME:
print('checkPoint file:{}'.format(checkpoint_path))
model.load_weights(checkpoint_path, by_name=False)
# model_path = os.path.join(save_path, "model.json")
# # save model structure
# with open(model_path, 'w') as f:
# model_json = model.to_json()
# f.write(model_json)
def lr_scheduler(epoch):
lr = lr_base * ((1 - float(epoch) / epochs)**0.9)
print('lr: %f' % lr)
return lr
scheduler = LearningRateScheduler(lr_scheduler)
tsb = TensorBoard(log_dir='./logs')
early_stopper = EarlyStopping(monitor='loss', min_delta=0.001, patience=30)
def train_model(name, experiment, image_size, training_data_list, training_mask_list,
model_spec=[16, 32, 64, 128, 256], preprocess_list=None,
preprocess_stretch=False, preprocess_mask=None,
preprocess_fisher=False, keep_image=True,
load_model=False, epochs=15):
# make copies of the input array before shuffling
training_data_list = list(training_data_list)
training_mask_list = list(training_mask_list)
random.Random(experiment*42).shuffle(training_data_list)
random.Random(experiment*42).shuffle(training_mask_list)
# we're augmenting data -- expand the list of training data
train_input_img_paths = training_data_list[:-(test_samples
+ val_samples)] * random_factor
train_target_img_paths = training_mask_list[:-(test_samples
+ val_samples)] * random_factor
val_input_img_paths = training_data_list[-(
test_samples + val_samples):-val_samples]
val_target_img_paths = training_mask_list[-(
test_samples + val_samples):-val_samples]
test_input_img_paths = training_data_list[-test_samples:]
test_target_img_paths = training_mask_list[-test_samples:]
pp = None
# Chain of preprocessing functions, first one added is performed first
if preprocess_list is not None:
# Instantiate data Sequences for each split
if not preprocess_stretch:
pp = ImagePreprocessGradient(preprocess_list, keep_image, pp)
else:
pp = ImagePreprocessStretchedGradient(preprocess_list, pp)
if preprocess_mask is not None:
# Apply mask after gradients - masking first only gets overwritten
pp = ImagePreprocessMask(preprocess_mask, pp)
if preprocess_fisher is True:
pp = ImagePreprocessFisherize(pp)
if pp is not None:
# Instantiate pre-processed data sequences for each split
train_gen = RoadSeq(batch_size, image_size,
train_input_img_paths, train_target_img_paths, augment_data=True,
preprocess_fn=pp.preprocess())
val_gen = RoadSeq(batch_size, image_size,
val_input_img_paths, val_target_img_paths, augment_data=False,
preprocess_fn=pp.preprocess())
test_gen = RoadSeq(len(test_input_img_paths), image_size,
test_input_img_paths, test_target_img_paths, augment_data=False,
preprocess_fn=pp.preprocess())
else:
# use the images as they are
train_gen = RoadSeq(batch_size, image_size,
train_input_img_paths, train_target_img_paths, augment_data=True)
val_gen = RoadSeq(batch_size, image_size,
val_input_img_paths, val_target_img_paths, augment_data=False)
test_gen = RoadSeq(len(test_input_img_paths), image_size,
test_input_img_paths, test_target_img_paths, augment_data=False)
model_name = name+'.'+str(experiment)+'.h5'
model = UNet(image_size, model_spec)
model.compile(optimizer="adam",
loss="binary_crossentropy", metrics=["acc"])
if load_model:
model.load_weights(model_name)
model.summary()
callbacks = [
keras.callbacks.ModelCheckpoint(
model_name, save_best_only=True)
]
model.fit(train_gen, epochs=epochs, verbose=1,
validation_data=val_gen, callbacks=callbacks)
x, y = test_gen.__getitem__(0)
start = timer()
results = model.predict(x)
end = timer()
prediction_time = (end - start) / len(results)
results = np.array(results > 0.5).astype(np.uint8)
return calculate_error(results, test_target_img_paths) + (prediction_time,)
def evaluate_model(target_checkpoints_path,
eval_required_checkpoint_paths: list):
@tf.function
def _test_step(inputs, labels, weights):
predictions = model(inputs, training=False)
pred_loss = _loss_fn(labels=labels,
label_weights=weights,
predictions=predictions)
predictions = tf.nn.softmax(predictions, axis=-1)
predictions = predictions[..., 1]
predictions = tf.where(predictions > opt.prediction_threshold, 1, 0)
test_loss.update_state(pred_loss)
test_mean_iou.update_state(labels, predictions)
def _loss_fn(labels, label_weights, predictions):
cross_entropy_loss_pixel = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=labels, logits=predictions)
cross_entropy_loss_pixel = tf.multiply(cross_entropy_loss_pixel,
label_weights)
cross_entropy_loss = tf.reduce_sum(cross_entropy_loss_pixel) / (
tf.reduce_sum(label_weights) + 0.00001)
# if opt.weight_decay > 0:
# cross_entropy_loss = cross_entropy_loss + opt.weight_decay * tf.add_n(
# [tf.nn.l2_loss(v) for v in tf.compat.v1.trainable_variables()
# if 'batch_normalization' not in v.name])
return cross_entropy_loss
test_dataset, test_batch_per_epoch_num = load_dataset(data_type='test')
model = UNet().create_model(img_shape=[opt.image_size, opt.image_size, 3],
num_class=opt.num_class)
test_loss = tf.keras.metrics.Mean('train_loss', dtype=tf.float32)
test_mean_iou = tf.keras.metrics.MeanIoU(num_classes=opt.num_class)
eval_checkpoint_dict = defaultdict(dict)
checkpoints_num = len(eval_required_checkpoint_paths)
for idx_checkpoint, curr_checkpoint in enumerate(
eval_required_checkpoint_paths):
curr_checkpoint_name = curr_checkpoint.split(os.path.sep)[-1]
curr_checkpoint_name = os.path.splitext(curr_checkpoint_name)[0]
model.load_weights(curr_checkpoint)
for idx_step, (images, labels, weights) in enumerate(
test_dataset.take(test_batch_per_epoch_num)):
_test_step(inputs=images, labels=labels, weights=weights)
print(
f'evaluate ({idx_checkpoint+1}/{checkpoints_num}) '
f'loss: {format(test_loss.result(), ".7f")} '
f'mean-iou: {format(test_mean_iou.result(), ".7f")}',
flush=True)
eval_checkpoint_dict[curr_checkpoint_name]['loss'] = float(
test_loss.result().numpy())
eval_checkpoint_dict[curr_checkpoint_name]['mean_iou'] = float(
test_mean_iou.result().numpy())
with open(
os.path.join(target_checkpoints_path,
opt.checkpoint_metadata_filename), 'w') as f:
f.write(json.dumps(eval_checkpoint_dict))
test_loss.reset_states()
test_mean_iou.reset_states()
def run_demo(name,
experiment,
image_size,
training_data_list,
training_mask_list,
model_spec=[16, 32, 64, 128, 256],
preprocess_list=None,
preprocess_stretch=False,
preprocess_mask=None,
keep_image=True):
# make copies of the input array before shuffling
training_data_list = list(training_data_list)
random.Random(experiment * 42).shuffle(training_data_list)
test_input_img_paths = training_data_list[-test_samples:]
if training_mask_list is not None:
training_mask_list = list(training_mask_list)
random.Random(experiment * 42).shuffle(training_mask_list)
test_target_img_paths = training_mask_list[-test_samples:]
else:
test_target_img_paths = None
pp = None
# Chain of preprocessing functions, first one added is performed first
if preprocess_list is not None:
# Instantiate data Sequences for each split
if not preprocess_stretch:
pp = ImagePreprocessGradient(preprocess_list, keep_image, pp)
else:
pp = ImagePreprocessStretchedGradient(preprocess_list, pp)
if preprocess_mask is not None:
# Apply mask after gradients - masking first only gets overwritten
pp = ImagePreprocessMask(preprocess_mask, pp)
if pp is not None:
# Instantiate pre-processed data sequences for each split
test_gen = RoadSeq(len(test_input_img_paths),
image_size,
test_input_img_paths,
test_target_img_paths,
augment_data=False,
preprocess_fn=pp.preprocess())
else:
# use the images as they are
test_gen = RoadSeq(len(test_input_img_paths),
image_size,
test_input_img_paths,
test_target_img_paths,
augment_data=False)
model_name = name + '.' + str(experiment) + '.h5'
model = UNet(image_size, model_spec)
model.compile(optimizer="adam",
loss="binary_crossentropy",
metrics=["acc"])
model.load_weights(model_name)
x, y = test_gen.__getitem__(0)
results = model.predict(x)
results = np.array(results > 0.5).astype(np.uint8)
display_results(test_input_img_paths, test_target_img_paths, [results])
return results
