def main(): model_deploy_config_path = os.path.join(opt.data_path, opt.model_deploy_config_filename) with open(model_deploy_config_path, 'r') as f: model_deploy_config = json.load(f) deployable_checkpoint_dict = json.loads(opt.deployable_checkpoint_info) model_deploy_config['checkpoint_path'] = deployable_checkpoint_dict[ 'checkpoint_path'] model_deploy_config['metrics'] = deployable_checkpoint_dict['metrics'] model_deploy_config['update_time'] = datetime.datetime.now().strftime( "%Y%m%d-%H%M%S") with open(model_deploy_config_path, 'w') as f: f.write(json.dumps(model_deploy_config)) model_serve_path = os.path.join(opt.data_path, opt.seldon_model_path, 'nwd') if not os.path.exists(model_serve_path): os.makedirs(model_serve_path) saved_model_paths = glob.glob(model_serve_path + '/*') if len(saved_model_paths) > 0: latest_model_version = max([ int(os.path.relpath(g, model_serve_path)) for g in glob.glob(model_serve_path + '/*') ]) else: latest_model_version = 0 model_save_path = os.path.join(model_serve_path, str(latest_model_version + 1)) print(f'msp: {model_save_path}') load_weight_path = os.path.join( opt.data_path, deployable_checkpoint_dict['checkpoint_path'] + '.h5') print(f'lwp: {load_weight_path}') model = UNet().create_model(img_shape=[256, 256, 3], num_class=2, rate=.0) model.load_weights(load_weight_path) model.save(model_save_path)
metrics=[ 'accuracy', categorical_accuracy_without_ambiguous, categorical_accuracy_only_valid_classes ]) model.summary() current_dir = os.getcwd() save_path = os.path.join(current_dir, 'output/unet') if not os.path.exists(save_path): os.mkdir(save_path) checkpoint_path = os.path.join(save_path, 'checkpoint_weights.hdf5') if RESUME: print('checkPoint file:{}'.format(checkpoint_path)) model.load_weights(checkpoint_path, by_name=False) # model_path = os.path.join(save_path, "model.json") # # save model structure # with open(model_path, 'w') as f: # model_json = model.to_json() # f.write(model_json) def lr_scheduler(epoch): lr = lr_base * ((1 - float(epoch) / epochs)**0.9) print('lr: %f' % lr) return lr scheduler = LearningRateScheduler(lr_scheduler) tsb = TensorBoard(log_dir='./logs') early_stopper = EarlyStopping(monitor='loss', min_delta=0.001, patience=30)
def train_model(name, experiment, image_size, training_data_list, training_mask_list, model_spec=[16, 32, 64, 128, 256], preprocess_list=None, preprocess_stretch=False, preprocess_mask=None, preprocess_fisher=False, keep_image=True, load_model=False, epochs=15): # make copies of the input array before shuffling training_data_list = list(training_data_list) training_mask_list = list(training_mask_list) random.Random(experiment*42).shuffle(training_data_list) random.Random(experiment*42).shuffle(training_mask_list) # we're augmenting data -- expand the list of training data train_input_img_paths = training_data_list[:-(test_samples + val_samples)] * random_factor train_target_img_paths = training_mask_list[:-(test_samples + val_samples)] * random_factor val_input_img_paths = training_data_list[-( test_samples + val_samples):-val_samples] val_target_img_paths = training_mask_list[-( test_samples + val_samples):-val_samples] test_input_img_paths = training_data_list[-test_samples:] test_target_img_paths = training_mask_list[-test_samples:] pp = None # Chain of preprocessing functions, first one added is performed first if preprocess_list is not None: # Instantiate data Sequences for each split if not preprocess_stretch: pp = ImagePreprocessGradient(preprocess_list, keep_image, pp) else: pp = ImagePreprocessStretchedGradient(preprocess_list, pp) if preprocess_mask is not None: # Apply mask after gradients - masking first only gets overwritten pp = ImagePreprocessMask(preprocess_mask, pp) if preprocess_fisher is True: pp = ImagePreprocessFisherize(pp) if pp is not None: # Instantiate pre-processed data sequences for each split train_gen = RoadSeq(batch_size, image_size, train_input_img_paths, train_target_img_paths, augment_data=True, preprocess_fn=pp.preprocess()) val_gen = RoadSeq(batch_size, image_size, val_input_img_paths, val_target_img_paths, augment_data=False, preprocess_fn=pp.preprocess()) test_gen = RoadSeq(len(test_input_img_paths), image_size, test_input_img_paths, test_target_img_paths, augment_data=False, preprocess_fn=pp.preprocess()) else: # use the images as they are train_gen = RoadSeq(batch_size, image_size, train_input_img_paths, train_target_img_paths, augment_data=True) val_gen = RoadSeq(batch_size, image_size, val_input_img_paths, val_target_img_paths, augment_data=False) test_gen = RoadSeq(len(test_input_img_paths), image_size, test_input_img_paths, test_target_img_paths, augment_data=False) model_name = name+'.'+str(experiment)+'.h5' model = UNet(image_size, model_spec) model.compile(optimizer="adam", loss="binary_crossentropy", metrics=["acc"]) if load_model: model.load_weights(model_name) model.summary() callbacks = [ keras.callbacks.ModelCheckpoint( model_name, save_best_only=True) ] model.fit(train_gen, epochs=epochs, verbose=1, validation_data=val_gen, callbacks=callbacks) x, y = test_gen.__getitem__(0) start = timer() results = model.predict(x) end = timer() prediction_time = (end - start) / len(results) results = np.array(results > 0.5).astype(np.uint8) return calculate_error(results, test_target_img_paths) + (prediction_time,)
def evaluate_model(target_checkpoints_path, eval_required_checkpoint_paths: list): @tf.function def _test_step(inputs, labels, weights): predictions = model(inputs, training=False) pred_loss = _loss_fn(labels=labels, label_weights=weights, predictions=predictions) predictions = tf.nn.softmax(predictions, axis=-1) predictions = predictions[..., 1] predictions = tf.where(predictions > opt.prediction_threshold, 1, 0) test_loss.update_state(pred_loss) test_mean_iou.update_state(labels, predictions) def _loss_fn(labels, label_weights, predictions): cross_entropy_loss_pixel = tf.nn.sparse_softmax_cross_entropy_with_logits( labels=labels, logits=predictions) cross_entropy_loss_pixel = tf.multiply(cross_entropy_loss_pixel, label_weights) cross_entropy_loss = tf.reduce_sum(cross_entropy_loss_pixel) / ( tf.reduce_sum(label_weights) + 0.00001) # if opt.weight_decay > 0: # cross_entropy_loss = cross_entropy_loss + opt.weight_decay * tf.add_n( # [tf.nn.l2_loss(v) for v in tf.compat.v1.trainable_variables() # if 'batch_normalization' not in v.name]) return cross_entropy_loss test_dataset, test_batch_per_epoch_num = load_dataset(data_type='test') model = UNet().create_model(img_shape=[opt.image_size, opt.image_size, 3], num_class=opt.num_class) test_loss = tf.keras.metrics.Mean('train_loss', dtype=tf.float32) test_mean_iou = tf.keras.metrics.MeanIoU(num_classes=opt.num_class) eval_checkpoint_dict = defaultdict(dict) checkpoints_num = len(eval_required_checkpoint_paths) for idx_checkpoint, curr_checkpoint in enumerate( eval_required_checkpoint_paths): curr_checkpoint_name = curr_checkpoint.split(os.path.sep)[-1] curr_checkpoint_name = os.path.splitext(curr_checkpoint_name)[0] model.load_weights(curr_checkpoint) for idx_step, (images, labels, weights) in enumerate( test_dataset.take(test_batch_per_epoch_num)): _test_step(inputs=images, labels=labels, weights=weights) print( f'evaluate ({idx_checkpoint+1}/{checkpoints_num}) ' f'loss: {format(test_loss.result(), ".7f")} ' f'mean-iou: {format(test_mean_iou.result(), ".7f")}', flush=True) eval_checkpoint_dict[curr_checkpoint_name]['loss'] = float( test_loss.result().numpy()) eval_checkpoint_dict[curr_checkpoint_name]['mean_iou'] = float( test_mean_iou.result().numpy()) with open( os.path.join(target_checkpoints_path, opt.checkpoint_metadata_filename), 'w') as f: f.write(json.dumps(eval_checkpoint_dict)) test_loss.reset_states() test_mean_iou.reset_states()
def run_demo(name, experiment, image_size, training_data_list, training_mask_list, model_spec=[16, 32, 64, 128, 256], preprocess_list=None, preprocess_stretch=False, preprocess_mask=None, keep_image=True): # make copies of the input array before shuffling training_data_list = list(training_data_list) random.Random(experiment * 42).shuffle(training_data_list) test_input_img_paths = training_data_list[-test_samples:] if training_mask_list is not None: training_mask_list = list(training_mask_list) random.Random(experiment * 42).shuffle(training_mask_list) test_target_img_paths = training_mask_list[-test_samples:] else: test_target_img_paths = None pp = None # Chain of preprocessing functions, first one added is performed first if preprocess_list is not None: # Instantiate data Sequences for each split if not preprocess_stretch: pp = ImagePreprocessGradient(preprocess_list, keep_image, pp) else: pp = ImagePreprocessStretchedGradient(preprocess_list, pp) if preprocess_mask is not None: # Apply mask after gradients - masking first only gets overwritten pp = ImagePreprocessMask(preprocess_mask, pp) if pp is not None: # Instantiate pre-processed data sequences for each split test_gen = RoadSeq(len(test_input_img_paths), image_size, test_input_img_paths, test_target_img_paths, augment_data=False, preprocess_fn=pp.preprocess()) else: # use the images as they are test_gen = RoadSeq(len(test_input_img_paths), image_size, test_input_img_paths, test_target_img_paths, augment_data=False) model_name = name + '.' + str(experiment) + '.h5' model = UNet(image_size, model_spec) model.compile(optimizer="adam", loss="binary_crossentropy", metrics=["acc"]) model.load_weights(model_name) x, y = test_gen.__getitem__(0) results = model.predict(x) results = np.array(results > 0.5).astype(np.uint8) display_results(test_input_img_paths, test_target_img_paths, [results]) return results