def main(dataset, net_config, _run): # Add all of the config into the helper class for key in net_config: setattr(a, key, net_config[key]) setattr(a, 'EXP_OUT', EXP_OUT) setattr(a, 'RUN_id', _run._id) output_dir = create_directories(_run._id, ex) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9) with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: # load the dataset class data = get_dataset(dataset['name']) data = data(**dataset) model = AutoEncoder(sess, image_size=a.input_image_size, batch_size=a.batch_size, output_size=a.input_image_size, dataset_name=dataset['name'], checkpoint_dir=output_dir, data=data, momentum=a.batch_momentum, aef_dim=a.naef, noise_std_dev=a.noise_std_dev) if a.mode == 'train': tmp = model.train(a) _run.info['predictions'] = tmp _run.info['mean_predictions'] = np.mean(tmp, axis=0) elif a.mode == 'valid': tmp = model.validate(a) _run.info['predictions'] = tmp _run.info['mean_predictions'] = np.mean(tmp, axis=0) else: model.test(a)
def evaluate_temperature_scaling(experiments, temperatures): for exp_id in experiments: exp = ExperimentData(exp_id) dataset = exp.get_record()['info']['dataset'] data = get_dataset(dataset['name']) data_description = list(data.get_data_description()) if exp.get_record()['config']['method'] == 'new_class': data_description[2] += 1 model = get_model(exp.get_record()['config']['modelname']) # get the label_flip specs label_flip = None if 'label_flip' in dataset['augmentation']: label_flip = dataset['augmentation']['label_flip'] def evaluation(parameters): with model(data_description=data_description, **parameters) as net: import_weights_into_network(net, exp_id) return measure_metrics( net, data(**dataset).get_testset(), exp.get_record()['config']['uncertainty_metrics'], label_flip=label_flip) result = grid_search(evaluation, {'temperature_scaling': temperatures}, exp.get_record()['config']['net_config']) info = exp.get_record()['info'] info['temperature_grid_search'] = force_bson_encodeable(result) exp.update_record({'info': info})
def main(dataset, net_config, img_h, img_w, _run): # Add all of the config into the helper class for key in net_config: setattr(a, key, net_config[key]) setattr(a, 'EXP_OUT', EXP_OUT) setattr(a, 'RUN_id', _run._id) output_dir = create_directories(_run._id, ex) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.99) with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: # load the dataset class data = get_dataset(dataset['name']) data = data(img_h=img_h, img_w=img_w, **dataset) cGAN_model = get_model('cascGAN') if a.checkpoint is not None: ckp = os.path.join(a.EXP_OUT, str(a.checkpoint)) else: ckp = None trainFlag = (a.mode == 'train') model = cGAN_model(sess, dataset_name=dataset['name'], image_size=img_h, checkpoint_dir=output_dir, data=data, data_desc=data.get_data_description(), is_training=trainFlag, checkpoint=ckp, vgg_checkpoint=net_config['vgg_checkpoint']) if a.mode == 'train': model.train(a) else: model.transformDatasets(a, data)
def train_ambiguous(modelname, net_config, dataset, starting_weights, method, num_iterations, uncertainty_metrics, _run): # Set up the directories for diagnostics output_dir = create_directories(_run._id, ex) data = get_dataset(dataset['name']) data_description = list(data.get_data_description()) num_classes = data_description[2] args = False if isinstance(method, list): args = method[1:] method = method[0] # augmentate the class labels if method == 'flip_classes': # randomly map two classes onto each other to make them ambiguos classes = np.random.choice(list(range(num_classes)), size=2, replace=False) dataset.setdefault('augmentation', {})['label_flip'] = (classes[0], classes[1], np.random.rand()) elif method == 'new_class': # randomly label a given class as a new, nonexisting class data_description[2] = num_classes + 1 if args: old_class = args[0] print(args) else: old_class = np.random.choice(list(range(num_classes))) dataset.setdefault('augmentation', {})['label_flip'] = (old_class, num_classes, np.random.rand()) elif method == 'merge': # randomly merge two classes together classes = np.random.choice(list(range(num_classes)), size=2, replace=False) dataset.setdefault('augmentation', {})['label_merge'] = (classes[0], classes[1]) _run.info.setdefault('dataset', {}).update(dataset) model = get_model(modelname) with model(data_description=data_description, output_dir=output_dir, **net_config) as net: data = data(**dataset) train_network(net, output_dir, data, num_iterations, starting_weights, ex) _run.info['measurements'] = measure_metrics( net, data.get_testset(), uncertainty_metrics, label_flip=dataset['augmentation'].get('label_flip', None)) net.close() print(threading.enumerate())
def load_data(data_config): """ Load the data specified in the data_config dict. """ dataset_params = { key: val for key, val in data_config.items() if key not in ['dataset', 'use_trainset'] } return get_dataset(data_config['dataset'], dataset_params)
def measure(modelname, net_config, dataset, starting_weights, uncertainty_metrics, _run): model = get_model(modelname) data = get_dataset(dataset['name']) data_description = list(data.get_data_description()) if 'num_classes' in dataset: data_description[2] = dataset['num_classes'] with model(data_description=data_description, **net_config) as net: data = data(**dataset) import_weights_into_network(net, starting_weights) _run.info['measurements'] = measure_metrics(net, data.get_testset(), uncertainty_metrics)
def main(dataset, net_config, _run): # Add all of the config into the helper class for key in net_config: setattr(a, key, net_config[key]) setattr(a, 'EXP_OUT', EXP_OUT) setattr(a, 'RUN_id', _run._id) output_dir = create_directories(_run._id, ex) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9) with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: data = get_dataset(dataset['name']) data = data(dataset['image_input_dir'], **dataset) data_id = dataset['image_input_dir'].split('/')[-1].split('_')[0] setattr(a, 'DATA_id', data_id) disc_model = get_model('simDisc') ckp = None if net_config['checkpoint'] is not None: ckp = os.path.join(a.EXP_OUT, str(net_config['checkpoint'])) if net_config['feature_extractor'] is not None: fe_ckp = os.path.join(a.EXP_OUT, str(net_config['feature_extractor'])) model = disc_model(sess=sess, image_size=a.input_image_size, batch_size=a.batch_size, df_dim=a.ndf, input_c_dim=3, checkpoint_dir=output_dir, data=data, momentum=a.batch_momentum, arch=net_config['arch'], checkpoint=ckp, feature_extractor=fe_ckp) if a.mode == "train": tmp = model.train(a) _run.info['predictions'] = tmp _run.info['mean_predictions'] = np.mean(tmp, axis=0) elif a.mode == "predict": input_list = glob.glob(os.path.join(a.predict_dir, "target_*.png")) synth_list = glob.glob(os.path.join(a.predict_dir, "synth_*.png")) segm_list = glob.glob(os.path.join(a.predict_dir, "input_*.png")) input_list.sort( key=lambda x: int(x.partition('_')[-1].partition('.')[0])) synth_list.sort( key=lambda x: int(x.partition('_')[-1].partition('.')[0])) segm_list.sort( key=lambda x: int(x.partition('_')[-1].partition('.')[0])) model.predict(a, input_list, synth_list, segm_list)
def uncertainty_benchmark(modelname, net_config, dataset, starting_weights, benchmark, uncertainty_metrics, _run): model = get_model(modelname) data = get_dataset(dataset['name']) with model(data_description=data.get_data_description(), **net_config) as net: data = data(**dataset) import_weights_into_network(net, starting_weights) for metric in uncertainty_metrics: measurements = evaluate_uncertainty(net, data.get_testset(), metric, benchmark=benchmark) _run.info.setdefault('measurements', {})[metric] = measurements
def main(modelname, dataset, net_config, _run): # Set up the directories for diagnostics output_dir = create_directories(_run._id, ex) # load the dataset class, but don't instantiate it data = get_dataset(dataset['name']) # create the network model = get_model(modelname) with model(data_description=data.get_data_description(), output_dir=output_dir, **net_config) as net: # now we can load the dataset inside the scope of the network graph data = data(**dataset) train_and_evaluate(net, output_dir, data)
def fit_and_evaluate(net_config, evaluation_data, starting_weights, _run): """Load weigths from training experiments and evalaute fusion against specified data.""" dataset = get_dataset(evaluation_data['dataset']) # evaluate individual experts model = get_model(net_config['expert_model']) confusion_matrices = {} for expert in net_config['num_channels']: model_config = deepcopy(net_config) model_config['modality'] = expert model_config['prefix'] = net_config['prefixes'][expert] with model(data_description=dataset.get_data_description(), **model_config) as net: data = dataset(**evaluation_data) import_weights_into_network( net, starting_weights[model_config['prefix']]) m, conf_mat = net.score(data.get_measureset()) confusion_matrices[expert] = conf_mat print('Evaluated network {} on {} measurement set:'.format( expert, evaluation_data['dataset'])) print("INFO now getting test results") m, _ = net.score(data.get_testset()) print('total accuracy {:.3f} IoU {:.3f}'.format( m['total_accuracy'], m['mean_IoU'])) _run.info.setdefault('measurements', {}).setdefault(expert, m) _run.info['confusion_matrices'] = confusion_matrices # now evaluate bayes mix with BayesFusion(data_description=dataset.get_data_description(), confusion_matrices=confusion_matrices, **net_config) as net: data = dataset(**evaluation_data) import_weights_into_network(net, starting_weights) measurements, confusion_matrix = net.score(data.get_testset()) _run.info['measurements']['fusion'] = measurements _run.info['confusion_matrix'] = confusion_matrix print('Evaluated Bayes Fusion on {} data:'.format( evaluation_data['dataset'])) print('total accuracy {:.3f} IoU {:.3f}'.format( measurements['total_accuracy'], measurements['mean_IoU'])) # There seems to be a problem with capturing the print output, flush to be sure stdout.flush()
def main(dataset, net_config, _run): # Add all of the config into the helper class for key in net_config: setattr(a, key, net_config[key]) setattr(a, 'EXP_OUT', EXP_OUT) setattr(a, 'RUN_id', _run._id) output_dir = create_directories(_run._id, ex) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9) with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: # data = Cityscapes_generated(base_path="/Users/David/masterThesis/pix2pix-tensorflow/dir/224_full") data = get_dataset(dataset['name']) # data = data(dataset['image_input_dir'], ppd=dataset['ppd']) data = data(dataset['image_input_dir'], **dataset) data_id = dataset['image_input_dir'].split('/')[-1].split('_')[0] setattr(a, 'DATA_id', data_id) model = DiffDiscrim(sess=sess, image_size=a.input_image_size, batch_size=a.batch_size, df_dim=a.ndf, input_c_dim=3, checkpoint_dir=output_dir, data=data, momentum=a.batch_momentum, checkpoint=os.path.join( a.EXP_OUT, str(net_config['checkpoint']))) if a.mode == "train": tmp = model.train(a) _run.info['predictions'] = tmp _run.info['mean_predictions'] = np.mean(tmp, axis=0) elif a.mode == "predict": input_list = glob.glob(os.path.join(a.predict_dir, "target_*.png")) synth_list = glob.glob(os.path.join(a.predict_dir, "synth_*.png")) segm_list = glob.glob(os.path.join(a.predict_dir, "input_*.png")) input_list.sort( key=lambda x: int(x.partition('_')[-1].partition('.')[0])) synth_list.sort( key=lambda x: int(x.partition('_')[-1].partition('.')[0])) segm_list.sort( key=lambda x: int(x.partition('_')[-1].partition('.')[0])) model.predict(a, input_list, synth_list, segm_list)
def main(modelname, net_config, dataset, starting_weights, input_folder, _run): # Set up the directories for diagnostics output_dir = create_directories(_run._id, ex) # load the data for the data description data_desc = get_dataset(dataset['name']) # load images in input_folder eval_image_paths = load_list_path(input_folder) # create the network model = get_model(modelname) with model(data_description=data_desc.get_data_description(), output_dir=output_dir, **net_config) as net: net.import_weights(filepath=starting_weights) print("INFO: Imported weights succesfully") predict_output(net, output_dir, eval_image_paths, data_desc)
def split_test_data(data_config): # Load the dataset, we expect config to include the arguments dataset_params = { key: val for key, val in data_config.items() if key not in ['dataset'] } dataset_params['augmentation'] = { key: False for key in [ 'crop', 'scale', 'vflip', 'hflip', 'gamma', 'rotate', 'shear', 'contrast', 'brightness' ] } data = get_dataset(data_config['dataset'])(**dataset_params) measure_set, test_set = train_test_split(data.testset, test_size=.5, random_state=1) return data, measure_set, test_set
def evaluate(net_config, evaluation_data, modelname, starting_weights, _run): """Load weigths from training experiments and evalaute fusion against specified data.""" data = get_dataset(evaluation_data['dataset']) model = get_model(modelname) # now evaluate average mix with model(data_description=data.get_data_description(), **net_config) as net: data = data(**evaluation_data) import_weights_into_network(net, starting_weights) measurements, confusion_matrix = net.score(data.get_set_data(test_set)) _run.info['measurements'] = measurements _run.info['confusion_matrix'] = confusion_matrix print('Evaluated on {} data:'.format(evaluation_data['dataset'])) print('total accuracy {:.3f} IoU {:.3f}'.format( measurements['total_accuracy'], measurements['mean_IoU'])) # There seems to be a problem with capturing the print output, flush to be sure stdout.flush()
def main(dataset, net_config, _run): # Add all of the config into the helper class for key in net_config: setattr(a, key, net_config[key]) setattr(a,'EXP_OUT',EXP_OUT) setattr(a,'RUN_id',_run._id) output_dir = create_directories(_run._id, ex) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9) with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: # load the dataset class data = get_dataset(dataset['name']) data = data(**dataset) cGAN_model = get_model('cGAN') if a.checkpoint is not None: ckp = os.path.join(a.EXP_OUT,str(a.checkpoint)) else: ckp = None model = cGAN_model(sess, image_size=a.input_image_size, batch_size=a.batch_size, output_size=a.input_image_size, dataset_name=dataset['name'], checkpoint_dir=output_dir, data=data, data_desc=data.get_data_description(), momentum=a.batch_momentum, feature_matching=net_config['feature_matching'], L1_lambda=float(a.l1_weight/a.gan_weight), gf_dim=a.ngf, df_dim=a.ndf, use_grayscale=net_config['use_grayscale'], noise_std_dev=a.noise_std_dev, checkpoint=ckp, gen_type=net_config['type']) if a.mode == 'train': tmp = model.train(a) _run.info['predictions'] = tmp _run.info['mean_predictions'] = np.mean(tmp, axis=0) elif a.mode == 'valid': tmp = model.validate(a) _run.info['predictions'] = tmp _run.info['mean_predictions'] = np.mean(tmp, axis=0) else: model.transformDatasets(a)
def main(dataset, net_config, _run): # Add all of the config into the helper class for key in net_config: setattr(a, key, net_config[key]) setattr(a, 'EXP_OUT', EXP_OUT) setattr(a, 'RUN_id', _run._id) output_dir = create_directories(_run._id, ex) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9) with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: # load the dataset class data = get_dataset(dataset['name']) data = data(**dataset) cycleGAN_model = get_model('cycleGAN') if a.checkpoint is not None: ckp = os.path.join(a.EXP_OUT, str(a.checkpoint)) else: ckp = None model = cycleGAN_model(sess, image_size=a.input_image_size, batch_size=a.batch_size, dataset_name=dataset['name'], checkpoint_dir=output_dir, data=data, data_desc=data.get_data_description(), checkpoint=ckp) if a.mode == 'train': tmp = model.train(a) # _run.info['predictions'] = tmp # _run.info['mean_predictions'] = np.mean(tmp, axis=0) elif a.mode == 'valid': tmp = model.validate(a) _run.info['predictions'] = tmp _run.info['mean_predictions'] = np.mean(tmp, axis=0) else: model.transformDatasets(a)
def uncertainty_parameter_search(modelname, net_config, dataset, starting_weights, search_parameters, benchmark, uncertainty_metrics, _run): model = get_model(modelname) data = get_dataset(dataset['name']) def evaluation(parameters): with model(data_description=data.get_data_description(), **parameters) as net: measure_set = data(**dataset).get_measureset() import_weights_into_network(net, starting_weights) return { metric: evaluate_uncertainty(net, measure_set, metric, benchmark=benchmark, print_results=False) for metric in uncertainty_metrics } _run.info['results'] = grid_search(evaluation, search_parameters, net_config)
def collect_data(net_config, dataset, starting_weights, save_to, _run): data = get_dataset(**dataset) model = get_model(net_config['expert_model']) predictions = {} for expert in net_config['prefixes']: model_config = deepcopy(net_config) model_config['modality'] = expert model_config['prefix'] = net_config['prefixes'][expert] with model(data_description=data.get_data_description(), **model_config) as net: import_weights_into_network(net, starting_weights[model_config['prefix']]) predictions['measure_%s' % expert] = net.predict(data.get_measureset()) predictions['test_%s' % expert] = net.predict(data.get_testset()) # add also gt labels predictions['measure_gt'] = data.get_measureset(tf_dataset=False)['labels'] predictions['test_gt'] = data.get_testset(tf_dataset=False)['labels'] # outpath = path.join(save_to, _run._id) outpath = save_to if not path.exists(outpath): mkdir(outpath) np.savez_compressed(path.join(outpath, 'predictions.npz'), **predictions)
def main(modelname, net_config, gan_config, disc_config, datasetSem, datasetGAN, datasetDisc, starting_weights, flag_measure, output_mat, flag_entropy, thresholds, start, _run): for key in gan_config: setattr(a, key, gan_config[key]) for key in disc_config: setattr(b, key, disc_config[key]) setattr(a, 'EXP_OUT', EXP_OUT) setattr(a, 'RUN_id', _run._id) setattr(b, 'EXP_OUT', EXP_OUT) setattr(b, 'RUN_id', _run._id) disc_data_path = os.path.join(datasetDisc['image_input_dir'], str(gan_config['checkpoint']) + "_full") data_id = str(gan_config['checkpoint']) setattr(b, 'DATA_id', data_id) # Set up the directories for diagnostics output_dir = create_directories(_run._id, ex) # load the data for the data description data_desc = get_dataset(datasetSem['name']) model = get_model(modelname) net = model(data_description=data_desc.get_data_description(), output_dir=output_dir, **net_config) # net.import_weights(filepath=starting_weights) print("INFO: SemSegNet Imported weights succesfully") GAN_graph = tf.Graph() with GAN_graph.as_default(): # create the network gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9) GAN_sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) if gan_config['type'] == 'cascRef': dataGAN = get_dataset('cityscapes_cascGAN') cGAN_model = get_model('cascGAN') if a.checkpoint is not None: ckp = os.path.join(a.EXP_OUT, str(a.checkpoint)) modelGAN = cGAN_model( GAN_sess, dataset_name='cityscapes_cascGAN', image_size=disc_config['input_image_size'], checkpoint_dir=output_dir, data_desc=dataGAN.get_data_description(), is_training=False, checkpoint=ckp, vgg_checkpoint= "/cluster/work/riner/users/haldavid/Checkpoints/VGG_Model/imagenet-vgg-verydeep-19.mat" ) else: # load the dataset class dataGAN = get_dataset(datasetGAN['name']) # data = data(**datasetGAN) cGAN_model = get_model('cGAN') modelGAN = cGAN_model( GAN_sess, checkpoint_dir=output_dir, data_desc=dataGAN.get_data_description(), feature_matching=gan_config['feature_matching'], checkpoint=os.path.join(a.EXP_OUT, str(a.checkpoint)), gen_type=gan_config['type'], use_grayscale=gan_config['use_grayscale']) print("INFO: Generative model imported weights succesfully") Disc_graph = tf.Graph() with Disc_graph.as_default(): gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9) sessD = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) dataD = get_dataset(datasetDisc['name']) dataD = dataD(disc_data_path, **datasetDisc) disc_model = get_model('simDisc') disc_checkpoint = None if disc_config['checkpoint'] is not None: disc_checkpoint = os.path.join(a.EXP_OUT, str(disc_config['checkpoint'])) modelDiff = disc_model(sess=sessD, checkpoint_dir=output_dir, pos_weight=disc_config['pos_weight'], data=dataD, arch=disc_config['arch'], use_grayscale=disc_config['use_grayscale'], checkpoint=disc_checkpoint, use_segm=disc_config['use_segm'], batch_size=disc_config['batch_size'], feature_extractor=os.path.join( a.EXP_OUT, str(a.checkpoint))) if disc_config['checkpoint'] is None: print("INFO: Begin training simDisc") tmp = modelDiff.train(b) _run.info['simDisc_predictions'] = tmp _run.info['simDisc_mean_predictions'] = np.mean(tmp, axis=0) _run.info['simDisc_stdDev'] = np.std(tmp, axis=0) print("INFO: Finished training simDisc") else: print("INFO: Init and loaded checpoint for simDisc") if flag_measure: benchmarks = ['measure'] else: benchmarks = ['wilddash', 'posneg', 'valid', 'measure'] data_SemSeg = data_desc(**datasetSem) _run.info['thresholds'] = thresholds ########################################################################### # mapping from Deeplab classes to Adapnet classes original_labelinfo = { 0: { 'name': 'road', 'mapping': 'road' }, 1: { 'name': 'sidewalk', 'mapping': 'sidewalk' }, 2: { 'name': 'building', 'mapping': 'building' }, 3: { 'name': 'wall', 'mapping': 'building' }, 4: { 'name': 'fence', 'mapping': 'fence' }, 5: { 'name': 'pole', 'mapping': 'pole' }, 6: { 'name': 'traffic light', 'mapping': 'void' }, 7: { 'name': 'traffic sign', 'mapping': 'traffic sign' }, 8: { 'name': 'vegetation', 'mapping': 'vegetation' }, 9: { 'name': 'terrain', 'mapping': 'vegetation' }, 10: { 'name': 'sky', 'mapping': 'sky' }, 11: { 'name': 'person', 'mapping': 'person' }, 12: { 'name': 'rider', 'mapping': 'person' }, 13: { 'name': 'car', 'mapping': 'vehicle' }, 14: { 'name': 'truck', 'mapping': 'vehicle' }, 15: { 'name': 'bus', 'mapping': 'vehicle' }, 16: { 'name': 'train', 'mapping': 'vehicle' }, 17: { 'name': 'motorcycle', 'mapping': 'vehicle' }, 18: { 'name': 'bicycle', 'mapping': 'bicycle' }, 255: { 'name': 'void', 'mapping': 'void' } } labelinfo = { 0: { 'name': 'void', 'color': [0, 0, 0] }, 1: { 'name': 'sky', 'color': [70, 130, 180] }, 2: { 'name': 'building', 'color': [70, 70, 70] }, 3: { 'name': 'road', 'color': [128, 64, 128] }, 4: { 'name': 'sidewalk', 'color': [244, 35, 232] }, 5: { 'name': 'fence', 'color': [190, 153, 153] }, 6: { 'name': 'vegetation', 'color': [107, 142, 35] }, 7: { 'name': 'pole', 'color': [153, 153, 153] }, 8: { 'name': 'vehicle', 'color': [0, 0, 142] }, 9: { 'name': 'traffic sign', 'color': [220, 220, 0] }, 10: { 'name': 'person', 'color': [220, 20, 60] }, 11: { 'name': 'bicycle', 'color': [119, 11, 32] } } label_lookup = [ next(i for i in labelinfo if labelinfo[i]['name'] == k['mapping']) for _, k in original_labelinfo.items() ] base_path = path.join(DATA_BASEPATH, 'fishyscapes_newfog') if 'TMPDIR' in environ: print('INFO loading dataset into machine ... ') # first load the zipfile into a closer memory location, then load all the # images zip = zipfile.ZipFile(path.join(base_path, 'testset.zip'), 'r') localtmp = environ['TMPDIR'] zip.extractall(localtmp) zip.close() base_path = localtmp print('DONE loading dataset into machine ... ') ########################################################################### set_size = 1000 h_orig = 1024 w_orig = 2048 sub_size = 100 semseg_path = "/cluster/work/riner/users/blumh/fishyscapes_deeplab_predictions_newfog" out_path = "/cluster/work/riner/users/blumh/resultsDH" for k in range(start, (start + 2)): kb = k * sub_size if k > 0: print('Done %d images' % (kb)) stdout.flush() img_array = np.zeros((sub_size, 256, 256, 3)) segm_array = np.zeros((sub_size, 256, 256, 3)) for i in range(sub_size): img = cv2.imread( path.join(base_path, 'testset', str(i + kb) + '_rgb.png')) dl_labels = np.expand_dims(cv2.imread( path.join(semseg_path, str(i + kb) + '_predict.png'))[:, :, 0], axis=0) cs_labels = np.asarray(label_lookup, dtype='int32')[dl_labels] lookup = np.array([ labelinfo[i]['color'] for i in range(max(labelinfo.keys()) + 1) ]).astype(int) segm = np.array(lookup[cs_labels[:]]).astype('uint8')[..., ::-1] #mask = cv2.imread(path.join(base_path, str(i)+'_mask.png'), cv2.IMREAD_ANYDEPTH) # blob['labels'] = cv2.imread(labels_filename, cv2.IMREAD_ANYDEPTH) # # apply label mapping # blob['labels'] = np.asarray(self.label_lookup, dtype='int32')[blob['labels']] img_array[i, ...] = cv2.resize(img, (256, 256), interpolation=cv2.INTER_LINEAR) segm_array[i, ...] = cv2.resize(segm[0, ...], (256, 256), interpolation=cv2.INTER_NEAREST) with GAN_sess.as_default(): with GAN_graph.as_default(): synth_images = modelGAN.transform(a, segm_array) with sessD.as_default(): with Disc_graph.as_default(): simMat = modelDiff.transform(img_array, synth_images, segm_array) for i in range(sub_size): # filename = path.join(out_path,str(i+kb)+'_rgb.png') # cv2.imwrite(filename,cv2.resize(img_array[i,...], (2048, 1024),interpolation=cv2.INTER_LINEAR)) # filename = path.join(out_path,str(i+kb)+'_segm.png') # cv2.imwrite(filename,cv2.resize(segm_array[i,...], (2048, 1024),interpolation=cv2.INTER_NEAREST)) filename = path.join(out_path, str(i + kb) + '_dissim.png') cv2.imwrite(filename, simMat[i, ...]) filename = path.join(out_path, str(i + kb) + '_dissim.npy') np.save( filename, cv2.resize(simMat[i, ...], (2048, 1024), interpolation=cv2.INTER_LINEAR))
def main(modelname, net_config, gan_config, disc_config, datasetSem, datasetGAN, datasetDisc, starting_weights, flag_measure, output_mat, flag_entropy, thresholds, start, _run): for key in gan_config: setattr(a, key, gan_config[key]) for key in disc_config: setattr(b, key, disc_config[key]) setattr(a, 'EXP_OUT', EXP_OUT) setattr(a, 'RUN_id', _run._id) setattr(b, 'EXP_OUT', EXP_OUT) setattr(b, 'RUN_id', _run._id) disc_data_path = os.path.join(datasetDisc['image_input_dir'], str(gan_config['checkpoint']) + "_full") data_id = str(gan_config['checkpoint']) setattr(b, 'DATA_id', data_id) # Set up the directories for diagnostics output_dir = create_directories(_run._id, ex) # load the data for the data description data_desc = get_dataset(datasetSem['name']) model = get_model(modelname) net = model(data_description=data_desc.get_data_description(), output_dir=output_dir, **net_config) net.import_weights(filepath=starting_weights) print("INFO: SemSegNet Imported weights succesfully") GAN_graph = tf.Graph() with GAN_graph.as_default(): # create the network gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9) GAN_sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) if gan_config['type'] == 'cascRef': dataGAN = get_dataset('cityscapes_cascGAN') cGAN_model = get_model('cascGAN') if a.checkpoint is not None: ckp = os.path.join(a.EXP_OUT, str(a.checkpoint)) modelGAN = cGAN_model( GAN_sess, dataset_name='cityscapes_cascGAN', image_size=disc_config['input_image_size'], checkpoint_dir=output_dir, data_desc=dataGAN.get_data_description(), is_training=False, checkpoint=ckp, vgg_checkpoint= "/cluster/work/riner/users/haldavid/Checkpoints/VGG_Model/imagenet-vgg-verydeep-19.mat" ) else: # load the dataset class dataGAN = get_dataset(datasetGAN['name']) # data = data(**datasetGAN) cGAN_model = get_model('cGAN') modelGAN = cGAN_model( GAN_sess, checkpoint_dir=output_dir, data_desc=dataGAN.get_data_description(), feature_matching=gan_config['feature_matching'], checkpoint=os.path.join(a.EXP_OUT, str(a.checkpoint)), gen_type=gan_config['type'], use_grayscale=gan_config['use_grayscale']) print("INFO: Generative model imported weights succesfully") Disc_graph = tf.Graph() with Disc_graph.as_default(): gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9) sessD = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) dataD = get_dataset(datasetDisc['name']) dataD = dataD(disc_data_path, **datasetDisc) disc_model = get_model('simDisc') disc_checkpoint = None if disc_config['checkpoint'] is not None: disc_checkpoint = os.path.join(a.EXP_OUT, str(disc_config['checkpoint'])) modelDiff = disc_model(sess=sessD, checkpoint_dir=output_dir, pos_weight=disc_config['pos_weight'], data=dataD, arch=disc_config['arch'], use_grayscale=disc_config['use_grayscale'], checkpoint=disc_checkpoint, use_segm=disc_config['use_segm'], batch_size=disc_config['batch_size'], feature_extractor=os.path.join( a.EXP_OUT, str(a.checkpoint))) if disc_config['checkpoint'] is None: print("INFO: Begin training simDisc") tmp = modelDiff.train(b) _run.info['simDisc_predictions'] = tmp _run.info['simDisc_mean_predictions'] = np.mean(tmp, axis=0) _run.info['simDisc_stdDev'] = np.std(tmp, axis=0) print("INFO: Finished training simDisc") else: print("INFO: Init and loaded checpoint for simDisc") if flag_measure: benchmarks = ['measure'] else: benchmarks = ['wilddash', 'posneg', 'valid', 'measure'] data_SemSeg = data_desc(**datasetSem) # thresholds = [0.2,0.4,0.6,0.8] # thresholds = [0.85,0.9,0.95,0.99] _run.info['thresholds'] = thresholds for set in benchmarks: if set == "measure": dataset = data_SemSeg.get_measureset(tf_dataset=False) elif set == "valid": dataset = data_SemSeg.get_validation_set(tf_dataset=False) else: data = get_dataset(set) head, _ = os.path.split(datasetDisc['image_input_dir']) data = data(os.path.join(head, set)) dataset = data.get_validation_set(tf_dataset=False) sem_seg_images, rgb_images, masks, gt_seg_images, seg_seg_labels, output_probs = predict_output( net, output_dir, dataset, data_SemSeg, flag_entropy, net_config['num_classes']) with GAN_sess.as_default(): with GAN_graph.as_default(): if gan_config['type'] == 'cascRef': synth_images = modelGAN.transform(a, seg_seg_labels) else: synth_images = modelGAN.transform(a, sem_seg_images) with sessD.as_default(): with Disc_graph.as_default(): simMat = modelDiff.transform(rgb_images, synth_images, sem_seg_images) ############################################################################################ # Perhaps need to also give number of patches per dimension. simMatSSIM = computePatchSSIM(rgb_images, synth_images, datasetDisc['ppd']) ############################################################################################ temp_iou = computeIOU(simMat, masks, thresholds) temp_pr = computePRvalues(simMat, masks, thresholds) _run.info[set + '_IOU'] = temp_iou _run.info[set + '_PRvals'] = temp_pr _run.info[set + '_F1score'] = 2 * np.asarray(temp_pr[1]) * np.asarray( temp_pr[2]) / (np.asarray(temp_pr[1]) + np.asarray(temp_pr[2])) # _run.info[set+'_SSIM_IOU'] = computeIOU(simMatSSIM, masks, thresholds) # _run.info[set+'_SSIM_PRvals'] = computePRvalues(simMatSSIM, masks) if flag_entropy and set is not 'posneg': entropy = ShannonEntropy(output_probs) # _run.info[set+'_meanVarEntropyOoD'] = [np.mean(entropy[masks.astype(bool)]),np.var(entropy[masks.astype(bool)],ddof=1)] # _run.info[set+'_meanVarEntropyID'] = [np.mean(entropy[~masks.astype(bool)]),np.var(entropy[~masks.astype(bool)],ddof=1)] temp_iou = computeIOU(entropy, masks, thresholds) temp_pr = computePRvalues(entropy, masks, thresholds) _run.info[set + '_entropy_IOU'] = temp_iou _run.info[set + '_entropy_PRvals'] = temp_pr _run.info[set + '_entropy_F1score'] = 2 * np.asarray( temp_pr[1]) * np.asarray(temp_pr[2]) / ( np.asarray(temp_pr[1]) + np.asarray(temp_pr[2])) k = masks.shape[0] if output_mat and set is not 'posneg': if not os.path.exists(os.path.join(output_dir, set)): os.makedirs(os.path.join(output_dir, set)) if set is 'measure': k = 25 matrix_path = os.path.join(output_dir, set, "mskMat.npy") np.save(matrix_path, masks[0:k, ...]) matrix_path = os.path.join(output_dir, set, "simMat.npy") np.save(matrix_path, simMat[0:k, ...]) # matrix_path = os.path.join(output_dir,set,"ssmMat.npy") # np.save(matrix_path, simMatSSIM[0:k, ...]) matrix_path = os.path.join(output_dir, set, "rgbMat.npy") np.save(matrix_path, rgb_images[0:k, ...]) matrix_path = os.path.join(output_dir, set, "synMat.npy") np.save(matrix_path, synth_images[0:k, ...]) matrix_path = os.path.join(output_dir, set, "semMat.npy") np.save(matrix_path, sem_seg_images[0:k, ...]) matrix_path = os.path.join(output_dir, set, "gtsMat.npy") np.save(matrix_path, gt_seg_images[0:k, ...]) if flag_entropy: matrix_path = os.path.join(output_dir, set, "entMat.npy") np.save(matrix_path, entropy)
def main(modelname, net_config, gan_config, disc_config, datasetSem, datasetGAN, datasetDisc, starting_weights, flag_measure, output_mat, flag_entropy, thresholds, start, _run): for key in gan_config: setattr(a, key, gan_config[key]) for key in disc_config: setattr(b, key, disc_config[key]) setattr(a, 'EXP_OUT', EXP_OUT) setattr(a, 'RUN_id', _run._id) setattr(b, 'EXP_OUT', EXP_OUT) setattr(b, 'RUN_id', _run._id) disc_data_path = os.path.join(datasetDisc['image_input_dir'], str(gan_config['checkpoint']) + "_full") data_id = str(gan_config['checkpoint']) setattr(b, 'DATA_id', data_id) # Set up the directories for diagnostics output_dir = create_directories(_run._id, ex) # load the data for the data description data_desc = get_dataset(datasetSem['name']) model = get_model(modelname) net = model(data_description=data_desc.get_data_description(), output_dir=output_dir, **net_config) # net.import_weights(filepath=starting_weights) print("INFO: SemSegNet Imported weights succesfully") GAN_graph = tf.Graph() with GAN_graph.as_default(): # create the network gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9) GAN_sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) if gan_config['type'] == 'cascRef': dataGAN = get_dataset('cityscapes_cascGAN') cGAN_model = get_model('cascGAN') if a.checkpoint is not None: ckp = os.path.join(a.EXP_OUT, str(a.checkpoint)) modelGAN = cGAN_model( GAN_sess, dataset_name='cityscapes_cascGAN', image_size=disc_config['input_image_size'], checkpoint_dir=output_dir, data_desc=dataGAN.get_data_description(), is_training=False, checkpoint=ckp, vgg_checkpoint= "/cluster/work/riner/users/haldavid/Checkpoints/VGG_Model/imagenet-vgg-verydeep-19.mat" ) else: # load the dataset class dataGAN = get_dataset(datasetGAN['name']) # data = data(**datasetGAN) cGAN_model = get_model('cGAN') modelGAN = cGAN_model( GAN_sess, checkpoint_dir=output_dir, data_desc=dataGAN.get_data_description(), feature_matching=gan_config['feature_matching'], checkpoint=os.path.join(a.EXP_OUT, str(a.checkpoint)), gen_type=gan_config['type'], use_grayscale=gan_config['use_grayscale']) print("INFO: Generative model imported weights succesfully") Disc_graph = tf.Graph() with Disc_graph.as_default(): gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9) sessD = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) dataD = get_dataset(datasetDisc['name']) dataD = dataD(disc_data_path, **datasetDisc) disc_model = get_model('simDisc') disc_checkpoint = None if disc_config['checkpoint'] is not None: disc_checkpoint = os.path.join(a.EXP_OUT, str(disc_config['checkpoint'])) modelDiff = disc_model(sess=sessD, checkpoint_dir=output_dir, pos_weight=disc_config['pos_weight'], data=dataD, arch=disc_config['arch'], use_grayscale=disc_config['use_grayscale'], checkpoint=disc_checkpoint, use_segm=disc_config['use_segm'], batch_size=disc_config['batch_size'], feature_extractor=os.path.join( a.EXP_OUT, str(a.checkpoint))) if disc_config['checkpoint'] is None: print("INFO: Begin training simDisc") tmp = modelDiff.train(b) _run.info['simDisc_predictions'] = tmp _run.info['simDisc_mean_predictions'] = np.mean(tmp, axis=0) _run.info['simDisc_stdDev'] = np.std(tmp, axis=0) print("INFO: Finished training simDisc") else: print("INFO: Init and loaded checpoint for simDisc") if flag_measure: benchmarks = ['measure'] else: benchmarks = ['wilddash', 'posneg', 'valid', 'measure'] data_SemSeg = data_desc(**datasetSem) _run.info['thresholds'] = thresholds ########################################################################### # mapping from Mapillary classes to Deeplab classes if start == 0: with open('/cluster/work/riner/users/haldavid/config.json') as f: config = json.load(f) else: with open( '/Volumes/Netti HD /Master Thesis/mapillary/config.json') as f: config = json.load(f) def map_class(name): """Map a Mapillary class onto one of the 19 cityscapes classes or void.""" direct_mapping = { 'construction--barrier--fence': 'fence', 'construction--barrier--wall': 'wall', 'construction--flat--road': 'road', 'construction--flat--sidewalk': 'sidewalk', 'construction--structure--building': 'building', 'human--person': 'person', 'nature--sky': 'sky', 'nature--terrain': 'terrain', 'nature--vegetation': 'vegetation', 'object--support--pole': 'pole', 'object--support--utility-pole': 'pole', 'object--traffic-light': 'traffic light', 'object--traffic-sign--front': 'traffic sign', 'object--vehicle--bicycle': 'bicycle', 'object--vehicle--bus': 'bus', 'object--vehicle--car': 'car', 'object--vehicle--motorcycle': 'motorcycle', 'object--vehicle--on-rails': 'train', 'object--vehicle--truck': 'truck', } if name in direct_mapping: return direct_mapping[name] elif name.startswith('human--rider'): return 'rider' elif name.startswith('marking'): return 'road' else: return 'void' original_labels_mapi = { i: { 'name': v['name'], 'color': v['color'], 'mapping': map_class(v['name']) } for i, v in enumerate(config['labels']) } # array to look up the label_id of a given color color_map = np.ndarray(shape=(256**3), dtype='int32') color_map[:] = -1 for c, v in original_labels_mapi.items(): rgb = v['color'] rgb = rgb[0] * 65536 + rgb[1] * 256 + rgb[2] color_map[rgb] = c # apply same label mapping as for original cityscapes labelinfo_mapi = { -1: { 'name': 'void', 'color': [0, 0, 0] }, 0: { 'name': 'road', 'color': [128, 64, 128] }, 1: { 'name': 'sidewalk', 'color': [244, 35, 232] }, 2: { 'name': 'building', 'color': [70, 70, 70] }, 3: { 'name': 'wall', 'color': [70, 70, 70] }, 4: { 'name': 'fence', 'color': [190, 153, 153] }, 5: { 'name': 'pole', 'color': [153, 153, 153] }, 6: { 'name': 'traffic light', 'color': [0, 0, 0] }, 7: { 'name': 'traffic sign', 'color': [220, 220, 0] }, 8: { 'name': 'vegetation', 'color': [107, 142, 35] }, 9: { 'name': 'terrain', 'color': [107, 142, 35] }, 10: { 'name': 'sky', 'color': [70, 130, 180] }, 11: { 'name': 'person', 'color': [220, 20, 60] }, 12: { 'name': 'rider', 'color': [220, 20, 60] }, 13: { 'name': 'car', 'color': [0, 0, 142] }, 14: { 'name': 'truck', 'color': [0, 0, 142] }, 15: { 'name': 'bus', 'color': [0, 0, 142] }, 16: { 'name': 'train', 'color': [0, 0, 142] }, 17: { 'name': 'motorcycle', 'color': [0, 0, 142] }, 18: { 'name': 'bicycle', 'color': [119, 11, 32] } } label_lookup_mapi = [ next(i for i in labelinfo_mapi if labelinfo_mapi[i]['name'] == v['mapping']) for v in original_labels_mapi.values() ] lookup_mapi = np.array([ labelinfo_mapi[i]['color'] for i in range(max(labelinfo_mapi.keys()) + 1) ]).astype(int) ########################################################################### # mapping from Deeplab classes to Adapnet classes original_labelinfo_dl = { 0: { 'name': 'road', 'mapping': 'road' }, 1: { 'name': 'sidewalk', 'mapping': 'sidewalk' }, 2: { 'name': 'building', 'mapping': 'building' }, 3: { 'name': 'wall', 'mapping': 'building' }, 4: { 'name': 'fence', 'mapping': 'fence' }, 5: { 'name': 'pole', 'mapping': 'pole' }, 6: { 'name': 'traffic light', 'mapping': 'void' }, 7: { 'name': 'traffic sign', 'mapping': 'traffic sign' }, 8: { 'name': 'vegetation', 'mapping': 'vegetation' }, 9: { 'name': 'terrain', 'mapping': 'vegetation' }, 10: { 'name': 'sky', 'mapping': 'sky' }, 11: { 'name': 'person', 'mapping': 'person' }, 12: { 'name': 'rider', 'mapping': 'person' }, 13: { 'name': 'car', 'mapping': 'vehicle' }, 14: { 'name': 'truck', 'mapping': 'vehicle' }, 15: { 'name': 'bus', 'mapping': 'vehicle' }, 16: { 'name': 'train', 'mapping': 'vehicle' }, 17: { 'name': 'motorcycle', 'mapping': 'vehicle' }, 18: { 'name': 'bicycle', 'mapping': 'bicycle' }, 255: { 'name': 'void', 'mapping': 'void' } } labelinfo = { 0: { 'name': 'void', 'color': [0, 0, 0] }, 1: { 'name': 'sky', 'color': [70, 130, 180] }, 2: { 'name': 'building', 'color': [70, 70, 70] }, 3: { 'name': 'road', 'color': [128, 64, 128] }, 4: { 'name': 'sidewalk', 'color': [244, 35, 232] }, 5: { 'name': 'fence', 'color': [190, 153, 153] }, 6: { 'name': 'vegetation', 'color': [107, 142, 35] }, 7: { 'name': 'pole', 'color': [153, 153, 153] }, 8: { 'name': 'vehicle', 'color': [0, 0, 142] }, 9: { 'name': 'traffic sign', 'color': [220, 220, 0] }, 10: { 'name': 'person', 'color': [220, 20, 60] }, 11: { 'name': 'bicycle', 'color': [119, 11, 32] } } label_lookup_dl = [ next(i for i in labelinfo if labelinfo[i]['name'] == k['mapping']) for _, k in original_labelinfo_dl.items() ] lookup = np.array([ labelinfo[i]['color'] for i in range(max(labelinfo.keys()) + 1) ]).astype(int) # base_path = path.join(DATA_BASEPATH, 'fishyscapes_newfog') # if 'TMPDIR' in environ: # print('INFO loading dataset into machine ... ') # # first load the zipfile into a closer memory location, then load all the # # images # zip = zipfile.ZipFile(path.join(base_path, 'testset.zip'), 'r') # localtmp = environ['TMPDIR'] # zip.extractall(localtmp) # zip.close() # base_path = localtmp # # print('DONE loading dataset into machine ... ') ########################################################################### sub_size = 100 if start == 0: input_path = "/cluster/work/riner/users/blumh/mapillary_evaluation_set" out_path = path.join( "/cluster/work/riner/users/haldavid/MapillaryResultsCropped", disc_config['arch']) else: input_path = "/Users/David/Downloads/mapillary_evaluation_set" out_path = path.join("/Users/David/Desktop/out", disc_config['arch']) if not os.path.exists(out_path): os.makedirs(out_path) #N H W C img_array = np.zeros((sub_size, 256, 256, 3)) segm_array = np.zeros((sub_size, 256, 256, 3)) gt_array = np.zeros((sub_size, 256, 256, 3)) mask_array = np.zeros((sub_size, 256, 256)) out_mask_array = np.zeros((sub_size, 256, 256)) for i in range(sub_size): #RGB img = cv2.imread(path.join(input_path, str(i) + '_rgb.png')) # dl_labels = np.expand_dims(cv2.imread( path.join(input_path, str(i) + '_predict.png'))[:, :, 0], axis=0) cs_labels = np.asarray(label_lookup_dl, dtype='int32')[dl_labels] segm = np.array(lookup[cs_labels[:]]).astype('uint8')[..., ::-1] gt_labels = cv2.imread(path.join(input_path, str(i) + '_segm.png'))[..., ::-1] gt_labels = gt_labels.dot(np.array([65536, 256, 1], dtype='int32')) gt_segm = color_map[gt_labels] # apply mapping gt_segm = np.asarray(label_lookup_mapi, dtype='int32')[gt_segm] gt_segm = np.array(lookup_mapi[gt_segm[:]]).astype('uint8')[..., ::-1] mask = cv2.imread(path.join(input_path, str(i) + '_mask.png'))[..., 0] #Crop all in the same manor 4:3 we crop to 4:2 crop_margin = int(img.shape[0] / 6) img = img[crop_margin:-crop_margin, ...] segm = segm[0, crop_margin:-crop_margin, ...] gt_segm = gt_segm[crop_margin:-crop_margin, ...] mask = mask[crop_margin:-crop_margin, ...] img_array[i, ...] = cv2.resize(img, (256, 256), interpolation=cv2.INTER_LINEAR) segm_array[i, ...] = cv2.resize(segm, (256, 256), interpolation=cv2.INTER_NEAREST) gt_array[i, ...] = cv2.resize(gt_segm, (256, 256), interpolation=cv2.INTER_NEAREST) mask_array[i, ...] = cv2.resize(mask, (256, 256), interpolation=cv2.INTER_NEAREST) with GAN_sess.as_default(): with GAN_graph.as_default(): synth_images, pred_descriminator = modelGAN.transform_withD( a, segm_array, img_array) with sessD.as_default(): with Disc_graph.as_default(): simMat = modelDiff.transform(img_array, synth_images, segm_array) for i in range(sub_size): gt_mask = error_mask(segm_array[i], gt_array[i]) out_mask = np.logical_or(gt_mask, mask_array[i]) out_mask_array[i, ...] = out_mask # filename = path.join(out_path,str(i)+'_dissim.npy') # np.save(filename,cv2.resize(simMat[i,...], (2048, 1024),interpolation=cv2.INTER_LINEAR)) # if disc_config['arch']=='arch13': # filename = path.join(out_path,str(i)+'_discsim.npy') # np.save(filename,cv2.resize(pred_descriminator[i,...], (2048, 1024),interpolation=cv2.INTER_NEAREST)) # filename = path.join(out_path,str(i)+'_mask.npy') # np.save(filename,cv2.resize(out_mask, (2048, 1024),interpolation=cv2.INTER_NEAREST)) # filename = path.join(out_path,str(i)+'_rgb.npy') # np.save(filename,cv2.resize(img_array[i,...], (2048, 1024),interpolation=cv2.INTER_LINEAR)) filename = path.join(out_path, str(i) + '_dissim.npy') np.save(filename, simMat) if disc_config['arch'] == 'arch13': filename = path.join(out_path, str(i) + '_discsim.npy') np.save(filename, pred_descriminator) filename = path.join(out_path, str(i) + '_mask.npy') np.save(filename, out_mask_array) filename = path.join(out_path, str(i) + '_rgb.npy') np.save(filename, img_array) filename = path.join(out_path, str(i) + '_synth.npy') np.save(filename, synth_images) filename = path.join(out_path, str(i) + '_segm.npy') np.save(filename, segm_array) filename = path.join(out_path, str(i) + '_gt.npy') np.save(filename, gt_array)
def main(dataset, net_config, _run): # Add all of the config into the helper class for key in net_config: setattr(a, key, net_config[key]) output_dir = create_directories(_run._id, ex) # load the dataset class data = get_dataset(dataset['name']) data = data(**dataset) # Get the data descriptors with the shape of data coming data_description = data.get_data_description() data_description = [data_description[0], { key: [None, *description] for key, description in data_description[1].items()}] # Create an iterator for the data iter_handle = tf.placeholder(tf.string, shape=[], name='training_placeholder') iterator = tf.data.Iterator.from_string_handle( iter_handle, *data_description) training_batch = iterator.get_next() # Create a placeholder for if one is training and a Variable # for the flipping probability a.is_train = tf.placeholder(tf.bool, name="is_train") a.decay_term = np.exp(-a.flip_lambda) # x = e^-lambda where N(t)=N(0)*x^t a.flip_prob = tf.Variable(a.flip_label_init_prob*a.flip_decay_perc, name="flip_prob") a.flip_end_prob = a.flip_label_init_prob*(1-a.flip_decay_perc) # Create pix2pix model model = create_model(training_batch['labels'], training_batch['rgb']) # Define fetches for evaluation such that these are returned by the graph with tf.name_scope("encode_images"): if a.val_target_output == True: display_fetches = { "targets": model.targets, "outputs": model.outputs } else: display_fetches = { "outputs": model.outputs } saver = tf.train.Saver() # Limit GPU fraction gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=0.9) sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) sess.run(tf.global_variables_initializer()) # There are local variables in the metrics sess.run(tf.local_variables_initializer()) # Add the images to the summary tf.summary.image("input",model.inputs[...,::-1]) tf.summary.image("output",model.outputs[...,::-1]) tf.summary.image("target",model.targets[...,::-1]) # Add variable values to summary for var in tf.trainable_variables(): tf.summary.histogram(var.op.name + "/values", var) # Add gradients values to summary for grad, var in model.discrim_grads_and_vars + model.gen_grads_and_vars: tf.summary.histogram(var.op.name + "/gradients", grad) # Merge all summaries to one instance merged_summary = tf.summary.merge_all() # Start tensorflow summary filewriter if output_dir is not None: train_writer = tf.summary.FileWriter(output_dir) train_writer.add_graph(sess.graph) if a.mode=="train": print("INFO: Got train set") input_data = data.get_trainset() data_iterator = input_data.repeat(a.max_epochs).batch(a.batch_size).make_one_shot_iterator() # data_iterator = input_data.batch(a.batch_size).make_initializable_iterator() validation_data = data.get_validation_set() valid_iterator = validation_data.batch(a.batch_size).make_one_shot_iterator() valid_handle = sess.run(valid_iterator.string_handle()) else: print("INFO: Got test set") input_data = data.get_testset(num_items=num_test_images) data_iterator = input_data.batch(a.batch_size).make_one_shot_iterator() data_handle = sess.run(data_iterator.string_handle()) if a.checkpoint is not None: print("loading model from checkpoint") checkpoint = tf.train.latest_checkpoint(os.path.join(EXP_OUT,str(a.checkpoint))) saver.restore(sess, checkpoint) if a.mode=="test": print("INFO: Starting Test") for i in range(0,num_test_images): results=sess.run(display_fetches, feed_dict={iter_handle: data_handle}) filename = "output" + str(i) + ".png" cv2.imwrite(os.path.join(a.file_output_dir,filename), (results['outputs'][0,:,:,:]), [int(cv2.IMWRITE_JPEG_QUALITY), 90]) else: print('INFO: Start training') stdout.flush() start = time.time() # for k in range(1,a.max_epochs+1): for k in range(1,2): i = 1 while True: # Makes the graph run through the complete graph from beginning # to end fetches ={ "train": model.train, } if i%a.num_print == 0: fetches["discrim_loss"] = model.discrim_loss fetches["gen_loss_GAN"] = model.gen_loss_GAN fetches["gen_loss_L1"] = model.gen_loss_L1 fetches["outputs"] = model.outputs fetches["inputs"] = model.inputs fetches["targets"] = model.targets fetches["predict_fake"] = model.predict_fake fetches["predict_real"] = model.predict_real try: sum, results=sess.run([merged_summary, fetches], feed_dict={iter_handle: data_handle, a.is_train: True}) except tf.errors.OutOfRangeError: print("INFO: Training set has %d elements, starting next epoch" % i) break rate = (i) / (time.time() - start) ssim_score = compare_ssim(cv2.cvtColor(results["targets"][0,:,:,:], cv2.COLOR_BGR2GRAY),cv2.cvtColor(results["outputs"][0,:,:,:], cv2.COLOR_BGR2GRAY), data_range=255) print ('Epoch %d\tStep %d: D_Loss: %f, \tG_Loss: %f, \tL1_Loss: %f, \tSSIM_Score: %f, \tRate: %f img/sec, \tglobal_step: %d' % (k,i,results['discrim_loss'],results['gen_loss_GAN'],results['gen_loss_L1'],ssim_score,rate,tf.train.global_step(sess, model.global_step))) stdout.flush() d_loss = tf.Summary( value=[tf.Summary.Value(tag='discriminator_loss', simple_value=results['discrim_loss'])]) g_loss = tf.Summary( value=[tf.Summary.Value(tag='generator_loss_GAN', simple_value=results['gen_loss_GAN'])]) l_loss = tf.Summary( value=[tf.Summary.Value(tag='generator_loss_L1', simple_value=results['gen_loss_L1'])]) ssim = tf.Summary( value=[tf.Summary.Value(tag='SSIM_score', simple_value=ssim_score)]) p_fake = tf.Summary( value=[tf.Summary.Value(tag='predict_fake_values', simple_value=np.mean(results['predict_fake']))]) p_real = tf.Summary( value=[tf.Summary.Value(tag='predict_real_values', simple_value=np.mean(results['predict_real']))]) train_writer.add_summary(d_loss, tf.train.global_step(sess, model.global_step)) train_writer.add_summary(g_loss, tf.train.global_step(sess, model.global_step)) train_writer.add_summary(l_loss, tf.train.global_step(sess, model.global_step)) train_writer.add_summary(ssim, tf.train.global_step(sess, model.global_step)) train_writer.add_summary(p_fake, tf.train.global_step(sess, model.global_step)) train_writer.add_summary(p_real, tf.train.global_step(sess, model.global_step)) train_writer.add_summary(sum, tf.train.global_step(sess, model.global_step)) else: try: results=sess.run(fetches, feed_dict={iter_handle: data_handle, a.is_train: True}) except tf.errors.OutOfRangeError: print("INFO: Training set has %d elements, starting next epoch" % (i-1)) break i=i+1 print("INFO: Saving model at: "+ output_dir) saver.save(sess, os.path.join(output_dir, "model")) print('INFO: Training finished.') stdout.flush() print('INFO: Starting Validation') i=0 fake_values = np.zeros(15) real_values = np.zeros(15) while True: fetches ={ "predict_fake": model.predict_fake, "predict_real": model.predict_real } try: results, preds = sess.run([display_fetches, fetches], feed_dict={iter_handle: valid_handle, a.is_train: False}) except tf.errors.OutOfRangeError: print("INFO: Finished validation of %d images after training" % i) break fake_values[i] = (np.mean(preds['predict_fake'])) real_values[i] = (np.mean(preds['predict_real'])) filename = str(_run._id) + "_validation" + str(i+1) + ".png" cv2.imwrite(os.path.join(a.file_output_dir,filename), (results['outputs'][0,:,:,:]), [int(cv2.IMWRITE_JPEG_QUALITY), 90]) if a.val_target_output == True: filename = str(_run._id) + "_target" + str(i+1) + ".png" cv2.imwrite(os.path.join(a.file_output_dir,filename), (results['targets'][0,:,:,:]), [int(cv2.IMWRITE_JPEG_QUALITY), 90]) i=i+1 _run.info['predictions_fake'] = fake_values _run.info['predictions_real'] = real_values if output_dir is not None: train_writer.close()
def main(modelname, net_config, gan_config, disc_config, datasetSem, datasetGAN, datasetDisc, starting_weights, input_folder, sets, _run): for key in gan_config: setattr(a, key, gan_config[key]) setattr(a, 'EXP_OUT', EXP_OUT) setattr(a, 'RUN_id', _run._id) # Set up the directories for diagnostics output_dir = create_directories(_run._id, ex) # load the data for the data description data_desc = get_dataset(datasetSem['name']) dataFlag = (input_folder is not None) if dataFlag: # load images in input_folder eval_image_paths = load_list_path(input_folder) elif sets == "measure": data = data_desc(**datasetSem) dataset = data.get_measureset(tf_dataset=False) else: data = data_desc(**datasetSem) dataset = data.get_validation_set(tf_dataset=False) model = get_model(modelname) net = model(data_description=data_desc.get_data_description(), output_dir=output_dir, **net_config) net.import_weights(filepath=starting_weights) print("INFO: Imported weights succesfully") GAN_graph = tf.Graph() with GAN_graph.as_default(): # create the network gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9) GAN_sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) # load the dataset class dataGAN = get_dataset(datasetGAN['name']) # data = data(**datasetGAN) cGAN_model = get_model('cGAN') modelGAN = cGAN_model(GAN_sess, checkpoint_dir=output_dir, data_desc=dataGAN.get_data_description(), checkpoint=os.path.join(a.EXP_OUT, str(a.checkpoint))) print("INFO: GAN Imported weights succesfully") # tf.reset_default_graph() Disc_graph = tf.Graph() with Disc_graph.as_default(): gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9) sessD = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) dataD = get_dataset(datasetDisc['name']) dataD = dataD(datasetDisc['image_input_dir'], **datasetDisc) disc_model = get_model('simDisc') modelDiff = disc_model(sess=sessD, checkpoint_dir=output_dir, data=dataD, is_training=False, checkpoint=os.path.join( a.EXP_OUT, str(disc_config['checkpoint']))) print("INFO: Disc Imported weights succesfully") ss_run_id = starting_weights.split('/')[-2] gan_run_id = str(a.checkpoint) folder_name = ss_run_id + "_" + gan_run_id + "_" + str( disc_config['checkpoint']) + "_" + sets base_output_path = os.path.join(a.file_output_dir, folder_name) if not os.path.exists(base_output_path): os.makedirs(base_output_path) if dataFlag: sem_seg_images, rgb_images = predict_output(net, output_dir, eval_image_paths, data_desc, dataFlag=dataFlag) else: sem_seg_images, rgb_images, masks, sem_seg_GT = predict_output( net, output_dir, dataset, data, dataFlag=dataFlag) matrix_path = os.path.join(base_output_path, "mskMat.npy") np.save(matrix_path, masks) matrix_path = os.path.join(base_output_path, "gtsMat.npy") np.save(matrix_path, sem_seg_GT) print("Done with prediction of semantic segmentation") with GAN_sess.as_default(): with GAN_graph.as_default(): synth_images = modelGAN.transform(a, sem_seg_images) print("Done with prediction of GAN") with sessD.as_default(): with Disc_graph.as_default(): simMat = modelDiff.transform(rgb_images, synth_images, sem_seg_images) matrix_path = os.path.join(base_output_path, "simMat.npy") np.save(matrix_path, simMat) matrix_path = os.path.join(base_output_path, "rgbMat.npy") np.save(matrix_path, rgb_images) matrix_path = os.path.join(base_output_path, "synMat.npy") np.save(matrix_path, synth_images) matrix_path = os.path.join(base_output_path, "semMat.npy") np.save(matrix_path, sem_seg_images)