def main(dataset, net_config, _run):
# Add all of the config into the helper class
for key in net_config:
setattr(a, key, net_config[key])
setattr(a, 'EXP_OUT', EXP_OUT)
setattr(a, 'RUN_id', _run._id)
output_dir = create_directories(_run._id, ex)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
# load the dataset class
data = get_dataset(dataset['name'])
data = data(**dataset)
model = AutoEncoder(sess,
image_size=a.input_image_size,
batch_size=a.batch_size,
output_size=a.input_image_size,
dataset_name=dataset['name'],
checkpoint_dir=output_dir,
data=data,
momentum=a.batch_momentum,
aef_dim=a.naef,
noise_std_dev=a.noise_std_dev)
if a.mode == 'train':
tmp = model.train(a)
_run.info['predictions'] = tmp
_run.info['mean_predictions'] = np.mean(tmp, axis=0)
elif a.mode == 'valid':
tmp = model.validate(a)
_run.info['predictions'] = tmp
_run.info['mean_predictions'] = np.mean(tmp, axis=0)
else:
model.test(a)
def evaluate_temperature_scaling(experiments, temperatures):
for exp_id in experiments:
exp = ExperimentData(exp_id)
dataset = exp.get_record()['info']['dataset']
data = get_dataset(dataset['name'])
data_description = list(data.get_data_description())
if exp.get_record()['config']['method'] == 'new_class':
data_description[2] += 1
model = get_model(exp.get_record()['config']['modelname'])
# get the label_flip specs
label_flip = None
if 'label_flip' in dataset['augmentation']:
label_flip = dataset['augmentation']['label_flip']
def evaluation(parameters):
with model(data_description=data_description, **parameters) as net:
import_weights_into_network(net, exp_id)
return measure_metrics(
net,
data(**dataset).get_testset(),
exp.get_record()['config']['uncertainty_metrics'],
label_flip=label_flip)
result = grid_search(evaluation, {'temperature_scaling': temperatures},
exp.get_record()['config']['net_config'])
info = exp.get_record()['info']
info['temperature_grid_search'] = force_bson_encodeable(result)
exp.update_record({'info': info})
def main(dataset, net_config, img_h, img_w, _run):
# Add all of the config into the helper class
for key in net_config:
setattr(a, key, net_config[key])
setattr(a, 'EXP_OUT', EXP_OUT)
setattr(a, 'RUN_id', _run._id)
output_dir = create_directories(_run._id, ex)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.99)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
# load the dataset class
data = get_dataset(dataset['name'])
data = data(img_h=img_h, img_w=img_w, **dataset)
cGAN_model = get_model('cascGAN')
if a.checkpoint is not None:
ckp = os.path.join(a.EXP_OUT, str(a.checkpoint))
else:
ckp = None
trainFlag = (a.mode == 'train')
model = cGAN_model(sess,
dataset_name=dataset['name'],
image_size=img_h,
checkpoint_dir=output_dir,
data=data,
data_desc=data.get_data_description(),
is_training=trainFlag,
checkpoint=ckp,
vgg_checkpoint=net_config['vgg_checkpoint'])
if a.mode == 'train':
model.train(a)
else:
model.transformDatasets(a, data)
def train_ambiguous(modelname, net_config, dataset, starting_weights, method,
num_iterations, uncertainty_metrics, _run):
# Set up the directories for diagnostics
output_dir = create_directories(_run._id, ex)
data = get_dataset(dataset['name'])
data_description = list(data.get_data_description())
num_classes = data_description[2]
args = False
if isinstance(method, list):
args = method[1:]
method = method[0]
# augmentate the class labels
if method == 'flip_classes':
# randomly map two classes onto each other to make them ambiguos
classes = np.random.choice(list(range(num_classes)),
size=2,
replace=False)
dataset.setdefault('augmentation',
{})['label_flip'] = (classes[0], classes[1],
np.random.rand())
elif method == 'new_class':
# randomly label a given class as a new, nonexisting class
data_description[2] = num_classes + 1
if args:
old_class = args[0]
print(args)
else:
old_class = np.random.choice(list(range(num_classes)))
dataset.setdefault('augmentation',
{})['label_flip'] = (old_class, num_classes,
np.random.rand())
elif method == 'merge':
# randomly merge two classes together
classes = np.random.choice(list(range(num_classes)),
size=2,
replace=False)
dataset.setdefault('augmentation',
{})['label_merge'] = (classes[0], classes[1])
_run.info.setdefault('dataset', {}).update(dataset)
model = get_model(modelname)
with model(data_description=data_description,
output_dir=output_dir,
**net_config) as net:
data = data(**dataset)
train_network(net, output_dir, data, num_iterations, starting_weights,
ex)
_run.info['measurements'] = measure_metrics(
net,
data.get_testset(),
uncertainty_metrics,
label_flip=dataset['augmentation'].get('label_flip', None))
net.close()
print(threading.enumerate())
def load_data(data_config):
"""
Load the data specified in the data_config dict.
"""
dataset_params = {
key: val
for key, val in data_config.items()
if key not in ['dataset', 'use_trainset']
}
return get_dataset(data_config['dataset'], dataset_params)
def measure(modelname, net_config, dataset, starting_weights,
uncertainty_metrics, _run):
model = get_model(modelname)
data = get_dataset(dataset['name'])
data_description = list(data.get_data_description())
if 'num_classes' in dataset:
data_description[2] = dataset['num_classes']
with model(data_description=data_description, **net_config) as net:
data = data(**dataset)
import_weights_into_network(net, starting_weights)
_run.info['measurements'] = measure_metrics(net, data.get_testset(),
uncertainty_metrics)
def main(dataset, net_config, _run):
# Add all of the config into the helper class
for key in net_config:
setattr(a, key, net_config[key])
setattr(a, 'EXP_OUT', EXP_OUT)
setattr(a, 'RUN_id', _run._id)
output_dir = create_directories(_run._id, ex)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
data = get_dataset(dataset['name'])
data = data(dataset['image_input_dir'], **dataset)
data_id = dataset['image_input_dir'].split('/')[-1].split('_')[0]
setattr(a, 'DATA_id', data_id)
disc_model = get_model('simDisc')
ckp = None
if net_config['checkpoint'] is not None:
ckp = os.path.join(a.EXP_OUT, str(net_config['checkpoint']))
if net_config['feature_extractor'] is not None:
fe_ckp = os.path.join(a.EXP_OUT,
str(net_config['feature_extractor']))
model = disc_model(sess=sess,
image_size=a.input_image_size,
batch_size=a.batch_size,
df_dim=a.ndf,
input_c_dim=3,
checkpoint_dir=output_dir,
data=data,
momentum=a.batch_momentum,
arch=net_config['arch'],
checkpoint=ckp,
feature_extractor=fe_ckp)
if a.mode == "train":
tmp = model.train(a)
_run.info['predictions'] = tmp
_run.info['mean_predictions'] = np.mean(tmp, axis=0)
elif a.mode == "predict":
input_list = glob.glob(os.path.join(a.predict_dir, "target_*.png"))
synth_list = glob.glob(os.path.join(a.predict_dir, "synth_*.png"))
segm_list = glob.glob(os.path.join(a.predict_dir, "input_*.png"))
input_list.sort(
key=lambda x: int(x.partition('_')[-1].partition('.')[0]))
synth_list.sort(
key=lambda x: int(x.partition('_')[-1].partition('.')[0]))
segm_list.sort(
key=lambda x: int(x.partition('_')[-1].partition('.')[0]))
model.predict(a, input_list, synth_list, segm_list)
def uncertainty_benchmark(modelname, net_config, dataset, starting_weights,
benchmark, uncertainty_metrics, _run):
model = get_model(modelname)
data = get_dataset(dataset['name'])
with model(data_description=data.get_data_description(),
**net_config) as net:
data = data(**dataset)
import_weights_into_network(net, starting_weights)
for metric in uncertainty_metrics:
measurements = evaluate_uncertainty(net,
data.get_testset(),
metric,
benchmark=benchmark)
_run.info.setdefault('measurements', {})[metric] = measurements
def main(modelname, dataset, net_config, _run):
# Set up the directories for diagnostics
output_dir = create_directories(_run._id, ex)
# load the dataset class, but don't instantiate it
data = get_dataset(dataset['name'])
# create the network
model = get_model(modelname)
with model(data_description=data.get_data_description(),
output_dir=output_dir,
**net_config) as net:
# now we can load the dataset inside the scope of the network graph
data = data(**dataset)
train_and_evaluate(net, output_dir, data)
def fit_and_evaluate(net_config, evaluation_data, starting_weights, _run):
"""Load weigths from training experiments and evalaute fusion against specified
data."""
dataset = get_dataset(evaluation_data['dataset'])
# evaluate individual experts
model = get_model(net_config['expert_model'])
confusion_matrices = {}
for expert in net_config['num_channels']:
model_config = deepcopy(net_config)
model_config['modality'] = expert
model_config['prefix'] = net_config['prefixes'][expert]
with model(data_description=dataset.get_data_description(),
**model_config) as net:
data = dataset(**evaluation_data)
import_weights_into_network(
net, starting_weights[model_config['prefix']])
m, conf_mat = net.score(data.get_measureset())
confusion_matrices[expert] = conf_mat
print('Evaluated network {} on {} measurement set:'.format(
expert, evaluation_data['dataset']))
print("INFO now getting test results")
m, _ = net.score(data.get_testset())
print('total accuracy {:.3f} IoU {:.3f}'.format(
m['total_accuracy'], m['mean_IoU']))
_run.info.setdefault('measurements', {}).setdefault(expert, m)
_run.info['confusion_matrices'] = confusion_matrices
# now evaluate bayes mix
with BayesFusion(data_description=dataset.get_data_description(),
confusion_matrices=confusion_matrices,
**net_config) as net:
data = dataset(**evaluation_data)
import_weights_into_network(net, starting_weights)
measurements, confusion_matrix = net.score(data.get_testset())
_run.info['measurements']['fusion'] = measurements
_run.info['confusion_matrix'] = confusion_matrix
print('Evaluated Bayes Fusion on {} data:'.format(
evaluation_data['dataset']))
print('total accuracy {:.3f} IoU {:.3f}'.format(
measurements['total_accuracy'], measurements['mean_IoU']))
# There seems to be a problem with capturing the print output, flush to be sure
stdout.flush()
def main(dataset, net_config, _run):
# Add all of the config into the helper class
for key in net_config:
setattr(a, key, net_config[key])
setattr(a, 'EXP_OUT', EXP_OUT)
setattr(a, 'RUN_id', _run._id)
output_dir = create_directories(_run._id, ex)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
# data = Cityscapes_generated(base_path="/Users/David/masterThesis/pix2pix-tensorflow/dir/224_full")
data = get_dataset(dataset['name'])
# data = data(dataset['image_input_dir'], ppd=dataset['ppd'])
data = data(dataset['image_input_dir'], **dataset)
data_id = dataset['image_input_dir'].split('/')[-1].split('_')[0]
setattr(a, 'DATA_id', data_id)
model = DiffDiscrim(sess=sess,
image_size=a.input_image_size,
batch_size=a.batch_size,
df_dim=a.ndf,
input_c_dim=3,
checkpoint_dir=output_dir,
data=data,
momentum=a.batch_momentum,
checkpoint=os.path.join(
a.EXP_OUT, str(net_config['checkpoint'])))
if a.mode == "train":
tmp = model.train(a)
_run.info['predictions'] = tmp
_run.info['mean_predictions'] = np.mean(tmp, axis=0)
elif a.mode == "predict":
input_list = glob.glob(os.path.join(a.predict_dir, "target_*.png"))
synth_list = glob.glob(os.path.join(a.predict_dir, "synth_*.png"))
segm_list = glob.glob(os.path.join(a.predict_dir, "input_*.png"))
input_list.sort(
key=lambda x: int(x.partition('_')[-1].partition('.')[0]))
synth_list.sort(
key=lambda x: int(x.partition('_')[-1].partition('.')[0]))
segm_list.sort(
key=lambda x: int(x.partition('_')[-1].partition('.')[0]))
model.predict(a, input_list, synth_list, segm_list)
def main(modelname, net_config, dataset, starting_weights, input_folder, _run):
# Set up the directories for diagnostics
output_dir = create_directories(_run._id, ex)
# load the data for the data description
data_desc = get_dataset(dataset['name'])
# load images in input_folder
eval_image_paths = load_list_path(input_folder)
# create the network
model = get_model(modelname)
with model(data_description=data_desc.get_data_description(),
output_dir=output_dir,
**net_config) as net:
net.import_weights(filepath=starting_weights)
print("INFO: Imported weights succesfully")
predict_output(net, output_dir, eval_image_paths, data_desc)
def split_test_data(data_config):
# Load the dataset, we expect config to include the arguments
dataset_params = {
key: val
for key, val in data_config.items() if key not in ['dataset']
}
dataset_params['augmentation'] = {
key: False
for key in [
'crop', 'scale', 'vflip', 'hflip', 'gamma', 'rotate', 'shear',
'contrast', 'brightness'
]
}
data = get_dataset(data_config['dataset'])(**dataset_params)
measure_set, test_set = train_test_split(data.testset,
test_size=.5,
random_state=1)
return data, measure_set, test_set
def evaluate(net_config, evaluation_data, modelname, starting_weights, _run):
"""Load weigths from training experiments and evalaute fusion against specified
data."""
data = get_dataset(evaluation_data['dataset'])
model = get_model(modelname)
# now evaluate average mix
with model(data_description=data.get_data_description(),
**net_config) as net:
data = data(**evaluation_data)
import_weights_into_network(net, starting_weights)
measurements, confusion_matrix = net.score(data.get_set_data(test_set))
_run.info['measurements'] = measurements
_run.info['confusion_matrix'] = confusion_matrix
print('Evaluated on {} data:'.format(evaluation_data['dataset']))
print('total accuracy {:.3f} IoU {:.3f}'.format(
measurements['total_accuracy'], measurements['mean_IoU']))
# There seems to be a problem with capturing the print output, flush to be sure
stdout.flush()
def main(dataset, net_config, _run):
# Add all of the config into the helper class
for key in net_config:
setattr(a, key, net_config[key])
setattr(a,'EXP_OUT',EXP_OUT)
setattr(a,'RUN_id',_run._id)
output_dir = create_directories(_run._id, ex)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
# load the dataset class
data = get_dataset(dataset['name'])
data = data(**dataset)
cGAN_model = get_model('cGAN')
if a.checkpoint is not None:
ckp = os.path.join(a.EXP_OUT,str(a.checkpoint))
else:
ckp = None
model = cGAN_model(sess, image_size=a.input_image_size, batch_size=a.batch_size,
output_size=a.input_image_size, dataset_name=dataset['name'],
checkpoint_dir=output_dir, data=data,
data_desc=data.get_data_description(), momentum=a.batch_momentum,
feature_matching=net_config['feature_matching'],
L1_lambda=float(a.l1_weight/a.gan_weight), gf_dim=a.ngf,
df_dim=a.ndf, use_grayscale=net_config['use_grayscale'],
noise_std_dev=a.noise_std_dev,
checkpoint=ckp, gen_type=net_config['type'])
if a.mode == 'train':
tmp = model.train(a)
_run.info['predictions'] = tmp
_run.info['mean_predictions'] = np.mean(tmp, axis=0)
elif a.mode == 'valid':
tmp = model.validate(a)
_run.info['predictions'] = tmp
_run.info['mean_predictions'] = np.mean(tmp, axis=0)
else:
model.transformDatasets(a)
def main(dataset, net_config, _run):
# Add all of the config into the helper class
for key in net_config:
setattr(a, key, net_config[key])
setattr(a, 'EXP_OUT', EXP_OUT)
setattr(a, 'RUN_id', _run._id)
output_dir = create_directories(_run._id, ex)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
# load the dataset class
data = get_dataset(dataset['name'])
data = data(**dataset)
cycleGAN_model = get_model('cycleGAN')
if a.checkpoint is not None:
ckp = os.path.join(a.EXP_OUT, str(a.checkpoint))
else:
ckp = None
model = cycleGAN_model(sess,
image_size=a.input_image_size,
batch_size=a.batch_size,
dataset_name=dataset['name'],
checkpoint_dir=output_dir,
data=data,
data_desc=data.get_data_description(),
checkpoint=ckp)
if a.mode == 'train':
tmp = model.train(a)
# _run.info['predictions'] = tmp
# _run.info['mean_predictions'] = np.mean(tmp, axis=0)
elif a.mode == 'valid':
tmp = model.validate(a)
_run.info['predictions'] = tmp
_run.info['mean_predictions'] = np.mean(tmp, axis=0)
else:
model.transformDatasets(a)
def uncertainty_parameter_search(modelname, net_config, dataset,
starting_weights, search_parameters,
benchmark, uncertainty_metrics, _run):
model = get_model(modelname)
data = get_dataset(dataset['name'])
def evaluation(parameters):
with model(data_description=data.get_data_description(),
**parameters) as net:
measure_set = data(**dataset).get_measureset()
import_weights_into_network(net, starting_weights)
return {
metric: evaluate_uncertainty(net,
measure_set,
metric,
benchmark=benchmark,
print_results=False)
for metric in uncertainty_metrics
}
_run.info['results'] = grid_search(evaluation, search_parameters,
net_config)
def collect_data(net_config, dataset, starting_weights, save_to, _run):
data = get_dataset(**dataset)
model = get_model(net_config['expert_model'])
predictions = {}
for expert in net_config['prefixes']:
model_config = deepcopy(net_config)
model_config['modality'] = expert
model_config['prefix'] = net_config['prefixes'][expert]
with model(data_description=data.get_data_description(),
**model_config) as net:
import_weights_into_network(net, starting_weights[model_config['prefix']])
predictions['measure_%s' % expert] = net.predict(data.get_measureset())
predictions['test_%s' % expert] = net.predict(data.get_testset())
# add also gt labels
predictions['measure_gt'] = data.get_measureset(tf_dataset=False)['labels']
predictions['test_gt'] = data.get_testset(tf_dataset=False)['labels']
# outpath = path.join(save_to, _run._id)
outpath = save_to
if not path.exists(outpath):
mkdir(outpath)
np.savez_compressed(path.join(outpath, 'predictions.npz'), **predictions)
def main(modelname, net_config, gan_config, disc_config, datasetSem,
datasetGAN, datasetDisc, starting_weights, flag_measure, output_mat,
flag_entropy, thresholds, start, _run):
for key in gan_config:
setattr(a, key, gan_config[key])
for key in disc_config:
setattr(b, key, disc_config[key])
setattr(a, 'EXP_OUT', EXP_OUT)
setattr(a, 'RUN_id', _run._id)
setattr(b, 'EXP_OUT', EXP_OUT)
setattr(b, 'RUN_id', _run._id)
disc_data_path = os.path.join(datasetDisc['image_input_dir'],
str(gan_config['checkpoint']) + "_full")
data_id = str(gan_config['checkpoint'])
setattr(b, 'DATA_id', data_id)
# Set up the directories for diagnostics
output_dir = create_directories(_run._id, ex)
# load the data for the data description
data_desc = get_dataset(datasetSem['name'])
model = get_model(modelname)
net = model(data_description=data_desc.get_data_description(),
output_dir=output_dir,
**net_config)
# net.import_weights(filepath=starting_weights)
print("INFO: SemSegNet Imported weights succesfully")
GAN_graph = tf.Graph()
with GAN_graph.as_default():
# create the network
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
GAN_sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
if gan_config['type'] == 'cascRef':
dataGAN = get_dataset('cityscapes_cascGAN')
cGAN_model = get_model('cascGAN')
if a.checkpoint is not None:
ckp = os.path.join(a.EXP_OUT, str(a.checkpoint))
modelGAN = cGAN_model(
GAN_sess,
dataset_name='cityscapes_cascGAN',
image_size=disc_config['input_image_size'],
checkpoint_dir=output_dir,
data_desc=dataGAN.get_data_description(),
is_training=False,
checkpoint=ckp,
vgg_checkpoint=
"/cluster/work/riner/users/haldavid/Checkpoints/VGG_Model/imagenet-vgg-verydeep-19.mat"
)
else:
# load the dataset class
dataGAN = get_dataset(datasetGAN['name'])
# data = data(**datasetGAN)
cGAN_model = get_model('cGAN')
modelGAN = cGAN_model(
GAN_sess,
checkpoint_dir=output_dir,
data_desc=dataGAN.get_data_description(),
feature_matching=gan_config['feature_matching'],
checkpoint=os.path.join(a.EXP_OUT, str(a.checkpoint)),
gen_type=gan_config['type'],
use_grayscale=gan_config['use_grayscale'])
print("INFO: Generative model imported weights succesfully")
Disc_graph = tf.Graph()
with Disc_graph.as_default():
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
sessD = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
dataD = get_dataset(datasetDisc['name'])
dataD = dataD(disc_data_path, **datasetDisc)
disc_model = get_model('simDisc')
disc_checkpoint = None
if disc_config['checkpoint'] is not None:
disc_checkpoint = os.path.join(a.EXP_OUT,
str(disc_config['checkpoint']))
modelDiff = disc_model(sess=sessD,
checkpoint_dir=output_dir,
pos_weight=disc_config['pos_weight'],
data=dataD,
arch=disc_config['arch'],
use_grayscale=disc_config['use_grayscale'],
checkpoint=disc_checkpoint,
use_segm=disc_config['use_segm'],
batch_size=disc_config['batch_size'],
feature_extractor=os.path.join(
a.EXP_OUT, str(a.checkpoint)))
if disc_config['checkpoint'] is None:
print("INFO: Begin training simDisc")
tmp = modelDiff.train(b)
_run.info['simDisc_predictions'] = tmp
_run.info['simDisc_mean_predictions'] = np.mean(tmp, axis=0)
_run.info['simDisc_stdDev'] = np.std(tmp, axis=0)
print("INFO: Finished training simDisc")
else:
print("INFO: Init and loaded checpoint for simDisc")
if flag_measure:
benchmarks = ['measure']
else:
benchmarks = ['wilddash', 'posneg', 'valid', 'measure']
data_SemSeg = data_desc(**datasetSem)
_run.info['thresholds'] = thresholds
###########################################################################
# mapping from Deeplab classes to Adapnet classes
original_labelinfo = {
0: {
'name': 'road',
'mapping': 'road'
},
1: {
'name': 'sidewalk',
'mapping': 'sidewalk'
},
2: {
'name': 'building',
'mapping': 'building'
},
3: {
'name': 'wall',
'mapping': 'building'
},
4: {
'name': 'fence',
'mapping': 'fence'
},
5: {
'name': 'pole',
'mapping': 'pole'
},
6: {
'name': 'traffic light',
'mapping': 'void'
},
7: {
'name': 'traffic sign',
'mapping': 'traffic sign'
},
8: {
'name': 'vegetation',
'mapping': 'vegetation'
},
9: {
'name': 'terrain',
'mapping': 'vegetation'
},
10: {
'name': 'sky',
'mapping': 'sky'
},
11: {
'name': 'person',
'mapping': 'person'
},
12: {
'name': 'rider',
'mapping': 'person'
},
13: {
'name': 'car',
'mapping': 'vehicle'
},
14: {
'name': 'truck',
'mapping': 'vehicle'
},
15: {
'name': 'bus',
'mapping': 'vehicle'
},
16: {
'name': 'train',
'mapping': 'vehicle'
},
17: {
'name': 'motorcycle',
'mapping': 'vehicle'
},
18: {
'name': 'bicycle',
'mapping': 'bicycle'
},
255: {
'name': 'void',
'mapping': 'void'
}
}
labelinfo = {
0: {
'name': 'void',
'color': [0, 0, 0]
},
1: {
'name': 'sky',
'color': [70, 130, 180]
},
2: {
'name': 'building',
'color': [70, 70, 70]
},
3: {
'name': 'road',
'color': [128, 64, 128]
},
4: {
'name': 'sidewalk',
'color': [244, 35, 232]
},
5: {
'name': 'fence',
'color': [190, 153, 153]
},
6: {
'name': 'vegetation',
'color': [107, 142, 35]
},
7: {
'name': 'pole',
'color': [153, 153, 153]
},
8: {
'name': 'vehicle',
'color': [0, 0, 142]
},
9: {
'name': 'traffic sign',
'color': [220, 220, 0]
},
10: {
'name': 'person',
'color': [220, 20, 60]
},
11: {
'name': 'bicycle',
'color': [119, 11, 32]
}
}
label_lookup = [
next(i for i in labelinfo if labelinfo[i]['name'] == k['mapping'])
for _, k in original_labelinfo.items()
]
base_path = path.join(DATA_BASEPATH, 'fishyscapes_newfog')
if 'TMPDIR' in environ:
print('INFO loading dataset into machine ... ')
# first load the zipfile into a closer memory location, then load all the
# images
zip = zipfile.ZipFile(path.join(base_path, 'testset.zip'), 'r')
localtmp = environ['TMPDIR']
zip.extractall(localtmp)
zip.close()
base_path = localtmp
print('DONE loading dataset into machine ... ')
###########################################################################
set_size = 1000
h_orig = 1024
w_orig = 2048
sub_size = 100
semseg_path = "/cluster/work/riner/users/blumh/fishyscapes_deeplab_predictions_newfog"
out_path = "/cluster/work/riner/users/blumh/resultsDH"
for k in range(start, (start + 2)):
kb = k * sub_size
if k > 0:
print('Done %d images' % (kb))
stdout.flush()
img_array = np.zeros((sub_size, 256, 256, 3))
segm_array = np.zeros((sub_size, 256, 256, 3))
for i in range(sub_size):
img = cv2.imread(
path.join(base_path, 'testset',
str(i + kb) + '_rgb.png'))
dl_labels = np.expand_dims(cv2.imread(
path.join(semseg_path,
str(i + kb) + '_predict.png'))[:, :, 0],
axis=0)
cs_labels = np.asarray(label_lookup, dtype='int32')[dl_labels]
lookup = np.array([
labelinfo[i]['color']
for i in range(max(labelinfo.keys()) + 1)
]).astype(int)
segm = np.array(lookup[cs_labels[:]]).astype('uint8')[..., ::-1]
#mask = cv2.imread(path.join(base_path, str(i)+'_mask.png'), cv2.IMREAD_ANYDEPTH)
# blob['labels'] = cv2.imread(labels_filename, cv2.IMREAD_ANYDEPTH)
# # apply label mapping
# blob['labels'] = np.asarray(self.label_lookup, dtype='int32')[blob['labels']]
img_array[i, ...] = cv2.resize(img, (256, 256),
interpolation=cv2.INTER_LINEAR)
segm_array[i, ...] = cv2.resize(segm[0, ...], (256, 256),
interpolation=cv2.INTER_NEAREST)
with GAN_sess.as_default():
with GAN_graph.as_default():
synth_images = modelGAN.transform(a, segm_array)
with sessD.as_default():
with Disc_graph.as_default():
simMat = modelDiff.transform(img_array, synth_images,
segm_array)
for i in range(sub_size):
# filename = path.join(out_path,str(i+kb)+'_rgb.png')
# cv2.imwrite(filename,cv2.resize(img_array[i,...], (2048, 1024),interpolation=cv2.INTER_LINEAR))
# filename = path.join(out_path,str(i+kb)+'_segm.png')
# cv2.imwrite(filename,cv2.resize(segm_array[i,...], (2048, 1024),interpolation=cv2.INTER_NEAREST))
filename = path.join(out_path, str(i + kb) + '_dissim.png')
cv2.imwrite(filename, simMat[i, ...])
filename = path.join(out_path, str(i + kb) + '_dissim.npy')
np.save(
filename,
cv2.resize(simMat[i, ...], (2048, 1024),
interpolation=cv2.INTER_LINEAR))
def main(modelname, net_config, gan_config, disc_config, datasetSem,
datasetGAN, datasetDisc, starting_weights, flag_measure, output_mat,
flag_entropy, thresholds, start, _run):
for key in gan_config:
setattr(a, key, gan_config[key])
for key in disc_config:
setattr(b, key, disc_config[key])
setattr(a, 'EXP_OUT', EXP_OUT)
setattr(a, 'RUN_id', _run._id)
setattr(b, 'EXP_OUT', EXP_OUT)
setattr(b, 'RUN_id', _run._id)
disc_data_path = os.path.join(datasetDisc['image_input_dir'],
str(gan_config['checkpoint']) + "_full")
data_id = str(gan_config['checkpoint'])
setattr(b, 'DATA_id', data_id)
# Set up the directories for diagnostics
output_dir = create_directories(_run._id, ex)
# load the data for the data description
data_desc = get_dataset(datasetSem['name'])
model = get_model(modelname)
net = model(data_description=data_desc.get_data_description(),
output_dir=output_dir,
**net_config)
net.import_weights(filepath=starting_weights)
print("INFO: SemSegNet Imported weights succesfully")
GAN_graph = tf.Graph()
with GAN_graph.as_default():
# create the network
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
GAN_sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
if gan_config['type'] == 'cascRef':
dataGAN = get_dataset('cityscapes_cascGAN')
cGAN_model = get_model('cascGAN')
if a.checkpoint is not None:
ckp = os.path.join(a.EXP_OUT, str(a.checkpoint))
modelGAN = cGAN_model(
GAN_sess,
dataset_name='cityscapes_cascGAN',
image_size=disc_config['input_image_size'],
checkpoint_dir=output_dir,
data_desc=dataGAN.get_data_description(),
is_training=False,
checkpoint=ckp,
vgg_checkpoint=
"/cluster/work/riner/users/haldavid/Checkpoints/VGG_Model/imagenet-vgg-verydeep-19.mat"
)
else:
# load the dataset class
dataGAN = get_dataset(datasetGAN['name'])
# data = data(**datasetGAN)
cGAN_model = get_model('cGAN')
modelGAN = cGAN_model(
GAN_sess,
checkpoint_dir=output_dir,
data_desc=dataGAN.get_data_description(),
feature_matching=gan_config['feature_matching'],
checkpoint=os.path.join(a.EXP_OUT, str(a.checkpoint)),
gen_type=gan_config['type'],
use_grayscale=gan_config['use_grayscale'])
print("INFO: Generative model imported weights succesfully")
Disc_graph = tf.Graph()
with Disc_graph.as_default():
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
sessD = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
dataD = get_dataset(datasetDisc['name'])
dataD = dataD(disc_data_path, **datasetDisc)
disc_model = get_model('simDisc')
disc_checkpoint = None
if disc_config['checkpoint'] is not None:
disc_checkpoint = os.path.join(a.EXP_OUT,
str(disc_config['checkpoint']))
modelDiff = disc_model(sess=sessD,
checkpoint_dir=output_dir,
pos_weight=disc_config['pos_weight'],
data=dataD,
arch=disc_config['arch'],
use_grayscale=disc_config['use_grayscale'],
checkpoint=disc_checkpoint,
use_segm=disc_config['use_segm'],
batch_size=disc_config['batch_size'],
feature_extractor=os.path.join(
a.EXP_OUT, str(a.checkpoint)))
if disc_config['checkpoint'] is None:
print("INFO: Begin training simDisc")
tmp = modelDiff.train(b)
_run.info['simDisc_predictions'] = tmp
_run.info['simDisc_mean_predictions'] = np.mean(tmp, axis=0)
_run.info['simDisc_stdDev'] = np.std(tmp, axis=0)
print("INFO: Finished training simDisc")
else:
print("INFO: Init and loaded checpoint for simDisc")
if flag_measure:
benchmarks = ['measure']
else:
benchmarks = ['wilddash', 'posneg', 'valid', 'measure']
data_SemSeg = data_desc(**datasetSem)
# thresholds = [0.2,0.4,0.6,0.8]
# thresholds = [0.85,0.9,0.95,0.99]
_run.info['thresholds'] = thresholds
for set in benchmarks:
if set == "measure":
dataset = data_SemSeg.get_measureset(tf_dataset=False)
elif set == "valid":
dataset = data_SemSeg.get_validation_set(tf_dataset=False)
else:
data = get_dataset(set)
head, _ = os.path.split(datasetDisc['image_input_dir'])
data = data(os.path.join(head, set))
dataset = data.get_validation_set(tf_dataset=False)
sem_seg_images, rgb_images, masks, gt_seg_images, seg_seg_labels, output_probs = predict_output(
net, output_dir, dataset, data_SemSeg, flag_entropy,
net_config['num_classes'])
with GAN_sess.as_default():
with GAN_graph.as_default():
if gan_config['type'] == 'cascRef':
synth_images = modelGAN.transform(a, seg_seg_labels)
else:
synth_images = modelGAN.transform(a, sem_seg_images)
with sessD.as_default():
with Disc_graph.as_default():
simMat = modelDiff.transform(rgb_images, synth_images,
sem_seg_images)
############################################################################################
# Perhaps need to also give number of patches per dimension.
simMatSSIM = computePatchSSIM(rgb_images, synth_images,
datasetDisc['ppd'])
############################################################################################
temp_iou = computeIOU(simMat, masks, thresholds)
temp_pr = computePRvalues(simMat, masks, thresholds)
_run.info[set + '_IOU'] = temp_iou
_run.info[set + '_PRvals'] = temp_pr
_run.info[set + '_F1score'] = 2 * np.asarray(temp_pr[1]) * np.asarray(
temp_pr[2]) / (np.asarray(temp_pr[1]) + np.asarray(temp_pr[2]))
# _run.info[set+'_SSIM_IOU'] = computeIOU(simMatSSIM, masks, thresholds)
# _run.info[set+'_SSIM_PRvals'] = computePRvalues(simMatSSIM, masks)
if flag_entropy and set is not 'posneg':
entropy = ShannonEntropy(output_probs)
# _run.info[set+'_meanVarEntropyOoD'] = [np.mean(entropy[masks.astype(bool)]),np.var(entropy[masks.astype(bool)],ddof=1)]
# _run.info[set+'_meanVarEntropyID'] = [np.mean(entropy[~masks.astype(bool)]),np.var(entropy[~masks.astype(bool)],ddof=1)]
temp_iou = computeIOU(entropy, masks, thresholds)
temp_pr = computePRvalues(entropy, masks, thresholds)
_run.info[set + '_entropy_IOU'] = temp_iou
_run.info[set + '_entropy_PRvals'] = temp_pr
_run.info[set + '_entropy_F1score'] = 2 * np.asarray(
temp_pr[1]) * np.asarray(temp_pr[2]) / (
np.asarray(temp_pr[1]) + np.asarray(temp_pr[2]))
k = masks.shape[0]
if output_mat and set is not 'posneg':
if not os.path.exists(os.path.join(output_dir, set)):
os.makedirs(os.path.join(output_dir, set))
if set is 'measure':
k = 25
matrix_path = os.path.join(output_dir, set, "mskMat.npy")
np.save(matrix_path, masks[0:k, ...])
matrix_path = os.path.join(output_dir, set, "simMat.npy")
np.save(matrix_path, simMat[0:k, ...])
# matrix_path = os.path.join(output_dir,set,"ssmMat.npy")
# np.save(matrix_path, simMatSSIM[0:k, ...])
matrix_path = os.path.join(output_dir, set, "rgbMat.npy")
np.save(matrix_path, rgb_images[0:k, ...])
matrix_path = os.path.join(output_dir, set, "synMat.npy")
np.save(matrix_path, synth_images[0:k, ...])
matrix_path = os.path.join(output_dir, set, "semMat.npy")
np.save(matrix_path, sem_seg_images[0:k, ...])
matrix_path = os.path.join(output_dir, set, "gtsMat.npy")
np.save(matrix_path, gt_seg_images[0:k, ...])
if flag_entropy:
matrix_path = os.path.join(output_dir, set, "entMat.npy")
np.save(matrix_path, entropy)
def main(modelname, net_config, gan_config, disc_config, datasetSem,
datasetGAN, datasetDisc, starting_weights, flag_measure, output_mat,
flag_entropy, thresholds, start, _run):
for key in gan_config:
setattr(a, key, gan_config[key])
for key in disc_config:
setattr(b, key, disc_config[key])
setattr(a, 'EXP_OUT', EXP_OUT)
setattr(a, 'RUN_id', _run._id)
setattr(b, 'EXP_OUT', EXP_OUT)
setattr(b, 'RUN_id', _run._id)
disc_data_path = os.path.join(datasetDisc['image_input_dir'],
str(gan_config['checkpoint']) + "_full")
data_id = str(gan_config['checkpoint'])
setattr(b, 'DATA_id', data_id)
# Set up the directories for diagnostics
output_dir = create_directories(_run._id, ex)
# load the data for the data description
data_desc = get_dataset(datasetSem['name'])
model = get_model(modelname)
net = model(data_description=data_desc.get_data_description(),
output_dir=output_dir,
**net_config)
# net.import_weights(filepath=starting_weights)
print("INFO: SemSegNet Imported weights succesfully")
GAN_graph = tf.Graph()
with GAN_graph.as_default():
# create the network
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
GAN_sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
if gan_config['type'] == 'cascRef':
dataGAN = get_dataset('cityscapes_cascGAN')
cGAN_model = get_model('cascGAN')
if a.checkpoint is not None:
ckp = os.path.join(a.EXP_OUT, str(a.checkpoint))
modelGAN = cGAN_model(
GAN_sess,
dataset_name='cityscapes_cascGAN',
image_size=disc_config['input_image_size'],
checkpoint_dir=output_dir,
data_desc=dataGAN.get_data_description(),
is_training=False,
checkpoint=ckp,
vgg_checkpoint=
"/cluster/work/riner/users/haldavid/Checkpoints/VGG_Model/imagenet-vgg-verydeep-19.mat"
)
else:
# load the dataset class
dataGAN = get_dataset(datasetGAN['name'])
# data = data(**datasetGAN)
cGAN_model = get_model('cGAN')
modelGAN = cGAN_model(
GAN_sess,
checkpoint_dir=output_dir,
data_desc=dataGAN.get_data_description(),
feature_matching=gan_config['feature_matching'],
checkpoint=os.path.join(a.EXP_OUT, str(a.checkpoint)),
gen_type=gan_config['type'],
use_grayscale=gan_config['use_grayscale'])
print("INFO: Generative model imported weights succesfully")
Disc_graph = tf.Graph()
with Disc_graph.as_default():
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
sessD = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
dataD = get_dataset(datasetDisc['name'])
dataD = dataD(disc_data_path, **datasetDisc)
disc_model = get_model('simDisc')
disc_checkpoint = None
if disc_config['checkpoint'] is not None:
disc_checkpoint = os.path.join(a.EXP_OUT,
str(disc_config['checkpoint']))
modelDiff = disc_model(sess=sessD,
checkpoint_dir=output_dir,
pos_weight=disc_config['pos_weight'],
data=dataD,
arch=disc_config['arch'],
use_grayscale=disc_config['use_grayscale'],
checkpoint=disc_checkpoint,
use_segm=disc_config['use_segm'],
batch_size=disc_config['batch_size'],
feature_extractor=os.path.join(
a.EXP_OUT, str(a.checkpoint)))
if disc_config['checkpoint'] is None:
print("INFO: Begin training simDisc")
tmp = modelDiff.train(b)
_run.info['simDisc_predictions'] = tmp
_run.info['simDisc_mean_predictions'] = np.mean(tmp, axis=0)
_run.info['simDisc_stdDev'] = np.std(tmp, axis=0)
print("INFO: Finished training simDisc")
else:
print("INFO: Init and loaded checpoint for simDisc")
if flag_measure:
benchmarks = ['measure']
else:
benchmarks = ['wilddash', 'posneg', 'valid', 'measure']
data_SemSeg = data_desc(**datasetSem)
_run.info['thresholds'] = thresholds
###########################################################################
# mapping from Mapillary classes to Deeplab classes
if start == 0:
with open('/cluster/work/riner/users/haldavid/config.json') as f:
config = json.load(f)
else:
with open(
'/Volumes/Netti HD /Master Thesis/mapillary/config.json') as f:
config = json.load(f)
def map_class(name):
"""Map a Mapillary class onto one of the 19 cityscapes classes or void."""
direct_mapping = {
'construction--barrier--fence': 'fence',
'construction--barrier--wall': 'wall',
'construction--flat--road': 'road',
'construction--flat--sidewalk': 'sidewalk',
'construction--structure--building': 'building',
'human--person': 'person',
'nature--sky': 'sky',
'nature--terrain': 'terrain',
'nature--vegetation': 'vegetation',
'object--support--pole': 'pole',
'object--support--utility-pole': 'pole',
'object--traffic-light': 'traffic light',
'object--traffic-sign--front': 'traffic sign',
'object--vehicle--bicycle': 'bicycle',
'object--vehicle--bus': 'bus',
'object--vehicle--car': 'car',
'object--vehicle--motorcycle': 'motorcycle',
'object--vehicle--on-rails': 'train',
'object--vehicle--truck': 'truck',
}
if name in direct_mapping:
return direct_mapping[name]
elif name.startswith('human--rider'):
return 'rider'
elif name.startswith('marking'):
return 'road'
else:
return 'void'
original_labels_mapi = {
i: {
'name': v['name'],
'color': v['color'],
'mapping': map_class(v['name'])
}
for i, v in enumerate(config['labels'])
}
# array to look up the label_id of a given color
color_map = np.ndarray(shape=(256**3), dtype='int32')
color_map[:] = -1
for c, v in original_labels_mapi.items():
rgb = v['color']
rgb = rgb[0] * 65536 + rgb[1] * 256 + rgb[2]
color_map[rgb] = c
# apply same label mapping as for original cityscapes
labelinfo_mapi = {
-1: {
'name': 'void',
'color': [0, 0, 0]
},
0: {
'name': 'road',
'color': [128, 64, 128]
},
1: {
'name': 'sidewalk',
'color': [244, 35, 232]
},
2: {
'name': 'building',
'color': [70, 70, 70]
},
3: {
'name': 'wall',
'color': [70, 70, 70]
},
4: {
'name': 'fence',
'color': [190, 153, 153]
},
5: {
'name': 'pole',
'color': [153, 153, 153]
},
6: {
'name': 'traffic light',
'color': [0, 0, 0]
},
7: {
'name': 'traffic sign',
'color': [220, 220, 0]
},
8: {
'name': 'vegetation',
'color': [107, 142, 35]
},
9: {
'name': 'terrain',
'color': [107, 142, 35]
},
10: {
'name': 'sky',
'color': [70, 130, 180]
},
11: {
'name': 'person',
'color': [220, 20, 60]
},
12: {
'name': 'rider',
'color': [220, 20, 60]
},
13: {
'name': 'car',
'color': [0, 0, 142]
},
14: {
'name': 'truck',
'color': [0, 0, 142]
},
15: {
'name': 'bus',
'color': [0, 0, 142]
},
16: {
'name': 'train',
'color': [0, 0, 142]
},
17: {
'name': 'motorcycle',
'color': [0, 0, 142]
},
18: {
'name': 'bicycle',
'color': [119, 11, 32]
}
}
label_lookup_mapi = [
next(i for i in labelinfo_mapi
if labelinfo_mapi[i]['name'] == v['mapping'])
for v in original_labels_mapi.values()
]
lookup_mapi = np.array([
labelinfo_mapi[i]['color']
for i in range(max(labelinfo_mapi.keys()) + 1)
]).astype(int)
###########################################################################
# mapping from Deeplab classes to Adapnet classes
original_labelinfo_dl = {
0: {
'name': 'road',
'mapping': 'road'
},
1: {
'name': 'sidewalk',
'mapping': 'sidewalk'
},
2: {
'name': 'building',
'mapping': 'building'
},
3: {
'name': 'wall',
'mapping': 'building'
},
4: {
'name': 'fence',
'mapping': 'fence'
},
5: {
'name': 'pole',
'mapping': 'pole'
},
6: {
'name': 'traffic light',
'mapping': 'void'
},
7: {
'name': 'traffic sign',
'mapping': 'traffic sign'
},
8: {
'name': 'vegetation',
'mapping': 'vegetation'
},
9: {
'name': 'terrain',
'mapping': 'vegetation'
},
10: {
'name': 'sky',
'mapping': 'sky'
},
11: {
'name': 'person',
'mapping': 'person'
},
12: {
'name': 'rider',
'mapping': 'person'
},
13: {
'name': 'car',
'mapping': 'vehicle'
},
14: {
'name': 'truck',
'mapping': 'vehicle'
},
15: {
'name': 'bus',
'mapping': 'vehicle'
},
16: {
'name': 'train',
'mapping': 'vehicle'
},
17: {
'name': 'motorcycle',
'mapping': 'vehicle'
},
18: {
'name': 'bicycle',
'mapping': 'bicycle'
},
255: {
'name': 'void',
'mapping': 'void'
}
}
labelinfo = {
0: {
'name': 'void',
'color': [0, 0, 0]
},
1: {
'name': 'sky',
'color': [70, 130, 180]
},
2: {
'name': 'building',
'color': [70, 70, 70]
},
3: {
'name': 'road',
'color': [128, 64, 128]
},
4: {
'name': 'sidewalk',
'color': [244, 35, 232]
},
5: {
'name': 'fence',
'color': [190, 153, 153]
},
6: {
'name': 'vegetation',
'color': [107, 142, 35]
},
7: {
'name': 'pole',
'color': [153, 153, 153]
},
8: {
'name': 'vehicle',
'color': [0, 0, 142]
},
9: {
'name': 'traffic sign',
'color': [220, 220, 0]
},
10: {
'name': 'person',
'color': [220, 20, 60]
},
11: {
'name': 'bicycle',
'color': [119, 11, 32]
}
}
label_lookup_dl = [
next(i for i in labelinfo if labelinfo[i]['name'] == k['mapping'])
for _, k in original_labelinfo_dl.items()
]
lookup = np.array([
labelinfo[i]['color'] for i in range(max(labelinfo.keys()) + 1)
]).astype(int)
# base_path = path.join(DATA_BASEPATH, 'fishyscapes_newfog')
# if 'TMPDIR' in environ:
# print('INFO loading dataset into machine ... ')
# # first load the zipfile into a closer memory location, then load all the
# # images
# zip = zipfile.ZipFile(path.join(base_path, 'testset.zip'), 'r')
# localtmp = environ['TMPDIR']
# zip.extractall(localtmp)
# zip.close()
# base_path = localtmp
#
# print('DONE loading dataset into machine ... ')
###########################################################################
sub_size = 100
if start == 0:
input_path = "/cluster/work/riner/users/blumh/mapillary_evaluation_set"
out_path = path.join(
"/cluster/work/riner/users/haldavid/MapillaryResultsCropped",
disc_config['arch'])
else:
input_path = "/Users/David/Downloads/mapillary_evaluation_set"
out_path = path.join("/Users/David/Desktop/out", disc_config['arch'])
if not os.path.exists(out_path):
os.makedirs(out_path)
#N H W C
img_array = np.zeros((sub_size, 256, 256, 3))
segm_array = np.zeros((sub_size, 256, 256, 3))
gt_array = np.zeros((sub_size, 256, 256, 3))
mask_array = np.zeros((sub_size, 256, 256))
out_mask_array = np.zeros((sub_size, 256, 256))
for i in range(sub_size):
#RGB
img = cv2.imread(path.join(input_path, str(i) + '_rgb.png'))
#
dl_labels = np.expand_dims(cv2.imread(
path.join(input_path,
str(i) + '_predict.png'))[:, :, 0],
axis=0)
cs_labels = np.asarray(label_lookup_dl, dtype='int32')[dl_labels]
segm = np.array(lookup[cs_labels[:]]).astype('uint8')[..., ::-1]
gt_labels = cv2.imread(path.join(input_path,
str(i) + '_segm.png'))[..., ::-1]
gt_labels = gt_labels.dot(np.array([65536, 256, 1], dtype='int32'))
gt_segm = color_map[gt_labels]
# apply mapping
gt_segm = np.asarray(label_lookup_mapi, dtype='int32')[gt_segm]
gt_segm = np.array(lookup_mapi[gt_segm[:]]).astype('uint8')[..., ::-1]
mask = cv2.imread(path.join(input_path, str(i) + '_mask.png'))[..., 0]
#Crop all in the same manor 4:3 we crop to 4:2
crop_margin = int(img.shape[0] / 6)
img = img[crop_margin:-crop_margin, ...]
segm = segm[0, crop_margin:-crop_margin, ...]
gt_segm = gt_segm[crop_margin:-crop_margin, ...]
mask = mask[crop_margin:-crop_margin, ...]
img_array[i, ...] = cv2.resize(img, (256, 256),
interpolation=cv2.INTER_LINEAR)
segm_array[i, ...] = cv2.resize(segm, (256, 256),
interpolation=cv2.INTER_NEAREST)
gt_array[i, ...] = cv2.resize(gt_segm, (256, 256),
interpolation=cv2.INTER_NEAREST)
mask_array[i, ...] = cv2.resize(mask, (256, 256),
interpolation=cv2.INTER_NEAREST)
with GAN_sess.as_default():
with GAN_graph.as_default():
synth_images, pred_descriminator = modelGAN.transform_withD(
a, segm_array, img_array)
with sessD.as_default():
with Disc_graph.as_default():
simMat = modelDiff.transform(img_array, synth_images, segm_array)
for i in range(sub_size):
gt_mask = error_mask(segm_array[i], gt_array[i])
out_mask = np.logical_or(gt_mask, mask_array[i])
out_mask_array[i, ...] = out_mask
# filename = path.join(out_path,str(i)+'_dissim.npy')
# np.save(filename,cv2.resize(simMat[i,...], (2048, 1024),interpolation=cv2.INTER_LINEAR))
# if disc_config['arch']=='arch13':
# filename = path.join(out_path,str(i)+'_discsim.npy')
# np.save(filename,cv2.resize(pred_descriminator[i,...], (2048, 1024),interpolation=cv2.INTER_NEAREST))
# filename = path.join(out_path,str(i)+'_mask.npy')
# np.save(filename,cv2.resize(out_mask, (2048, 1024),interpolation=cv2.INTER_NEAREST))
# filename = path.join(out_path,str(i)+'_rgb.npy')
# np.save(filename,cv2.resize(img_array[i,...], (2048, 1024),interpolation=cv2.INTER_LINEAR))
filename = path.join(out_path, str(i) + '_dissim.npy')
np.save(filename, simMat)
if disc_config['arch'] == 'arch13':
filename = path.join(out_path, str(i) + '_discsim.npy')
np.save(filename, pred_descriminator)
filename = path.join(out_path, str(i) + '_mask.npy')
np.save(filename, out_mask_array)
filename = path.join(out_path, str(i) + '_rgb.npy')
np.save(filename, img_array)
filename = path.join(out_path, str(i) + '_synth.npy')
np.save(filename, synth_images)
filename = path.join(out_path, str(i) + '_segm.npy')
np.save(filename, segm_array)
filename = path.join(out_path, str(i) + '_gt.npy')
np.save(filename, gt_array)
def main(dataset, net_config, _run):
# Add all of the config into the helper class
for key in net_config:
setattr(a, key, net_config[key])
output_dir = create_directories(_run._id, ex)
# load the dataset class
data = get_dataset(dataset['name'])
data = data(**dataset)
# Get the data descriptors with the shape of data coming
data_description = data.get_data_description()
data_description = [data_description[0], {
key: [None, *description]
for key, description in data_description[1].items()}]
# Create an iterator for the data
iter_handle = tf.placeholder(tf.string, shape=[],
name='training_placeholder')
iterator = tf.data.Iterator.from_string_handle(
iter_handle, *data_description)
training_batch = iterator.get_next()
# Create a placeholder for if one is training and a Variable
# for the flipping probability
a.is_train = tf.placeholder(tf.bool, name="is_train")
a.decay_term = np.exp(-a.flip_lambda) # x = e^-lambda where N(t)=N(0)*x^t
a.flip_prob = tf.Variable(a.flip_label_init_prob*a.flip_decay_perc, name="flip_prob")
a.flip_end_prob = a.flip_label_init_prob*(1-a.flip_decay_perc)
# Create pix2pix model
model = create_model(training_batch['labels'], training_batch['rgb'])
# Define fetches for evaluation such that these are returned by the graph
with tf.name_scope("encode_images"):
if a.val_target_output == True:
display_fetches = {
"targets": model.targets,
"outputs": model.outputs
}
else:
display_fetches = {
"outputs": model.outputs
}
saver = tf.train.Saver()
# Limit GPU fraction
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=0.9)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
sess.run(tf.global_variables_initializer())
# There are local variables in the metrics
sess.run(tf.local_variables_initializer())
# Add the images to the summary
tf.summary.image("input",model.inputs[...,::-1])
tf.summary.image("output",model.outputs[...,::-1])
tf.summary.image("target",model.targets[...,::-1])
# Add variable values to summary
for var in tf.trainable_variables():
tf.summary.histogram(var.op.name + "/values", var)
# Add gradients values to summary
for grad, var in model.discrim_grads_and_vars + model.gen_grads_and_vars:
tf.summary.histogram(var.op.name + "/gradients", grad)
# Merge all summaries to one instance
merged_summary = tf.summary.merge_all()
# Start tensorflow summary filewriter
if output_dir is not None:
train_writer = tf.summary.FileWriter(output_dir)
train_writer.add_graph(sess.graph)
if a.mode=="train":
print("INFO: Got train set")
input_data = data.get_trainset()
data_iterator = input_data.repeat(a.max_epochs).batch(a.batch_size).make_one_shot_iterator()
# data_iterator = input_data.batch(a.batch_size).make_initializable_iterator()
validation_data = data.get_validation_set()
valid_iterator = validation_data.batch(a.batch_size).make_one_shot_iterator()
valid_handle = sess.run(valid_iterator.string_handle())
else:
print("INFO: Got test set")
input_data = data.get_testset(num_items=num_test_images)
data_iterator = input_data.batch(a.batch_size).make_one_shot_iterator()
data_handle = sess.run(data_iterator.string_handle())
if a.checkpoint is not None:
print("loading model from checkpoint")
checkpoint = tf.train.latest_checkpoint(os.path.join(EXP_OUT,str(a.checkpoint)))
saver.restore(sess, checkpoint)
if a.mode=="test":
print("INFO: Starting Test")
for i in range(0,num_test_images):
results=sess.run(display_fetches, feed_dict={iter_handle: data_handle})
filename = "output" + str(i) + ".png"
cv2.imwrite(os.path.join(a.file_output_dir,filename), (results['outputs'][0,:,:,:]), [int(cv2.IMWRITE_JPEG_QUALITY), 90])
else:
print('INFO: Start training')
stdout.flush()
start = time.time()
# for k in range(1,a.max_epochs+1):
for k in range(1,2):
i = 1
while True:
# Makes the graph run through the complete graph from beginning
# to end
fetches ={
"train": model.train,
}
if i%a.num_print == 0:
fetches["discrim_loss"] = model.discrim_loss
fetches["gen_loss_GAN"] = model.gen_loss_GAN
fetches["gen_loss_L1"] = model.gen_loss_L1
fetches["outputs"] = model.outputs
fetches["inputs"] = model.inputs
fetches["targets"] = model.targets
fetches["predict_fake"] = model.predict_fake
fetches["predict_real"] = model.predict_real
try:
sum, results=sess.run([merged_summary, fetches],
feed_dict={iter_handle: data_handle,
a.is_train: True})
except tf.errors.OutOfRangeError:
print("INFO: Training set has %d elements, starting next epoch" % i)
break
rate = (i) / (time.time() - start)
ssim_score = compare_ssim(cv2.cvtColor(results["targets"][0,:,:,:], cv2.COLOR_BGR2GRAY),cv2.cvtColor(results["outputs"][0,:,:,:], cv2.COLOR_BGR2GRAY), data_range=255)
print ('Epoch %d\tStep %d: D_Loss: %f, \tG_Loss: %f, \tL1_Loss: %f, \tSSIM_Score: %f, \tRate: %f img/sec, \tglobal_step: %d' % (k,i,results['discrim_loss'],results['gen_loss_GAN'],results['gen_loss_L1'],ssim_score,rate,tf.train.global_step(sess, model.global_step)))
stdout.flush()
d_loss = tf.Summary(
value=[tf.Summary.Value(tag='discriminator_loss',
simple_value=results['discrim_loss'])])
g_loss = tf.Summary(
value=[tf.Summary.Value(tag='generator_loss_GAN',
simple_value=results['gen_loss_GAN'])])
l_loss = tf.Summary(
value=[tf.Summary.Value(tag='generator_loss_L1',
simple_value=results['gen_loss_L1'])])
ssim = tf.Summary(
value=[tf.Summary.Value(tag='SSIM_score',
simple_value=ssim_score)])
p_fake = tf.Summary(
value=[tf.Summary.Value(tag='predict_fake_values',
simple_value=np.mean(results['predict_fake']))])
p_real = tf.Summary(
value=[tf.Summary.Value(tag='predict_real_values',
simple_value=np.mean(results['predict_real']))])
train_writer.add_summary(d_loss, tf.train.global_step(sess, model.global_step))
train_writer.add_summary(g_loss, tf.train.global_step(sess, model.global_step))
train_writer.add_summary(l_loss, tf.train.global_step(sess, model.global_step))
train_writer.add_summary(ssim, tf.train.global_step(sess, model.global_step))
train_writer.add_summary(p_fake, tf.train.global_step(sess, model.global_step))
train_writer.add_summary(p_real, tf.train.global_step(sess, model.global_step))
train_writer.add_summary(sum, tf.train.global_step(sess, model.global_step))
else:
try:
results=sess.run(fetches,
feed_dict={iter_handle: data_handle,
a.is_train: True})
except tf.errors.OutOfRangeError:
print("INFO: Training set has %d elements, starting next epoch" % (i-1))
break
i=i+1
print("INFO: Saving model at: "+ output_dir)
saver.save(sess, os.path.join(output_dir, "model"))
print('INFO: Training finished.')
stdout.flush()
print('INFO: Starting Validation')
i=0
fake_values = np.zeros(15)
real_values = np.zeros(15)
while True:
fetches ={
"predict_fake": model.predict_fake,
"predict_real": model.predict_real
}
try:
results, preds = sess.run([display_fetches, fetches],
feed_dict={iter_handle: valid_handle,
a.is_train: False})
except tf.errors.OutOfRangeError:
print("INFO: Finished validation of %d images after training" % i)
break
fake_values[i] = (np.mean(preds['predict_fake']))
real_values[i] = (np.mean(preds['predict_real']))
filename = str(_run._id) + "_validation" + str(i+1) + ".png"
cv2.imwrite(os.path.join(a.file_output_dir,filename), (results['outputs'][0,:,:,:]), [int(cv2.IMWRITE_JPEG_QUALITY), 90])
if a.val_target_output == True:
filename = str(_run._id) + "_target" + str(i+1) + ".png"
cv2.imwrite(os.path.join(a.file_output_dir,filename), (results['targets'][0,:,:,:]), [int(cv2.IMWRITE_JPEG_QUALITY), 90])
i=i+1
_run.info['predictions_fake'] = fake_values
_run.info['predictions_real'] = real_values
if output_dir is not None:
train_writer.close()
def main(modelname, net_config, gan_config, disc_config, datasetSem,
datasetGAN, datasetDisc, starting_weights, input_folder, sets, _run):
for key in gan_config:
setattr(a, key, gan_config[key])
setattr(a, 'EXP_OUT', EXP_OUT)
setattr(a, 'RUN_id', _run._id)
# Set up the directories for diagnostics
output_dir = create_directories(_run._id, ex)
# load the data for the data description
data_desc = get_dataset(datasetSem['name'])
dataFlag = (input_folder is not None)
if dataFlag:
# load images in input_folder
eval_image_paths = load_list_path(input_folder)
elif sets == "measure":
data = data_desc(**datasetSem)
dataset = data.get_measureset(tf_dataset=False)
else:
data = data_desc(**datasetSem)
dataset = data.get_validation_set(tf_dataset=False)
model = get_model(modelname)
net = model(data_description=data_desc.get_data_description(),
output_dir=output_dir,
**net_config)
net.import_weights(filepath=starting_weights)
print("INFO: Imported weights succesfully")
GAN_graph = tf.Graph()
with GAN_graph.as_default():
# create the network
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
GAN_sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
# load the dataset class
dataGAN = get_dataset(datasetGAN['name'])
# data = data(**datasetGAN)
cGAN_model = get_model('cGAN')
modelGAN = cGAN_model(GAN_sess,
checkpoint_dir=output_dir,
data_desc=dataGAN.get_data_description(),
checkpoint=os.path.join(a.EXP_OUT,
str(a.checkpoint)))
print("INFO: GAN Imported weights succesfully")
# tf.reset_default_graph()
Disc_graph = tf.Graph()
with Disc_graph.as_default():
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
sessD = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
dataD = get_dataset(datasetDisc['name'])
dataD = dataD(datasetDisc['image_input_dir'], **datasetDisc)
disc_model = get_model('simDisc')
modelDiff = disc_model(sess=sessD,
checkpoint_dir=output_dir,
data=dataD,
is_training=False,
checkpoint=os.path.join(
a.EXP_OUT, str(disc_config['checkpoint'])))
print("INFO: Disc Imported weights succesfully")
ss_run_id = starting_weights.split('/')[-2]
gan_run_id = str(a.checkpoint)
folder_name = ss_run_id + "_" + gan_run_id + "_" + str(
disc_config['checkpoint']) + "_" + sets
base_output_path = os.path.join(a.file_output_dir, folder_name)
if not os.path.exists(base_output_path):
os.makedirs(base_output_path)
if dataFlag:
sem_seg_images, rgb_images = predict_output(net,
output_dir,
eval_image_paths,
data_desc,
dataFlag=dataFlag)
else:
sem_seg_images, rgb_images, masks, sem_seg_GT = predict_output(
net, output_dir, dataset, data, dataFlag=dataFlag)
matrix_path = os.path.join(base_output_path, "mskMat.npy")
np.save(matrix_path, masks)
matrix_path = os.path.join(base_output_path, "gtsMat.npy")
np.save(matrix_path, sem_seg_GT)
print("Done with prediction of semantic segmentation")
with GAN_sess.as_default():
with GAN_graph.as_default():
synth_images = modelGAN.transform(a, sem_seg_images)
print("Done with prediction of GAN")
with sessD.as_default():
with Disc_graph.as_default():
simMat = modelDiff.transform(rgb_images, synth_images,
sem_seg_images)
matrix_path = os.path.join(base_output_path, "simMat.npy")
np.save(matrix_path, simMat)
matrix_path = os.path.join(base_output_path, "rgbMat.npy")
np.save(matrix_path, rgb_images)
matrix_path = os.path.join(base_output_path, "synMat.npy")
np.save(matrix_path, synth_images)
matrix_path = os.path.join(base_output_path, "semMat.npy")
np.save(matrix_path, sem_seg_images)
