def scatterplot_data(nip_models, camera, root_dir='./data/raw/train_manipulation'): nip_models = nip_models if type(nip_models) is list else [nip_models] if camera is None: print('! warning: camera not specified- using first available one!') camera = camera or coreutils.listdir(root_dir, '.', dirs_only=True)[0] df = pd.DataFrame(columns=['camera', 'nip', 'lr', 'source', 'psnr', 'ssim', 'accuracy']) for nip in nip_models: find_dir = os.path.join(root_dir, camera, nip) experiment_dirs = coreutils.listdir(os.path.join(find_dir), 'lr-.*', dirs_only=True) for ed in experiment_dirs: exp_dir = os.path.join(find_dir, ed) jsons_files = sorted(glob.glob(os.path.join(exp_dir, '**', 'training.json'))) for jf in jsons_files: with open(jf) as f: data = json.load(f) df = df.append({'camera': camera, 'nip': nip, 'lr': re.findall('(lr-[0-9]\.[0-9]{4})', jf.replace(find_dir, ''))[0], 'source': jf.replace(find_dir, '').replace('training.json', ''), 'psnr': data['nip']['validation']['psnr'][-1], 'ssim': data['nip']['validation']['ssim'][-1], 'accuracy': data['forensics']['validation']['accuracy'][-1] }, ignore_index=True) return df
def confusion_data(nip_model, cameras, root_dir='./data/raw/train_manipulation'): cameras = cameras or coreutils.listdir(root_dir, '.', dirs_only=True) confusion = OrderedDict() for camera in cameras: experiment_dirs = sorted(coreutils.listdir(os.path.join(root_dir, camera, nip_model), '.', dirs_only=True)) for ed in experiment_dirs: find_dir = os.path.join(root_dir, camera, nip_model, ed) jsons_files = sorted(glob.glob(os.path.join(find_dir, '**', 'training.json'))) if len(jsons_files) > 1: print('WARNING Using the first found repetition of the experiment') jf = jsons_files[0] with open(jf) as f: data = json.load(f) confusion['{}/{}'.format(camera, ed)] = { 'data': np.array(data['forensics']['validation']['confusion']), 'labels': data['summary']['Classes'] if isinstance(data['summary']['Classes'], list) else eval(data['summary']['Classes']) } return confusion
def discover_files(data_directory, n_images=120, v_images=30, extension='png', randomize=False): """ Find available images and split them into training / validation sets. :param data_directory: directory :param n_images: number of training images :param v_images: number of validation images :param extension: file extension :param randomize: whether to shuffle files before the split """ files = coreutils.listdir(data_directory, '.*\.{}$'.format(extension)) print('In total {} files available'.format(len(files)), flush=True) if randomize: np.random.shuffle(files) if n_images == 0 and v_images == -1: v_images = len(files) if n_images == -1 and v_images == 0: n_images = len(files) if len(files) >= n_images + v_images: val_files = files[n_images:(n_images + v_images)] files = files[0:n_images] else: raise ValueError('Not enough images!') return files, val_files
def autodetect_cameras(dirname): counter = 5 while counter > 0 and not os.path.exists(os.path.join(dirname, 'models', 'nip')): dirname = os.path.split(dirname)[0] counter -= 1 if counter == 0: raise ValueError('The {} directory does not seem to be a valid results directory'.format(dirname)) return coreutils.listdir(os.path.join(dirname, 'models', 'nip'), '.*', dirs_only=True)
def boxplot_data(nip_model, cameras=None, field='accuracy', root_dir='./data/raw/train_manipulation'): cameras = cameras or coreutils.listdir(root_dir, '.', dirs_only=True) find_dir = os.path.join(root_dir, cameras[0], nip_model) experiment_dirs = coreutils.listdir(os.path.join(find_dir), 'lr-.*', dirs_only=True) df = pd.DataFrame(columns=['{} ({})'.format(c, e) for c, e in product(cameras, experiment_dirs)]) for camera in cameras: for ed in experiment_dirs: column = '{} ({})'.format(camera, ed) find_dir = os.path.join(root_dir, camera, nip_model, ed) jsons_files = sorted(glob.glob(os.path.join(find_dir, '**', 'training.json'))) accuracies = [] for jf in jsons_files: with open(jf) as f: data = json.load(f) if field == 'accuracy': accuracies.append(data['forensics']['validation']['accuracy'][-1]) elif field == 'psnr': accuracies.append(data['nip']['validation']['psnr'][-1]) elif field == 'ssim': accuracies.append(data['nip']['validation']['ssim'][-1]) if len(jsons_files): for _ in range(len(df) - len(accuracies)): accuracies.append(np.nan) df[column] = accuracies else: df = df.drop(columns=column) return df
def manipulation_metrics(nip_models, cameras, root_dir=ROOT_DIRNAME): nip_models = [nip_models] if type(nip_models) is str else nip_models cameras = cameras or coreutils.listdir(root_dir, '.', dirs_only=True) if any(cam not in autodetect_cameras(root_dir) for cam in cameras): raise ValueError('The auto-detected camera list does not seem to make sense: {}'.format(cameras)) df = pd.DataFrame(columns=['camera', 'nip', 'ln', 'source', 'psnr', 'ssim', 'accuracy']) for camera in cameras: nip_models = nip_models or coreutils.listdir(os.path.join(root_dir, camera), '.', dirs_only=True) for nip in nip_models: find_dir = os.path.join(root_dir, camera, nip) experiment_dirs = coreutils.listdir(os.path.join(find_dir), '.*', dirs_only=True) for ed in experiment_dirs: exp_dir = os.path.join(find_dir, ed) jsons_files = sorted(str(f) for f in Path(exp_dir).glob('**/training.json')) for jf in jsons_files: with open(jf) as f: data = json.load(f) df = df.append({'camera': camera, 'nip': nip, 'ln': ed, 'source': jf.replace(find_dir, '').replace('training.json', ''), 'psnr': data['nip']['validation']['psnr'][-1], 'ssim': data['nip']['validation']['ssim'][-1], 'accuracy': data['forensics']['validation']['accuracy'][-1] }, ignore_index=True) return df
def develop_image(camera, pipeline, ps=128, image_id=None, root_dir='./data'): """ Display a patch developed by a neural imaging pipeline. """ supported_cameras = coreutils.listdir( os.path.join(root_dir, 'models', 'nip'), '.*') if ps < 4 or ps > 2048: raise ValueError('Patch size seems to be invalid!') if pipeline not in supported_pipelines: raise ValueError( 'Unsupported pipeline model ({})! Available models: {}'.format( pipeline, ', '.join(supported_pipelines))) if camera not in supported_cameras: raise ValueError( 'Camera data not found ({})! Available cameras: {}'.format( camera, ', '.join(supported_cameras))) image_id = image_id or 0 # Lazy imports to minimize delay for invalid command line parameters import numpy as np import imageio as io import matplotlib.pylab as plt import tensorflow as tf from models import pipelines from skimage.measure import compare_psnr root_dirname = os.path.join(root_dir, 'models', 'nip') data_dirname = os.path.join(root_dir, 'raw', 'training_data', camera) files = coreutils.listdir(data_dirname, '.*\.npy') if len(files) == 0: print('ERROR Not training files found for the given camera model!') sys.exit(3) # Get model class instance nip_model = getattr(pipelines, pipeline) # Construct the NIP model tf.reset_default_graph() sess = tf.Session() model = nip_model(sess, tf.get_default_graph()) log.info('Using NIP: {}'.format(model.summary())) log.info('Loading weights from: {}'.format( os.path.join(root_dirname, camera))) model.load_model(os.path.join(root_dirname, camera)) # Load sample data sample_x = np.load(os.path.join(data_dirname, files[image_id])) sample_x = np.expand_dims(sample_x, axis=0) xx = (sample_x.shape[2] - ps) // 2 yy = (sample_x.shape[1] - ps) // 2 log.info('Using image {}'.format(files[image_id])) log.info('Cropping patch from input image x={}, y={}, size={}'.format( xx, yy, ps)) sample_x = sample_x[:, yy:yy + ps, xx:xx + ps, :].astype( np.float32) / (2**16 - 1) sample_x = np.repeat(sample_x, 20, axis=0) sample_y = model.process(sample_x) sample_y = sample_y[0:1] target_y = io.imread( os.path.join(data_dirname, files[image_id].replace('.npy', '.png'))) target_y = target_y[2 * yy:2 * (yy + ps), 2 * xx:2 * (xx + ps), :].astype(np.float32) / (2**8 - 1) psnr_value = compare_psnr(target_y, sample_y.squeeze(), data_range=1.0) log.info('PSNR={:.1f} dB'.format(psnr_value)) # Plot the images plt.figure(figsize=(6, 6)) plt.subplot(1, 2, 1) plt.imshow(sample_y.squeeze()) plt.title('{}, PSNR={:.1f} dB'.format(type(model).__name__, psnr_value)) plt.subplot(1, 2, 2) plt.imshow(target_y) plt.title('Target') plt.show() plt.close()
def develop_images(camera, pipeline, n_images=0, root_dir='./data/raw/', model_dir='nip_model_snapshots', dev_dir='nip_developed', nip_params=None): if pipeline not in supported_pipelines: raise ValueError( 'Unsupported pipeline model ({})! Available models: {}'.format( pipeline, ', '.join(supported_pipelines))) dir_models = os.path.join(root_dir, model_dir) nip_directory = os.path.join(root_dir, 'nip_training_data', camera) out_directory = os.path.join(root_dir, dev_dir, camera, pipeline) raw_directory = os.path.join(root_dir, 'images', camera) if not os.path.exists(nip_directory): raise IOError('Directory not found! {}'.format(nip_directory)) if not os.path.exists(out_directory): os.makedirs(out_directory) # Lazy loading of remaining dependencies to ensure responsiveness of the CLI import numpy as np import imageio import tqdm from helpers import raw_api from models import pipelines print('Camera: {}'.format(camera)) print('Pipeline: {}'.format(pipeline)) print('NIP Models: {}'.format(dir_models)) print('NIP Training Directory: {}'.format(nip_directory)) print('Out Directory: {}'.format(out_directory)) # %% Process Bayer stacks with the given pipeline npy_filenames = coreutils.listdir(nip_directory, '.*\.{}$'.format(extensions)) log.info('Camera {} matched {:,} Bayer stacks'.format( camera, len(npy_filenames))) manual_dev_settings = { 'use_srgb': True, 'use_gamma': True, 'brightness': None } # Setup the NIP model if 'Net' in pipeline: sess = tf.Session() model = getattr(pipelines, pipeline)(sess, tf.get_default_graph(), loss_metric='L1', **nip_params) model.load_model(camera, out_directory_root=dir_models) # Limit the number of images if n_images > 0: npy_filenames = npy_filenames[:n_images] for npy_file in tqdm.tqdm(npy_filenames, ncols=120, desc='Developing ({}/{})'.format( camera, pipeline)): # Find the original RAW file (for standard pipelines.py) raw_file = os.path.join(raw_directory, os.path.splitext(npy_file)[0]) raw_found = False for extension in raw_extensions: if os.path.exists(raw_file + extension): raw_file = raw_file + extension raw_found = True break if not raw_found: raise RuntimeError( 'RAW file not found for Bayer stack: {}'.format(npy_file)) out_png = os.path.join(out_directory, os.path.splitext(npy_file)[0] + '.png') if not os.path.exists(out_png): # Process with the desired pipeline if pipeline == 'libRAW': rgb = raw_api.process_auto(raw_file) elif pipeline == 'Python': rgb = 255 * raw_api.process(raw_file, **manual_dev_settings) rgb = rgb.astype(np.uint8) else: # Find the cached Bayer stack bayer_file = os.path.join(nip_directory, npy_file) bayer_stack = np.load(bayer_file).astype( np.float32) / (2**16 - 1) rgb = 255 * model.process(bayer_stack).squeeze() rgb = rgb.astype(np.uint8) imageio.imwrite(out_png, rgb.astype(np.uint8))
def compare_nips(camera, pipeline, model_a_dirname, model_b_dirname, ps=128, image_id=None, root_dir='./data', output_dir=None): """ Display a comparison of two variants of a neural imaging pipeline. :param camera: camera name (e.g., 'Nikon D90') :param pipeline: neural pipeline name (e.g., 'UNet') :param model_a_dirname: directory with the first variant of the model :param model_b_dirname: directory with the second variant of the model :param ps: patch size (patch will be taken from the middle) :param image_id: index of the test image :param root_dir: root data directory :param output_dir: set an output directory if the figure should be saved (matplotlib2tikz will be used) """ # Lazy imports to minimize delay for invalid command line parameters import re import inspect import imageio as io import matplotlib.pyplot as plt import tensorflow as tf from models import pipelines supported_cameras = coreutils.listdir( os.path.join(root_dir, 'models', 'nip'), '.*') supported_pipelines = pipelines.supported_models if ps < 4 or ps > 2048: raise ValueError('Patch size seems to be invalid!') if pipeline not in supported_pipelines: raise ValueError( 'Unsupported pipeline model ({})! Available models: {}'.format( pipeline, ', '.join(supported_pipelines))) if camera not in supported_cameras: raise ValueError( 'Camera data not found ({})! Available cameras: {}'.format( camera, ', '.join(supported_cameras))) image_id = image_id or 0 # Find available Bayer stacks for the camera dirname = os.path.join(root_dir, 'raw', 'training_data', camera) files = coreutils.listdir(dirname, '.*\.npy') if len(files) == 0: print('ERROR No training files found for the given camera model!') sys.exit(2) # Get model class instance nip_model = getattr(pipelines, pipeline) # Construct the NIP models g1 = tf.Graph() g2 = tf.Graph() sess1 = tf.Session(graph=g1) sess2 = tf.Session(graph=g2) model_a = nip_model(sess1, g1) model_a.init() model_a.load_model( os.path.join(model_a_dirname, camera) if camera not in model_a_dirname else model_a_dirname) model_b = nip_model(sess2, g2) model_b.init() model_b.load_model( os.path.join(model_b_dirname, camera) if camera not in model_b_dirname else model_b_dirname) log.info('Model A: {}'.format(model_a.summary())) log.info('Model B: {}'.format(model_b.summary())) # Load sample data sample_x = np.load(os.path.join(dirname, files[image_id])) sample_x = np.expand_dims(sample_x, axis=0) xx = (sample_x.shape[2] - ps) // 2 yy = (sample_x.shape[1] - ps) // 2 log.info('Using image {}'.format(files[image_id])) log.info('Cropping patch from input image x={}, y={}, size={}'.format( xx, yy, ps)) sample_x = sample_x[:, yy:yy + ps, xx:xx + ps, :].astype( np.float32) / (2**16 - 1) # Develop images sample_ya = model_a.process(sample_x) sample_yb = model_b.process(sample_x) target_y = io.imread( os.path.join(dirname, files[image_id].replace('.npy', '.png'))) target_y = target_y[2 * yy:2 * (yy + ps), 2 * xx:2 * (xx + ps), :].astype(np.float32) / (2**8 - 1) # Plot images fig = compare_images_ab_ref(target_y, sample_ya, sample_yb) if output_dir is not None: from tikzplotlib import save as tikz_save dcomp = [ x for x in coreutils.splitall(model_b_dirname) if re.match('(ln-.*|[0-9]{3})', x) ] tikz_save('{}/examples-{}-{}-{}-{}-{}.tex'.format( output_dir, camera, pipeline, image_id, dcomp[0], dcomp[1]), figureheight='8cm', figurewidth='8cm', strict=False) else: fig.tight_layout() fig.show(fig) plt.show() plt.close(fig)
def compare_nips(camera, pipeline, model_a_dirname, model_b_dirname, ps=128, image_id=None, root_dir='./data/raw', output_dir=None): """ Display a comparison of two variants of a neural imaging pipeline. :param camera: camera name (e.g., 'Nikon D90') :param pipeline: neural pipeline name (e.g., 'UNet') :param model_a_dirname: directory with the first variant of the model :param model_b_dirname: directory with the second variant of the model :param ps: patch size (patch will be taken from the middle) :param image_id: index of the test image :param root_dir: root data directory :param output_dir: set an output directory if the figure should be saved (matplotlib2tikz will be used) """ supported_cameras = coreutils.listdir( os.path.join(root_dir, 'nip_model_snapshots'), '.*') if ps < 4 or ps > 2048: raise ValueError('Patch size seems to be invalid!') if pipeline not in supported_pipelines: raise ValueError( 'Unsupported pipeline model ({})! Available models: {}'.format( pipeline, ', '.join(supported_pipelines))) if camera not in supported_cameras: raise ValueError( 'Camera data not found ({})! Available cameras: {}'.format( camera, ', '.join(supported_cameras))) image_id = image_id or 0 # Lazy imports to minimize delay for invalid command line parameters import re import imageio as io import matplotlib.pyplot as plt import tensorflow as tf from models import pipelines from skimage.measure import compare_psnr, compare_ssim # Find available Bayer stacks for the camera dirname = os.path.join(root_dir, 'nip_training_data', camera) files = coreutils.listdir(dirname, '.*\.npy') if len(files) == 0: print('ERROR Not training files found for the given camera model!') sys.exit(2) # Get model class instance nip_model = getattr(pipelines, pipeline) # Construct the NIP models g1 = tf.Graph() g2 = tf.Graph() sess1 = tf.Session(graph=g1) sess2 = tf.Session(graph=g2) model_a = nip_model(sess1, g1) model_a.init() model_a.load_model(os.path.join(model_a_dirname)) model_b = nip_model(sess2, g2) model_b.init() model_b.load_model(os.path.join(model_b_dirname)) log.info('Model A: {}'.format(model_a.summary())) log.info('Model B: {}'.format(model_b.summary())) # Load sample data sample_x = np.load(os.path.join(dirname, files[image_id])) sample_x = np.expand_dims(sample_x, axis=0) xx = (sample_x.shape[2] - ps) // 2 yy = (sample_x.shape[1] - ps) // 2 log.info('Using image {}'.format(files[image_id])) log.info('Cropping patch from input image x={}, y={}, size={}'.format( xx, yy, ps)) sample_x = sample_x[:, yy:yy + ps, xx:xx + ps, :].astype( np.float32) / (2**16 - 1) # Develop images sample_ya = model_a.process(sample_x) sample_yb = model_b.process(sample_x) target_y = io.imread( os.path.join(dirname, files[image_id].replace('.npy', '.png'))) target_y = target_y[2 * yy:2 * (yy + ps), 2 * xx:2 * (xx + ps), :].astype(np.float32) / (2**8 - 1) # Plot images fig = plt.figure(figsize=(9, 9)) quick_show(fig.add_subplot(3, 3, 1), target_y, '(T)arget') quick_show( fig.add_subplot(3, 3, 2), sample_ya.squeeze(), '{}(A) {:.1f} dB / {:.3f}'.format( pipeline, compare_psnr(target_y, sample_ya.squeeze(), data_range=1.0), compare_ssim(target_y, sample_ya.squeeze(), multichannel=True))) quick_show( fig.add_subplot(3, 3, 4), sample_yb.squeeze(), '{}(B) {:.1f} dB / {:.3f}'.format( pipeline, compare_psnr(target_y, sample_yb.squeeze(), data_range=1.0), compare_ssim(target_y, sample_yb.squeeze(), multichannel=True))) # Compute and plot difference images diff_a = np.abs(sample_ya.squeeze() - target_y) diff_a_mean = diff_a.mean() diff_a = nm(diff_a) diff_b = np.abs(sample_yb.squeeze() - target_y) diff_b_mean = diff_b.mean() diff_b = nm(diff_b) diff_ab = np.abs(sample_yb.squeeze() - sample_ya.squeeze()) diff_ab_mean = diff_ab.mean() diff_ab = nm(diff_ab) quick_show(fig.add_subplot(3, 3, 3), diff_a, 'T - A: mean abs {:.3f}'.format(diff_a_mean)) quick_show(fig.add_subplot(3, 3, 7), diff_b, 'T - B: mean abs {:.3f}'.format(diff_b_mean)) quick_show(fig.add_subplot(3, 3, 5), diff_ab, 'A - B: mean abs {:.3f}'.format(diff_ab_mean)) # Compute and plot spectra fft_a = fft_log_norm(diff_a) fft_b = fft_log_norm(diff_b) # fft_ab = nm(np.abs(fft_a - fft_b)) fft_ab = nm( np.abs( fft_log_norm(sample_yb.squeeze()) - fft_log_norm(sample_ya.squeeze()))) quick_show(fig.add_subplot(3, 3, 6), fft_a, 'FFT(T - A)') quick_show(fig.add_subplot(3, 3, 8), fft_b, 'FFT(T - B)') quick_show(fig.add_subplot(3, 3, 9), fft_ab, 'FFT(A) - FFT(B)') if output_dir is not None: from matplotlib2tikz import save as tikz_save dcomp = [ x for x in model_b_dirname.split('/') if re.match('(lr-.*|[0-9]{3})', x) ] tikz_save('{}/examples-{}-{}-{}-{}-{}.tex'.format( output_dir, camera, pipeline, image_id, dcomp[0], dcomp[1]), figureheight='8cm', figurewidth='8cm', strict=False) else: fig.tight_layout() plt.show(fig) plt.close(fig)
def prepare_training_set(camera, target_pipeline, dev_settings, n_images=150, root_dir='./data/raw/', raw_dir=''): if target_pipeline not in ['auto', 'manual', 'camera']: raise ValueError('Unsupported target pipeline!') raw_directory = raw_dir out_directory = os.path.join(root_dir, camera) if not os.path.exists(raw_directory): log.error('Directory not found! {}'.format(raw_directory)) sys.exit(2) if not os.path.exists(out_directory): os.makedirs(out_directory) print('RAW Directory: {}'.format(raw_directory)) print('Out Directory: {}'.format(out_directory)) # List RAW files and find the ones with horizontal orientation raw_filenames = coreutils.listdir(raw_directory, '.*\.{}$'.format(EXTENSIONS)) log.info('Camera {} matched {:,} RAW images'.format( camera, len(raw_filenames))) raw_filenames_selected = [] for nef_file in raw_filenames: with open(os.path.join(raw_directory, nef_file), 'rb') as f: tags = exifread.process_file(f, details=False, stop_tag='Image Orientation') orientation = tags['Image Orientation'].printable log.info('{} -> {}'.format(nef_file, orientation)) if orientation.startswith('Horizontal'): raw_filenames_selected.append(nef_file) if len(raw_filenames_selected) >= n_images: break log.info('Collected {} landscape-oriented photos for training'.format( len(raw_filenames_selected))) if len(raw_filenames_selected) < n_images: log.error( 'Not enough horizontal images! Found {} but expected {}.'.format( len(raw_filenames_selected), n_images)) dev_settings = dev_settings or { 'use_srgb': True, 'use_gamma': True, 'brightness': None } # Iterate over RAW files and produce: # 1. RGGB Bayer stacks (H/2, W/2, 4) # 2. RGB Optimization target (H, W, 3) for nef_file in tqdm.tqdm( raw_filenames_selected, ncols=120, desc='Preparing train. data ({})'.format(camera)): out_npy = os.path.join(out_directory, os.path.splitext(nef_file)[0] + '.npy') out_png = os.path.join(out_directory, os.path.splitext(nef_file)[0] + '.png') try: if not os.path.exists(out_npy): image_bayer = raw_api.stacked_bayer(os.path.join( raw_directory, nef_file), camera, use_wb=True) image_bayer = ((2**16 - 1) * image_bayer).astype(np.uint16) np.save(out_npy, image_bayer) if not os.path.exists(out_png): if target_pipeline == 'auto': rgb = raw_api.process_auto( os.path.join(raw_directory, nef_file)) elif target_pipeline == 'manual': rgb = 255 * raw_api.process( os.path.join(raw_directory, nef_file), **dev_settings) elif target_pipeline == 'camera': out_jpg = os.path.join( out_directory, os.path.splitext(nef_file)[0] + '.JPG') else: raise ValueError('Unsupported develop mode!') if target_pipeline != 'camera': imageio.imwrite(out_png, rgb.astype(np.uint8)) except Exception as error: log.error('RAW Processing failed for file: {}'.format(nef_file)) log.error(error) sys.exit(2) sys.exit(0)
def display_results(args): sns.set('paper', font_scale=1, style="ticks") plot = coreutils.match_option(args.plot, supported_plots) if not os.path.isdir(args.dir): raise FileNotFoundError('Directory {} not found!'.format(args.dir)) print('Results from: {}'.format(args.dir)) print('Matched plotting command: {}'.format(plot)) postfix = [ coreutils.splitall(args.dir)[-1], ','.join(args.nips) if args.nips is not None else None, ','.join(args.cameras) if args.cameras is not None else None, ] postfix = '-'.join(x for x in postfix if x is not None) if plot in ['ssim', 'psnr', 'accuracy']: df = results_data.manipulation_metrics(args.nips, args.cameras, root_dir=args.dir) sns.catplot(x='ln', y=plot, col='camera', row='nip', data=df, kind='box') save_df(df, args.df, 'manipulation_metrics-{}.csv'.format(postfix)) plt.show() return if plot == 'scatter-psnr' or plot == 'scatter-ssim': df = results_data.manipulation_metrics(args.nips, args.cameras, root_dir=args.dir) if len(df) == 0: print('ERROR No results found!') sys.exit(2) print(df) g = sns.relplot(x=plot.split('-')[-1], y='accuracy', hue='ln', col='camera', row='nip', data=df, palette=sns.color_palette("Set2", len(df['ln'].unique()))) save_df(df, args.df, 'manipulation_metrics-{}.csv'.format(postfix)) plt.show() return if plot == 'progress': cases = [] if args.cameras is None: args.cameras = coreutils.listdir(args.dir, '.', dirs_only=True) for cam in args.cameras: nip_models = args.nips or coreutils.listdir( os.path.join(args.dir, cam), '.', dirs_only=True) for nip in nip_models: reg_path = os.path.join(args.dir, cam, nip) if args.regularization: # If given, use specified regularization strengths reg_list = args.regularization else: # Otherwise, auto-detect available scenarios reg_list = coreutils.listdir(reg_path, '.*', dirs_only=True) if len(reg_list) > 4: indices = np.linspace(0, len(reg_list) - 1, 4).astype(np.int32) reg_list = [reg_list[i] for i in indices] print( '! warning - too many experiments to show - sampling: {}' .format(reg_list)) for reg in reg_list: for r in coreutils.listdir(os.path.join(reg_path, reg), '[0-9]+', dirs_only=True): print('* found scenario {}'.format( (cam, nip, reg, int(r)))) cases.append((cam, nip, reg, int(r))) df, labels = results_data.manipulation_progress(cases, root_dir=args.dir) save_df(df, args.df, 'progress-{}.csv'.format(postfix)) for col in ['psnr', 'accuracy']: sns.relplot(x="step", y=col, hue='exp', row='nip', col='camera', style='exp', kind="line", legend="full", aspect=2, height=3, data=df) plt.show() return if plot == 'conf' or plot == 'conf-tex': if isinstance(args.nips, list): if len(args.nips) > 1: print('WARNING Only one NIP will be used for this plot!') args.nips = args.nips[0] conf = results_data.confusion_data(args.run, root_dir=args.dir) if len(conf) == 0: print('ERROR No results found!') return tex_output = plot == 'conf-tex' plot_data = not tex_output if len(conf.keys()) < 20 else False if plot_data: images_x = np.ceil(np.sqrt(len(conf))) images_y = np.ceil(len(conf) / images_x) f_size = 3 fig = plt.figure(figsize=(images_x * f_size, images_y * f_size)) for i, (k, c) in enumerate(conf.items()): data = (100 * c['data']).round(0) labels = c['labels'] if tex_output: print(results_data.confusion_to_text(data, labels, k, 'tex')) else: print(results_data.confusion_to_text(data, labels, k, 'txt')) if plot_data: acc = np.mean(np.diag(data)) ax = fig.add_subplot(images_y, images_x, i + 1) sns.heatmap(data, annot=True, fmt=".0f", linewidths=.5, xticklabels=[x[0] for x in labels], yticklabels=labels) ax.set_title('{} : acc={:.1f}'.format(k, acc)) if plot_data: plt.tight_layout() plt.show() return if plot == 'df': print('Searching for "training.json" in', args.dir) df = results_data.manipulation_summary(args.dir) if len(df) > 0: if False: print(df.groupby('scenario').mean().to_string()) else: gb = df.groupby('scenario') counts = gb.size().to_frame(name='reps') print(counts.join(gb.agg('mean')).reset_index().to_string()) save_df(df, args.df, 'summary-{}.csv'.format(postfix)) return if plot == 'auto': print('Searching for "training.json" in', args.dir) df = results_data.manipulation_summary(args.dir) df = df.sort_values('scenario') guessed_names = {} # Guess scenario components = df['scenario'].str.split("/", expand=True) for i in components: # Try to guess the column name based on content template = 'scenario:{}'.format(i) if components.iloc[0, i].endswith('Net'): guessed_names[template] = 'nip' elif components.iloc[0, i].startswith('ln-'): guessed_names[template] = 'nip reg.' elif components.iloc[0, i].startswith('lc-'): guessed_names[template] = 'dcn reg.' elif set(components.iloc[:, i].unique()) == {'4k', '8k', '16k'}: guessed_names[template] = 'dcn' elif all([ re.match('^[0-9]{2,3}$', x) for x in components.iloc[:, i].unique() ]): guessed_names[template] = 'jpeg' else: guessed_names[template] = template df[guessed_names[template]] = components[i] df['scenario'] = coreutils.remove_commons(df['scenario']) mapping = {} mapping_targets = ['col', 'col', 'hue', 'style', 'size'] mapping_id = 0 # Choose the feature with most unique values as x axis uniques = [ len(df[guessed_names['scenario:{}'.format(i)]].unique()) for i in components ] x_feature = np.argmax(uniques) for i in components: if i == x_feature: continue if len(df[guessed_names['scenario:{}'.format(i)]].unique()) > 1: mapping[mapping_targets[mapping_id]] = guessed_names[ 'scenario:{}'.format(i)] mapping_id += 1 sns.catplot(x=guessed_names['scenario:{}'.format(x_feature)], y='accuracy', data=df, kind='box', **mapping) # sns.catplot(x='scenario:0', y='dcn_ssim', data=df, kind='box', **mapping) # sns.scatterplot(x='dcn_ssim', y='accuracy', data=df) plt.show() if len(df) > 0: gb = df.groupby('scenario') counts = gb.size().to_frame(name='reps') print(counts.join(gb.agg('mean')).reset_index().to_string()) return raise RuntimeError('No plot matched! Available plots {}'.format( ', '.join(supported_plots)))
def display_results(args): sns.set() plot = coreutils.match_option(args.plot, supported_plots) if plot == 'boxplot': for nip in args.nips: df = boxplot_data(nip, args.cameras, root_dir=args.dir) print(df) print('Averages') print(df.mean().round(2)) plt.figure() sns.boxplot(data=df) plt.xticks(rotation=90) plt.gca().set_title(nip) if args.df is not None: if not os.path.isdir(args.df): os.makedirs(args.df) df_filename = '{}/box-{}-{}-{}.csv'.format(args.df, 'accuracy', nip, plot) df.to_csv(df_filename, index=False) print('> saving dataframe to {}'.format(df_filename)) plt.show() return if plot == 'psnr' or plot == 'ssim': for nip in args.nips: df = boxplot_data(nip, args.cameras, field=plot, root_dir=args.dir) print(df) print('Averages') print(df.mean().round(1 if plot == 'psnr' else 3)) plt.figure() sns.boxplot(data=df) plt.xticks(rotation=90) plt.gca().set_title(nip) if args.df is not None: if not os.path.isdir(args.df): os.makedirs(args.df) df_filename = '{}/box-{}-{}-{}.csv'.format(args.df, plot, nip, plot) df.to_csv(df_filename, index=False) print('> saving dataframe to {}'.format(df_filename)) plt.show() return if plot == 'scatter-psnr' or plot == 'scatter-ssim': if args.cameras is None: args.cameras = coreutils.listdir(args.dir, '.', dirs_only=True) for cam in args.cameras: df = scatterplot_data(args.nips, cam, root_dir=args.dir) print(df) sns.relplot(x=plot.split('-')[-1], y='accuracy', hue='lr', col='camera', data=df) if args.df is not None: if not os.path.isdir(args.df): os.makedirs(args.df) df_filename = '{}/scatter-{}-{}.csv'.format(args.df, cam, ','.join(args.nips)) df.to_csv(df_filename, index=False) print('> saving dataframe to {}'.format(df_filename)) plt.show() return if plot == 'progressplot': cases = [] if args.cameras is None: args.cameras = coreutils.listdir(args.dir, '.', dirs_only=True) for cam in args.cameras: for nip in args.nips: reg_path = os.path.join(args.dir, cam, nip) if args.regularization: # If given, use specified regularization strengths reg_list = args.regularization else: # Otherwise, auto-detect available scenarios reg_list = coreutils.listdir(reg_path, 'lr-[0-9\.]+', dirs_only=True) if len(reg_list) > 4: indices = np.linspace(0, len(reg_list)-1, 4).astype(np.int32) reg_list = [reg_list[i] for i in indices] print('! warning - too many experiments to show - sampling: {}'.format(reg_list)) for reg in reg_list: for r in coreutils.listdir(os.path.join(reg_path, reg), '[0-9]+', dirs_only=True): print('* found scenario {}'.format((cam, nip, reg, int(r)))) cases.append((cam, nip, reg, int(r))) df, labels = progressplot_data(cases, root_dir=args.dir) if args.df is not None: if not os.path.isdir(args.df): os.makedirs(args.df) df_filename = '{}/progress-{}-{}.csv'.format(args.df, ','.join(args.cameras), ','.join(args.nips)) df.to_csv(df_filename, index=False) print('> saving dataframe to {}'.format(df_filename)) for col in ['psnr', 'accuracy']: sns.relplot(x="step", y=col, hue='exp', col='nip', row='camera', style='exp', kind="line", legend="full", aspect=2, height=3, data=df) plt.show() return if plot == 'confusion': if isinstance(args.nips, list): if len(args.nips) > 1: print('WARNING Only one NIP will be used for this plot!') args.nips = args.nips[0] conf = confusion_data(args.nips, args.cameras, root_dir=args.dir) images_x = np.ceil(np.sqrt(len(conf))) images_y = np.ceil(len(conf) / images_x) f_size = 3 fig = plt.figure(figsize=(images_x*f_size, images_y*f_size)) for i, (k, c) in enumerate(conf.items()): data = (100*c['data']).round(0) labels = c['labels'] print('\n', k, '=') print(data) print(labels) acc = np.mean(np.diag(data)) print('Accuracy = {}'.format(acc)) ax = fig.add_subplot(images_y, images_x, i+1) sns.heatmap(data, annot=True, fmt=".0f", linewidths=.5, xticklabels=[x[0] for x in labels], yticklabels=labels) ax.set_title('{} : acc={}'.format(k, acc)) plt.tight_layout() plt.show() return raise RuntimeError('No plot matched! Available plots {}'.format(', '.join(supported_plots)))