def __init__(self, data_directory, *, randomize=True, load='xy', n_images=120, v_images=30, val_rgb_patch_size=128, val_n_patches=1, rgb_extension='png'): if not any(load == allowed for allowed in ['xy', 'x', 'y']): raise ValueError('Invalid X/Y data requested!') self.files = {} self._loaded_data = load self._data_directory = data_directory self.files['training'], self.files[ 'validation'] = loading.discover_files(data_directory, randomize=randomize, n_images=n_images, v_images=v_images, extension=rgb_extension) self.data = { 'training': loading.load_images(self.files['training'], data_directory=data_directory, load=load, extension=rgb_extension), 'validation': loading.load_patches(self.files['validation'], data_directory=data_directory, patch_size=val_rgb_patch_size // 2, n_patches=val_n_patches, load=load, discard_flat=True, extension=rgb_extension) } if 'y' in self.data['training']: self.H, self.W = self.data['training']['y'].shape[1:3] else: self.H, self.W = (2 * dim for dim in self.data['training']['x'].shape[1:3]) print('Loaded dataset with {}x{} images'.format(self.W, self.H))
def __init__(self, data_directory, *, randomize=2468, load='xy', n_images=120, v_images=30, val_rgb_patch_size=128, val_n_patches=1, val_discard='flat-aggressive'): """ Represents a [RAW-]RGB dataset for training imaging pipelines. The class preloads full resolution images and samples from them when requesting training batches. (Validation images are sampled upon creation.) Patch selection takes care of proper alignment between RAW images (represented as half-size 4-channel RGGB stacks) and their corresponding rendered RGB versions. Selection can be controlled to prefer certain types of patches (not strictly enforced). The following DISCARD modes are available: - flat - attempts to discard flat patches based on patch variance (not strict) - flat-aggressive - a more aggressive version that avoids patches with variance < 0.01 - dark-n-textured - avoid dark (mean < 0.35) and textured patches (variance > 0.005) Usage examples: --------------- # Load a RAW -> RGB dataset data = Dataset('data/raw/training_data/D90') batch_raw, batch_rgb = data.next_training_batch(0, 10, 128, 'flat-aggressive') # Load RGB only dataset data = Dataset('data/rgb/native12k/', load='y') batch_rgb = data.next_training_batch(0, 10, 128, 'flat-aggressive') :param data_directory: directory path with RAW-RGB pairs (*.npy & *.png) or only RGB images (*.png) :param randomize: randomization seed :param load: what data to load: 'xy' load RAW+RGB, 'x' load RAW only, 'y' load RGB only :param n_images: number of training images (full resolution) :param v_images: number of validation images (patches sampled upon creation) :param val_rgb_patch_size: validation patch size :param val_n_patches: number of validation patches to load per full-resolution image :param val_discard: patch discard mode (for validation data) """ if not any(load == allowed for allowed in ['xy', 'x', 'y']): raise ValueError('Invalid X/Y data requested!') if not os.path.isdir(data_directory): if '/' in data_directory or '\\' in data_directory: raise ValueError( f'Cannot find the data directory: {data_directory}') if os.path.isdir( os.path.join('data/raw/training_data/', data_directory)): data_directory = os.path.join('data/raw/training_data/', data_directory) elif os.path.isdir(os.path.join('data/rgb/', data_directory)): data_directory = os.path.join('data/rgb/', data_directory) else: raise ValueError( f'Cannot find the data directory: {data_directory}') self.files = {} self._loaded_data = load self._data_directory = data_directory self._counts = (n_images, v_images, val_n_patches) self._val_discard = 'flat-aggressive' self.files['training'], self.files[ 'validation'] = loading.discover_images(data_directory, randomize=randomize, n_images=n_images, v_images=v_images) self.data = { 'training': loading.load_images(self.files['training'], data_directory, load=load), 'validation': loading.load_patches(self.files['validation'], data_directory, patch_size=val_rgb_patch_size // 2, n_patches=val_n_patches, load=load, discard=val_discard) } if 'y' in self.data['training']: self.H, self.W = self.data['training']['y'].shape[1:3] else: self.H, self.W = (2 * dim for dim in self.data['training']['x'].shape[1:3])
def get_dcn_df(directory, model_directory, write_files=False, force_calc=False): """ Compute and return (as Pandas DF) the rate distortion curve for the learned DCN codec. The result is saved as a CSV file in the source directory. If the file exists, the DF is loaded and returned. """ # Discover test files files, _ = loading.discover_images(directory, n_images=-1, v_images=0) batch_x = loading.load_images(files, directory, load='y') batch_x = batch_x['y'].astype(np.float32) / (2**8 - 1) # Create a new table for the DCN df = pd.DataFrame(columns=[ 'image_id', 'filename', 'model_dir', 'codec', 'ssim', 'psnr', 'msssim', 'msssim_db', 'entropy', 'bytes', 'bpp', 'layers', 'quantization', 'entropy_reg', 'codebook', 'latent', 'latent_shape', 'n_features' ]) # Discover available models model_dirs = list(Path(model_directory).glob('**/progress.json')) logger.info('Found {} models'.format(len(model_dirs))) df_path = os.path.join( directory, 'dcn-{}.csv'.format( [x for x in fsutil.split(model_directory) if len(x) > 0][-1])) if os.path.isfile(df_path) and not force_calc: logger.info('Restoring DCN stats from {}'.format(df_path)) df = pd.read_csv(df_path, index_col=False) else: for model_dir in model_dirs: logger.info('Processing model dir: {}'.format(model_dir)) dcn = codec.restore( os.path.split(str(model_dir))[0], batch_x.shape[1]) # Dump compressed images for image_id, filename in enumerate(files): try: batch_y, image_bytes = codec.simulate_compression( batch_x[image_id:image_id + 1], dcn) batch_z = dcn.compress(batch_x[image_id:image_id + 1]) entropy = helpers.stats.entropy(batch_z, dcn.get_codebook()) except Exception as e: logger.error( 'Error while processing {} with {} : {}'.format( filename, dcn.model_code, e)) raise e if write_files: image_dir = os.path.join(directory, os.path.splitext(filename)[0]) if not os.path.isdir(image_dir): os.makedirs(image_dir) image_path = os.path.join( image_dir, dcn.model_code.replace('/', '-') + '.png') imageio.imwrite(image_path, (255 * batch_y[0]).astype(np.uint8)) msssim_value = msssim(batch_x[image_id], batch_y[0], MAX=1).real df = df.append( { 'image_id': image_id, 'filename': filename, 'model_dir': os.path.relpath( os.path.split(str(model_dir))[0], model_directory).replace(dcn.scoped_name, ''), 'codec': dcn.model_code, 'ssim': compare_ssim(batch_x[image_id], batch_y[0], multichannel=True, data_range=1), 'psnr': compare_psnr( batch_x[image_id], batch_y[0], data_range=1), 'msssim': msssim_value, 'msssim_db': -10 * np.log10(1 - msssim_value), 'entropy': entropy, 'bytes': image_bytes, 'bpp': 8 * image_bytes / batch_x[image_id].shape[0] / batch_x[image_id].shape[1], 'layers': dcn.n_layers if 'n_layers' in dcn._h else None, 'quantization': '{}-{:.0f}bpf'.format(dcn._h.rounding, dcn.latent_bpf), 'entropy_reg': dcn.entropy_weight, 'codebook': dcn._h.rounding, 'latent': dcn.n_latent, 'latent_shape': '{}x{}x{}'.format(*dcn.latent_shape[1:]), 'n_features': dcn.latent_shape[-1] }, ignore_index=True) df.to_csv(df_path, index=False) return df
def get_bpg_df(directory, write_files=False, effective_bytes=True, force_calc=False): """ Compute and return (as Pandas DF) the rate distortion curve for BPG. The result is saved as a CSV file in the source directory. If the file exists, the DF is loaded and returned. The files are saved using the reference codec: https://bellard.org/bpg/ """ files, _ = loading.discover_images(directory, n_images=-1, v_images=0) batch_x = loading.load_images(files, directory, load='y') batch_x = batch_x['y'].astypre(np.float32) / (2**8 - 1) quality_levels = np.arange(10, 40, 1) df_jpeg_path = os.path.join(directory, 'bpg.csv') if os.path.isfile(df_jpeg_path) and not force_calc: logger.info('Restoring BPG stats from {}'.format(df_jpeg_path)) df = pd.read_csv(df_jpeg_path, index_col=False) else: df = pd.DataFrame(columns=[ 'image_id', 'filename', 'codec', 'quality', 'ssim', 'psnr', 'msssim', 'msssim_db', 'bytes', 'bpp' ]) with tqdm.tqdm(total=len(files) * len(quality_levels), ncols=120, desc='BPG') as pbar: for image_id, filename in enumerate(files): # Read the original image image = batch_x[image_id] for qi, q in enumerate(quality_levels): # Compress to BPG # Save as temporary file imageio.imwrite('/tmp/image.png', (255 * image).astype(np.uint8)) bpp_path = bpg_helpers.bpg_compress( '/tmp/image.png', q, '/tmp') image_compressed = imageio.imread( bpg_helpers.decode_bpg_to_png(bpp_path)).astype( np.float) / (2**8 - 1) if effective_bytes: bpp = bpg_helpers.bpp_of_bpg_image(bpp_path) image_bytes = round(bpp * image.shape[0] * image.shape[1] / 8) else: image_bytes = os.stat(bpp_path).st_size bpp = 8 * image_bytes / image.shape[0] / image.shape[1] if write_files: image_dir = os.path.join(directory, os.path.splitext(filename)[0]) if not os.path.isdir(image_dir): os.makedirs(image_dir) image_path = os.path.join(image_dir, 'bpg_q{:03d}.png'.format(q)) imageio.imwrite(image_path, (255 * image_compressed).astype( np.uint8)) msssim_value = msssim(image, image_compressed, MAX=1).real df = df.append( { 'image_id': image_id, 'filename': filename, 'codec': 'bpg', 'quality': q, 'ssim': compare_ssim(image, image_compressed, multichannel=True, data_range=1), 'psnr': compare_psnr(image, image_compressed, data_range=1), 'msssim': msssim_value, 'msssim_db': -10 * np.log10(1 - msssim_value), 'bytes': image_bytes, 'bpp': bpp }, ignore_index=True) pbar.set_postfix(image_id=image_id, quality=q) pbar.update(1) df.to_csv(df_jpeg_path, index=False) return df
def get_jpeg2k_df(directory, write_files=False, effective_bytes=True, force_calc=False): """ Compute and return (as Pandas DF) the rate distortion curve for JPEG 2000. The result is saved as a CSV file in the source directory. If the file exists, the DF is loaded and returned. Files are saved as JPEG using glymur. """ files, _ = loading.discover_images(directory, n_images=-1, v_images=0) batch_x = loading.load_images(files, directory, load='y') batch_x = batch_x['y'].astype(np.float32) / (2**8 - 1) # Get trade-off for JPEG quality_levels = np.arange(25, 45, 1) df_jpeg_path = os.path.join(directory, 'jpeg2000.csv') if os.path.isfile(df_jpeg_path) and not force_calc: logger.info('Restoring JPEG 2000 stats from {}'.format(df_jpeg_path)) df = pd.read_csv(df_jpeg_path, index_col=False) else: df = pd.DataFrame(columns=[ 'image_id', 'filename', 'codec', 'quality', 'ssim', 'psnr', 'msssim', 'msssim_db', 'bytes', 'bpp' ]) with tqdm.tqdm(total=len(files) * len(quality_levels), ncols=120, desc='JP2k') as pbar: for image_id, filename in enumerate(files): # Read the original image image = batch_x[image_id] for qi, q in enumerate(quality_levels): # TODO Use Glymur to save JPEG 2000 images to a temp file image_np = (255 * image.clip(0, 1)).astype(np.uint8) glymur.Jp2k('/tmp/image.jp2', data=image_np, psnr=[q]) if effective_bytes: image_bytes = jpeg_helpers.jp2bytes('/tmp/image.jp2') else: image_bytes = os.path.getsize('/tmp/image.jp2') image_compressed = imageio.imread('/tmp/image.jp2').astype( np.float) / (2**8 - 1) # TODO Use Pillow to save JPEG 2000 images to a memory buffer # TODO This has been disabled = their implementation seems to be invalid # with io.BytesIO() as output: # image_pillow = PIL.Image.fromarray((255*image.clip(0, 1)).astype(np.uint8)) # image_pillow.save(output, format='jpeg2000', quality_layers=[q]) # image_compressed = imageio.imread(output.getvalue()).astype(np.float) / (2**8 - 1) # image_bytes = len(output.getvalue()) if write_files: image_dir = os.path.join(directory, os.path.splitext(filename)[0]) if not os.path.isdir(image_dir): os.makedirs(image_dir) image_path = os.path.join( image_dir, 'jp2_q{:.1f}dB.png'.format(q)) imageio.imwrite(image_path, (255 * image_compressed).astype( np.uint8)) msssim_value = msssim(image, image_compressed, MAX=1).real df = df.append( { 'image_id': image_id, 'filename': filename, 'codec': 'jpeg2000', 'quality': q, 'ssim': compare_ssim(image, image_compressed, multichannel=True, data_range=1), 'psnr': compare_psnr(image, image_compressed, data_range=1), 'msssim': msssim_value, 'msssim_db': -10 * np.log10(1 - msssim_value), 'bytes': image_bytes, 'bpp': 8 * image_bytes / image.shape[0] / image.shape[1] }, ignore_index=True) pbar.set_postfix(image_id=image_id, quality=q) pbar.update(1) df.to_csv(df_jpeg_path, index=False) return df
def get_jpeg_df(directory, write_files=False, effective_bytes=True, force_calc=False): """ Compute and return (as Pandas DF) the rate distortion curve for JPEG. The result is saved as a CSV file in the source directory. If the file exists, the DF is loaded and returned. Files are saved as JPEG using imageio. """ files, _ = loading.discover_images(directory, n_images=-1, v_images=0) batch_x = loading.load_images(files, directory, load='y') batch_x = batch_x['y'].astype(np.float32) / (2**8 - 1) # Get trade-off for JPEG quality_levels = np.arange(95, 5, -5) df_jpeg_path = os.path.join(directory, 'jpeg.csv') if os.path.isfile(df_jpeg_path) and not force_calc: logger.info('Restoring JPEG stats from {}'.format(df_jpeg_path)) df = pd.read_csv(df_jpeg_path, index_col=False) else: df = pd.DataFrame(columns=[ 'image_id', 'filename', 'codec', 'quality', 'ssim', 'psnr', 'msssim', 'msssim_db', 'bytes', 'bpp' ]) with tqdm.tqdm(total=len(files) * len(quality_levels), ncols=120, desc='JPEG') as pbar: for image_id, filename in enumerate(files): # Read the original image image = batch_x[image_id] for qi, q in enumerate(quality_levels): # Compress images and get effective bytes (only image data - no headers) image_compressed, image_bytes = jpeg_helpers.compress_batch( image, q, effective=effective_bytes) if write_files: image_dir = os.path.join(directory, os.path.splitext(filename)[0]) if not os.path.isdir(image_dir): os.makedirs(image_dir) image_path = os.path.join(image_dir, 'jpeg_q{:03d}.png'.format(q)) imageio.imwrite(image_path, (255 * image_compressed).astype( np.uint8)) msssim_value = msssim(image, image_compressed, MAX=1).real df = df.append( { 'image_id': image_id, 'filename': filename, 'codec': 'jpeg', 'quality': q, 'ssim': compare_ssim(image, image_compressed, multichannel=True, data_range=1), 'psnr': compare_psnr(image, image_compressed, data_range=1), 'msssim': msssim_value, 'msssim_db': -10 * np.log10(1 - msssim_value), 'bytes': image_bytes, 'bpp': 8 * image_bytes / image.shape[0] / image.shape[1] }, ignore_index=True) pbar.set_postfix(image_id=image_id, quality=q) pbar.update(1) df.to_csv(os.path.join(directory, 'jpeg.csv'), index=False) return df
def get_diff_jpeg_df(directory, write_files=False, effective_bytes=True, force_calc=False): """ Compute and return (as Pandas DF) the rate distortion curve for diff JPEG. The result is saved as a CSV file in the source directory. If the file exists, the DF is loaded and returned. Files are saved as JPEG using imagemagick """ files, _ = loading.discover_images(directory, n_images=-1, v_images=0) batch_x = loading.load_images(files, directory, load='y') batch_x = batch_x['y'].astype(np.float32) / (2**8 - 1) # Get trade-off for JPEG alpha_range = np.arange(1000, 100, -100) quality_levels = np.arange(95, 5, -5) df_jpeg_path = os.path.join(directory, 'diff_jpeg.csv') if os.path.isfile(df_jpeg_path) and not force_calc: print('Restoring diff JPEG stats from {}'.format(df_jpeg_path)) df = pd.read_csv(df_jpeg_path, index_col=False) else: df = pd.DataFrame(columns=[ 'image_id', 'filename', 'codec', 'quality', 'ssim', 'psnr', 'msssim', 'msssim_db', 'bytes', 'bpp' ]) data = dataset.Dataset(directory, n_images=1, v_images=0, val_n_patches=1, load='y') batch_rgb = data.next_training_batch(0, 1, 64) codec_differentiable = jpeg.JPEG(50, codec='soft', trainable=True) with tqdm.tqdm(total=len(files) * len(quality_levels), ncols=120, desc='JPEG') as pbar: for image_id, filename in enumerate(files): for q_i, q in enumerate(quality_levels): codec_differentiable.quality = q # Read the original image image = batch_x[image_id] luma, chroma = codec_differentiable.train_q_table( batch_rgb, alpha=1, beta=100, n_times=20) with open( 'data/diff_jpeg/image1/quantization-table.xml', 'r' ) as input_file, open( 'data/diff_jpeg/image1/quantization-table-diff.xml', 'w') as output_file: for line in input_file: xml_data = input_file.readlines() index_1 = 43 for i in range(0, len(luma.numpy())): q_table_this_line_luma = luma.numpy()[i] xml_data[index_1 + i] = " " + str( int(q_table_this_line_luma[0])) + ", " + str( int(q_table_this_line_luma[1]) ) + ", " + str(int( q_table_this_line_luma[2])) + ", " + str( int(q_table_this_line_luma[3]) ) + ", " + str( int(q_table_this_line_luma[4]) ) + ", " + str( int(q_table_this_line_luma[5]) ) + ", " + str( int(q_table_this_line_luma[6]) ) + ", " + str( int(q_table_this_line_luma[7]) ) + ", " + "\n" index_2 = 68 for i in range(0, len(chroma.numpy())): q_table_this_line_chroma = chroma.numpy()[i] xml_data[index_2 + i] = " " + str( int(q_table_this_line_chroma[0])) + ", " + str( int(q_table_this_line_chroma[1]) ) + ", " + str(int( q_table_this_line_chroma[2])) + ", " + str( int(q_table_this_line_chroma[3]) ) + ", " + str( int(q_table_this_line_chroma[4]) ) + ", " + str( int(q_table_this_line_chroma[5]) ) + ", " + str( int(q_table_this_line_chroma[6]) ) + ", " + str( int(q_table_this_line_chroma[7]) ) + ", " + "\n" output_file.writelines(xml_data) # Compress images and get effective bytes (only image data - no headers) os.system( 'magick convert -quality ' + str(50) + ' -define jpeg:q-table=data/diff_jpeg/image1/quantization-table-diff.xml' + ' ' + directory + '/' + filename + ' ' + directory + '/' + filename + '_' + str(50) + '_compressed.jpeg') if effective_bytes: with open( directory + '/' + filename + "_" + str(50) + '_compressed.jpeg', 'rb') as fh: buf = io.BytesIO(fh.read()) image_bytes = JPEGMarkerStats( buf.getvalue()).get_effective_bytes() else: image_bytes = os.path.getsize(directory + '/' + filename + "_" + str(50) + '_compressed.jpeg') image = imageio.imread(directory + '/' + filename).astype( np.float) / (2**8 - 1) image_compressed = imageio.imread( directory + '/' + filename + "_" + str(50) + '_compressed.jpeg').astype(np.float) / (2**8 - 1) image_compressed_path = directory + '/' + filename + "_" + str( 50) + '_compressed.jpeg' image_path = directory + '/' + filename if not write_files: os.remove(directory + '/' + filename + "_" + str(50) + '_compressed.jpeg') msssim_value = msssim(image, image_compressed, MAX=1).real df = df.append( { 'image_id': image_id, 'filename': filename, 'codec': 'jpeg', 'quality': q, 'ssim': compare_ssim(image, image_compressed, multichannel=True, data_range=1), 'psnr': compare_psnr(image, image_compressed, data_range=1), 'msssim': msssim_value, 'msssim_db': -10 * np.log10(1 - msssim_value), #'perceptual similarity': perceptual_similarity, 'bytes': image_bytes, 'bpp': 8 * image_bytes / image.shape[0] / image.shape[1] }, ignore_index=True) pbar.set_postfix(image_id=image_id, quality=q) pbar.update(1) df.to_csv(os.path.join(directory, 'diff_jpeg.csv'), index=False) return df
def get_jpeg_imagemagick_df(directory, write_files=False, effective_bytes=True, force_calc=False): """ Compute and return (as Pandas DF) the rate distortion curve for JPEG. The result is saved as a CSV file in the source directory. If the file exists, the DF is loaded and returned. Files are saved as JPEG using imageio. """ files, _ = loading.discover_images(directory, n_images=-1, v_images=0) batch_x = loading.load_images(files, directory, load='y') batch_x = batch_x['y'].astype(np.float32) / (2**8 - 1) # Get trade-off for JPEG quality_levels = np.arange(95, 5, -5) df_jpeg_path = os.path.join(directory, 'jpeg.csv') if os.path.isfile(df_jpeg_path) and not force_calc: logger.info('Restoring JPEG stats from {}'.format(df_jpeg_path)) df = pd.read_csv(df_jpeg_path, index_col=False) else: df = pd.DataFrame(columns=[ 'image_id', 'filename', 'codec', 'quality', 'ssim', 'psnr', 'msssim', 'msssim_db', 'bytes', 'bpp' ]) with tqdm.tqdm(total=len(files) * len(quality_levels), ncols=120, desc='JPEG') as pbar: for image_id, filename in enumerate(files): # Read the original image image = batch_x[image_id] for qi, q in enumerate(quality_levels): os.system( 'magick convert -quality ' + str(q) + ' -define jpeg:q-table=data/diff_jpeg/image1/quantization-table.xml' + ' ' + directory + '/' + filename + ' ' + directory + '/' + filename + '_' + str(q) + '_compressed.jpeg') if effective_bytes: with open( directory + '/' + filename + "_" + str(q) + '_compressed.jpeg', 'rb') as fh: buf = io.BytesIO(fh.read()) image_bytes = JPEGMarkerStats( buf.getvalue()).get_effective_bytes() else: image_bytes = os.path.getsize(directory + '/' + filename + "_" + str(q) + '_compressed.jpeg') image = imageio.imread(directory + '/' + filename).astype( np.float) / (2**8 - 1) image_compressed = imageio.imread( directory + '/' + filename + "_" + str(q) + '_compressed.jpeg').astype(np.float) / (2**8 - 1) image_compressed_path = directory + '/' + filename + "_" + str( 50) + '_compressed.jpeg' image_path = directory + '/' + filename if not write_files: os.remove(directory + '/' + filename + "_" + str(q) + '_compressed.jpeg') msssim_value = msssim(image, image_compressed, MAX=1).real df = df.append( { 'image_id': image_id, 'filename': filename, 'codec': 'jpeg', 'quality': q, 'ssim': compare_ssim(image, image_compressed, multichannel=True, data_range=1), 'psnr': compare_psnr(image, image_compressed, data_range=1), 'msssim': msssim_value, 'msssim_db': -10 * np.log10(1 - msssim_value), 'bytes': image_bytes, 'bpp': 8 * image_bytes / image.shape[0] / image.shape[1] }, ignore_index=True) pbar.set_postfix(image_id=image_id, quality=q) pbar.update(1) df.to_csv(os.path.join(directory, 'jpeg.csv'), index=False) return df
def main(): parser = argparse.ArgumentParser( description='Test a neural imaging pipeline') parser.add_argument('plot', help='Plot type ({})'.format( ', '.join(supported_plots))) parser.add_argument( '--data', dest='data', action='store', default='./data/rgb/clic256/', help='directory with training & validation images (png)') parser.add_argument('--images', dest='images', action='store', default=10, type=int, help='number of images to test') parser.add_argument('--image', dest='image_id', action='store', default=1, type=int, help='ID of the image to load') parser.add_argument('--patch', dest='patch_size', action='store', default=128, type=int, help='training patch size') parser.add_argument('--dcn', dest='dcn', action='store', help='directory with a trained DCN model') args = parser.parse_args() # Match the current args.plot = coreutils.match_option(args.plot, supported_plots) if args.plot == 'batch': model, stats = codec.restore_model(args.dcn, args.patch_size, fetch_stats=True) print('Training stats:', stats) data = dataset.IPDataset(args.data, load='y', n_images=0, v_images=args.images, val_rgb_patch_size=args.patch_size) batch_x = data.next_validation_batch(0, args.images) fig = show_example(model, batch_x) plt.show() plt.close() elif args.plot == 'jpeg-match-ssim': files, _ = loading.discover_files(args.data, n_images=-1, v_images=0) files = files[args.image_id:args.image_id + 1] batch_x = loading.load_images(files, args.data, load='y') batch_x = batch_x['y'].astype(np.float32) / (2**8 - 1) model = codec.restore_model(args.dcn, batch_x.shape[1]) fig = match_jpeg(model, batch_x, match='ssim') plt.show() plt.close() elif args.plot == 'jpeg-match-bpp': files, _ = loading.discover_files(args.data, n_images=-1, v_images=0) files = files[args.image_id:args.image_id + 1] batch_x = loading.load_images(files, args.data, load='y') batch_x = batch_x['y'].astype(np.float32) / (2**8 - 1) model = codec.restore_model(args.dcn, batch_x.shape[1]) fig = match_jpeg(model, batch_x, match='bpp') plt.show() plt.close() elif args.plot == 'jpg-trade-off': df = ratedistortion.get_jpeg_df(args.data, write_files=True) print(df.to_string()) elif args.plot == 'jp2-trade-off': df = ratedistortion.get_jpeg2k_df(args.data, write_files=True) print(df.to_string()) elif args.plot == 'dcn-trade-off': df = ratedistortion.get_dcn_df(args.data, args.dcn, write_files=False) print(df.to_string()) elif args.plot == 'bpg-trade-off': df = ratedistortion.get_bpg_df(args.data, write_files=False) print(df.to_string()) else: print('Error: Unknown plot!')
def compare_nips(model_a_dirname, model_b_dirname, camera=None, image=None, patch_size=128, root_dirname='./data', output_dir=None, model_a_args=None, model_b_args=None, extras=False): """ Display a comparison of two variants of a neural imaging pipeline. :param camera: camera name (e.g., 'Nikon D90') :param model_a_dirname: directory with the first variant of the model :param model_b_dirname: directory with the second variant of the model :param ps: patch size (patch will be taken from the middle) :param image_id: index of the test image :param root_dir: root data directory :param output_dir: set an output directory if the figure should be saved (matplotlib2tikz will be used) """ # Lazy imports to minimize delay for invalid command line parameters import re import inspect import imageio as io import matplotlib.pyplot as plt import tensorflow as tf from models import pipelines, tfmodel from helpers import raw, loading supported_cameras = fsutil.listdir( os.path.join(root_dirname, 'models', 'nip'), '.*') supported_pipelines = pipelines.supported_models if patch_size > 0 and (patch_size < 8 or patch_size > 2048): raise ValueError('Patch size seems to be invalid!') if camera is not None and camera not in supported_cameras: raise ValueError( 'Camera data not found ({})! Available cameras: {}'.format( camera, ', '.join(supported_cameras))) # Check if the image is an integer try: image = int(image) except: pass # Construct the NIP models if os.path.isdir(model_a_dirname): # Restore a NIP model from a training log model_a = tfmodel.restore(model_a_dirname, pipelines) else: # Construct the NIP model from class name (and optional arguments) if model_a_args is None: model_a = getattr(pipelines, model_a_dirname)() else: model_a = getattr(pipelines, model_a_dirname)(**model_a_args) model_a.load_model(os.path.join(root_dirname, model_a.model_code)) if os.path.isdir(model_b_dirname): # Restore a NIP model from a training log model_b = tfmodel.restore(model_b_dirname, pipelines) else: # Construct the NIP model from class name (and optional arguments) if model_b_args is None: model_b = getattr(pipelines, model_b_dirname)() else: model_b = getattr(pipelines, model_b_dirname)(**model_b_args) model_b.load_model(os.path.join(root_dirname, model_b.model_code)) print('ISP-A: {}'.format(model_a.summary())) print('ISP-B: {}'.format(model_b.summary())) # Load sample data if isinstance(image, int) and camera is not None: data_dirname = os.path.join(root_dirname, 'raw', 'training_data', camera) files = fsutil.listdir(data_dirname, '.*\.png') files = files[image:image + 1] print('Loading image {} from the training set: {}'.format( image, files)) data = loading.load_images(files, data_dirname) sample_x, sample_y = data['x'].astype(np.float32) / ( 2**16 - 1), data['y'].astype(np.float32) / (2**8 - 1) with open('config/cameras.json') as f: cameras = json.load(f) cfa, srgb = cameras[camera]['cfa'], np.array( cameras[camera]['srgb']) image = files[0] elif image is not None: print('Loading a RAW image {}'.format(image)) sample_x, cfa, srgb, _ = raw.unpack(image, expand=True) sample_y = raw.process(image, brightness=None, expand=True) image = os.path.split(image)[-1] if isinstance(model_a, pipelines.ClassicISP): print('Configuring ISP-A to CFA: {} & sRGB {}'.format( cfa, srgb.round(2).tolist())) model_a.set_cfa_pattern(cfa) model_a.set_srgb_conversion(srgb) if isinstance(model_b, pipelines.ClassicISP): print('Configuring ISP-B to CFA: {} & sRGB {}'.format( cfa, srgb.round(2).tolist())) model_b.set_cfa_pattern(cfa) model_b.set_srgb_conversion(srgb) # Develop images sample_ya = model_a.process(sample_x).numpy() sample_yb = model_b.process(sample_x).numpy() if patch_size > 0: print('Cropping a {p}x{p} patch from the middle'.format(p=patch_size)) xx = (sample_x.shape[2] - patch_size // 2) // 2 yy = (sample_x.shape[1] - patch_size // 2) // 2 sample_x = sample_x[:, yy:yy + patch_size, xx:xx + patch_size, :] sample_y = sample_y[:, 2 * yy:2 * (yy + patch_size), 2 * xx:2 * (xx + patch_size), :] sample_ya = sample_ya[:, 2 * yy:2 * (yy + patch_size), 2 * xx:2 * (xx + patch_size), :] sample_yb = sample_yb[:, 2 * yy:2 * (yy + patch_size), 2 * xx:2 * (xx + patch_size), :] # Plot images fig = imdiff.compare_ab_ref(sample_y, sample_ya, sample_yb, fig=plt.figure(), extras=extras) if output_dir is not None: from tikzplotlib import save as tikz_save dcomp = [ x for x in fsutil.split(model_b_dirname) if re.match('(ln-.*|[0-9]{3})', x) ] tikz_save('{}/examples_{}_{}_{}_{}.tex'.format(output_dir, camera, image, model_a_dirname, model_b_dirname), figureheight='8cm', figurewidth='8cm', strict=False) else: fig.tight_layout() fig.show(fig) fig.suptitle('{}, A={}, B={}'.format(image, model_a.model_code, model_b.model_code)) plt.show() plt.close(fig)