def single_gpu_inference(path, gpu, iteration, offset_folder): assert os.path.exists(path), path meta_graph = '/groups/saalfeld/home/papec/Work/my_projects/nnets/gunpowder-experiments/experiments/cremi-tf/unet_default/unet_checkpoint_%i' % iteration net_io_json = '/groups/saalfeld/home/papec/Work/my_projects/nnets/gunpowder-experiments/experiments/cremi-tf/unet_default/net_io_names.json' with open(net_io_json, 'r') as f: net_io_names = json.load(f) offset_file = os.path.join(os.path.split(os.path.realpath(__file__))[0], offset_folder, 'list_gpu_%i.json' % gpu) with open(offset_file, 'r') as f: offset_list = json.load(f) input_key = net_io_names["raw"] output_key = net_io_names["affs"] input_shape = (84, 268, 268) output_shape = (56, 56, 56) prediction = TensorflowPredict(meta_graph, input_key=input_key, output_key=output_key) t_predict = time.time() run_inference_n5(prediction, preprocess, path, path, offset_list, input_shape=input_shape, output_shape=output_shape, input_key='gray') t_predict = time.time() - t_predict with open(os.path.join(path, 't-inf_gpu%i.txt' % gpu), 'w') as f: f.write("Inference with gpu %i in %f s" % (gpu, t_predict))
def single_gpu_inference(sample, gpu, iteration): raw_path = '/groups/saalfeld/saalfeldlab/larissa/data/cremi/cremi_warped_sample{0:}.n5/volumes'.format( sample) assert os.path.exists( raw_path), "Path to N5 dataset with raw data and mask does not exist" rf = z5py.File(raw_path, use_zarr_format=False) shape = rf['raw'].shape weight_meta_graph = '/nrs/saalfeld/heinrichl/synapses/cremi_all_0116_01/unet_checkpoint_{0:}'.format( iteration) inference_meta_graph = '/nrs/saalfeld/heinrichl/synapses/cremi_all_0116_01/unet_inference' net_io_json = '/nrs/saalfeld/heinrichl/synapses/cremi_all_0116_01/net_io_names.json' out_file = '/nrs/saalfeld/heinrichl/synapses/cremi_all_0116_01/' out_file = os.path.join( out_file, 'prediction_cremi_warped_sample{0:}_{1:}.n5'.format(sample, iteration)) assert os.path.exists(out_file) with open(net_io_json, 'r') as f: net_io_names = json.load(f) input_key = net_io_names["raw"] output_key = net_io_names["dist"] input_shape = (91, 862, 862) output_shape = (71, 650, 650) offset_file = '/nrs/saalfeld/heinrichl/synapses/cremi_all_0116_01/' \ 'offsets_{0:}_{1:}_x{2:}_y{3:}_z{4:}/list_gpu_{5:}.json'.format(sample, iteration, output_shape[0], output_shape[1], output_shape[2], gpu) with open(offset_file, 'r') as f: offset_list = json.load(f) prediction = TensorflowPredict(weight_meta_graph, inference_meta_graph, input_key=input_key, output_key=output_key) t_predict = time.time() run_inference_n5(prediction, preprocess, partial(clip_float_to_uint8, float_range=(-1, 1), safe_scale=False), raw_path, out_file, offset_list, input_key='raw', input_shape_wc=input_shape, output_shape_wc=output_shape, target_keys=('syncleft_dist'), log_processed=os.path.join( os.path.dirname(offset_file), 'list_gpu_{0:}_processed.txt'.format(gpu))) t_predict = time.time() - t_predict with open( os.path.join(os.path.dirname(offset_file), 't-inf_gpu{0:}.txt'.format(gpu)), 'w') as f: f.write("Inference with gpu {0:} in {1:} s\n".format(gpu, t_predict))
def single_gpu_inference(path, data_eval, samples, gpu, iteration): weight_meta_graph = os.path.join(path, 'unet_checkpoint_{0:}'.format(iteration)) inference_meta_graph = os.path.join(path, 'unet_inference') net_io_json = os.path.join(path, 'net_io_names.json') with open(net_io_json, 'r') as f: net_io_names = json.load(f) input_key = net_io_names["raw"] output_key = [ net_io_names["pre_dist"], net_io_names["post_dist"], net_io_names["cleft_dist"] ] input_shape = (91, 862, 862) output_shape = (71, 650, 650) prediction = TensorflowPredict(weight_meta_graph, inference_meta_graph, input_key=input_key, output_key=output_key) t_predict = time.time() for k, de in enumerate(data_eval): for s in samples: print('{0:} ({1:}/{2:}), {3:}'.format(de, k, len(data_eval), s)) raw_file = '/groups/saalfeld/saalfeldlab/larissa/data/cremieval/{0:}/{1:}.n5'.format( de, s) out_file = os.path.join( path, 'evaluation/{0:}/{1:}/{2:}.n5'.format(iteration, de, s)) offset_file = os.path.join(out_file, 'list_gpu_{0:}.json'.format(gpu)) with open(offset_file, 'r') as f: offset_list = json.load(f) run_inference_n5(prediction, preprocess, partial(clip_float_to_uint8, safe_scale=False, float_range=(-1, 1)), raw_file, out_file, offset_list, input_shape=input_shape, output_shape=output_shape, target_keys=('pre_dist', 'post_dist', 'clefts'), input_key='volumes/raw', log_processed=os.path.join( out_file, 'list_gpu_{0:}processed.txt'.format(gpu))) t_predict = time.time() - t_predict with open( os.path.join( os.path.dirname(offset_file), 't-inf_gpu_{0:}_{1:}.txt'.format(gpu, iteration)), 'w') as f: f.write("Inference with gpu %i in %f s\n" % (gpu, t_predict))
def single_gpu_inference(path, network_key, gpu, iteration): assert os.path.exists(path), path net_top_folder = '/groups/saalfeld/home/papec/Work/my_projects/nnets/gunpowder-experiments/new_experiments' net_sub_folder = 'experiments_unet_mala_mask_glia_predict_glia' net_folder = os.path.join(net_top_folder, net_sub_folder) assert os.path.exists(net_folder), net_folder prefix = networks[network_key]['prefix'] graph_weights = os.path.join(net_folder, '%s_checkpoint_%i' % (prefix, iteration)) # we don't use inference model for dtu2 if network_key == 'dtu2': graph_inference = os.path.join(net_folder, prefix) else: graph_inference = os.path.join(net_folder, '%s_inference' % prefix) net_io_json = os.path.join(net_folder, 'net_io_names.json') with open(net_io_json, 'r') as f: net_io_names = json.load(f) offset_file = './offsets/list_gpu_%i.json' % gpu with open(offset_file, 'r') as f: offset_list = json.load(f) input_key = net_io_names["raw"] output_key = net_io_names["affs"] input_shape = networks[network_key]['input_shape'] output_shape = networks[network_key]['output_shape'] prediction = TensorflowPredict(graph_weights, graph_inference, input_key=input_key, output_key=output_key) target_key = 'predictions/affs_glia' t_predict = time.time() run_inference_n5(prediction, preprocess, clip_float_to_uint8, path, path, offset_list, input_shape=input_shape, output_shape=output_shape, input_key='gray', target_keys=target_key, num_cpus=10, channel_order=[list(range(13))]) t_predict = time.time() - t_predict with open(os.path.join(path, 't-inf_gpu%i.txt' % gpu), 'w') as f: f.write("Inference with gpu %i in %f s" % (gpu, t_predict))
def single_gpu_inference(raw_path, out_path, net_folder, gpu, iteration): assert os.path.exists(raw_path) assert os.path.exists(net_folder) rf = z5py.File(raw_path, use_zarr_format=False) shape = rf['data'].shape input_shape = (84, 268, 268) output_shape = (56, 56, 56) # the n5 file might exist already if not os.path.exists(out_path): f = z5py.File(out_path, use_zarr_format=False) f.create_dataset('affs_xy', shape=shape, compressor='gzip', dtype='float32', chunks=output_shape) f.create_dataset('affs_z', shape=shape, compressor='gzip', dtype='float32', chunks=output_shape) # make offset list get_offset_lists(shape, [gpu], './offsets', output_shape=output_shape) meta_graph = os.path.join(net_folder, 'unet_checkpoint_%i' % iteration) net_io_json = os.path.join(net_folder, 'net_io_names.json') with open(net_io_json, 'r') as f: net_io_names = json.load(f) offset_file = './offsets/list_gpu_%i.json' % gpu with open(offset_file, 'r') as f: offset_list = json.load(f) input_key = net_io_names["raw"] output_key = net_io_names["affs"] prediction = TensorflowPredict(meta_graph, input_key=input_key, output_key=output_key) t_predict = time.time() run_inference_n5(prediction, preprocess, raw_path, out_path, offset_list, input_shape_wc=input_shape, output_shape_wc=output_shape) t_predict = time.time() - t_predict print("Running inference in %f s" % t_predict)
def single_gpu_inference(sample, gpu, iteration): path = '/nrs/saalfeld/lauritzen/%s/workspace.n5/raw' % sample assert os.path.exists(path), path rf = z5py.File(path, use_zarr_format=False) shape = rf['gray'].shape weight_meta_graph = '/nrs/saalfeld/heinrichl/synapses/cremi_all_0116_01/unet_checkpoint_%i' % iteration inference_meta_graph = '/nrs/saalfeld/heinrichl/synapses/cremi_all_0116_01/unet_inference' net_io_json = '/nrs/saalfeld/heinrichl/synapses/cremi_all_0116_01/net_io_names.json' out_file = '/nrs/saalfeld/heinrichl/test/lauritzen/%s/workspace.n5' % sample with open(net_io_json, 'r') as f: net_io_names = json.load(f) mhash = hash(path) offset_file = '/nrs/saalfeld/heinrichl/synapses/scott_offsets_{0:}_DTU2_inf/offsets_{' \ '1:}/list_gpu_{2:}.json'.format(sample, mhash, gpu) with open(offset_file, 'r') as f: offset_list = json.load(f) input_key = net_io_names["raw"] output_key = net_io_names["dist"] input_shape = (91, 862, 862) output_shape = (71, 650, 650) prediction = TensorflowPredict(weight_meta_graph, inference_meta_graph, input_key=input_key, output_key=output_key) t_predict = time.time() run_inference_n5(prediction, preprocess, partial(threshold_cc, thr=0., output_shape=output_shape, ds_shape=shape), path, out_file, offset_list, input_shape_wc=input_shape, output_shape_wc=output_shape, target_keys=('syncleft_dist_DTU-2_{0:}'.format(iteration), 'syncleft_cc_DTU-2_{0:}'.format(iteration)), input_key='gray', log_processed=os.path.join( os.path.dirname(offset_file), 'list_gpu_{0:}_{' '1:}_processed.txt'.format(gpu, iteration))) t_predict = time.time() - t_predict with open( os.path.join(os.path.dirname(offset_file), 't-inf_gpu_{0:}_{1:}.txt'.format(gpu, iteration)), 'w') as f: f.write("Inference with gpu %i in %f s\n" % (gpu, t_predict))
def single_gpu_inference(gpu, iteration): raw_path = '/groups/saalfeld/saalfeldlab/larissa/data/fib25/grayscale_sub.h5' save_folder = '/nrs/saalfeld/heinrichl/segmentation/distance_thirdtest/fib25_sub_prediction_at_%i' % iteration meta_graph = '/nrs/saalfeld/heinrichl/segmentation/distance_thirdtest/unet_checkpoint_%i' % iteration net_io_json = '/nrs/saalfeld/heinrichl/segmentation/distance_thirdtest/net_io_names.json' offset_file = '/nrs/saalfeld/heinrichl/segmentation/distance_thirdtest/offsets/list_gpu_%i.json' % gpu with open(offset_file, 'r') as f: offset_list = json.load(f) with open(net_io_json, 'r') as f: net_io_names = json.load(f) prediction = TensorflowPredict(meta_graph, net_io_names['raw'], net_io_names['dist']) t_predict = time.time() run_inference(prediction, preprocess, raw_path, save_folder, offset_list, output_shape=(44,)*3, input_shape=(132, )*3, rejection_criterion=reject_empty_batch, padding_mode='constant') t_predict = time.time() - t_predict with open(os.path.join(save_folder, 't-inf_gpu%i.txt' % gpu), 'w') as f: f.write("Inference with gpu %i in %f s" % (gpu, t_predict)) TensorflowPredict.stop()
def single_gpu_inference(gpu, iteration, gpu_offset): # path to the raw data raw_path = '/nrs/saalfeld/sample_E/sample_E.n5' path_in_file = 'volumes/raw/s0' save_path = '/nrs/saalfeld/sample_E/sample_E.n5' net_folder = '/groups/saalfeld/home/papec/Work/my_projects/nnets/gunpowder-experiments/experiments/cremi-tf' graph_weights = os.path.join(net_folder, 'unet_default/unet_checkpoint_%i' % iteration) graph_inference = os.path.join(net_folder, 'unet_default/unet_inference') net_io_json = os.path.join(net_folder, 'unet_default/net_io_names.json') with open(net_io_json, 'r') as f: net_io_names = json.load(f) offset_file = './offset_lists/list_gpu_%i.json' % (gpu + gpu_offset, ) with open(offset_file, 'r') as f: offset_list = json.load(f) input_shape = (88, 808, 808) output_shape = (60, 596, 596) input_key = net_io_names["raw"] output_key = net_io_names["affs"] prediction = TensorflowPredict(graph_weights, graph_inference, input_key=input_key, output_key=output_key) t_predict = time.time() run_inference_n5(prediction, preprocess, float_to_uint8, raw_path, save_path, offset_list, input_shape, output_shape, input_key=path_in_file, target_keys=['volumes/predictions/full_affs'], num_cpus=10, channel_order=[list(range(12))], log_processed='./processed_gpu_%i.txt' % (gpu + gpu_offset)) t_predict = time.time() - t_predict with open( os.path.join(save_path, 't-inf_gpu%i.txt' % (gpu + gpu_offset, )), 'w') as f: f.write("Inference with gpu %i in %f s" % (gpu, t_predict))
def single_gpu_inference(config, gpu): offset_file = os.path.join( config['meta_path'], 'list_gpu_{0:}{1:}.json'.format(gpu, config['offset_list_name_extension'])) with open(offset_file, 'r') as f: offset_list = json.load(f) print(config['input_key'], config['output_key']) prediction = TensorflowPredict(config['weight_meta_graph'], config['inference_meta_graph'], input_key=config['input_key'], output_key=config['output_key']) if 'postprocess' in config: if config['postprocess'] == 'clip_float32_to_uint8_range_0_1': postprocess = clip_float32_to_uint8_range_0_1 elif config['postprocess'] == 'clip_float32_to_uint8' or config['postprocess'] == \ 'clip_float32_to_uint8_range': #todo remove this dirty bug hack fix postprocess = clip_float32_to_uint8 else: postprocess = clip_float32_to_uint8 t_predict = time.time() run_inference_n5(prediction, preprocess, postprocess, config['raw_path'], config['out_file'], offset_list, input_key=config['data_key'], input_shape=config['input_shape'], output_shape=config['output_shape'], target_keys=config['target_keys'], log_processed=os.path.join( os.path.dirname(offset_file), 'list_gpu_{0:}{1:}_processed.txt'.format( gpu, config['offset_list_name_extension']))) t_predict = time.time() - t_predict with open( os.path.join(os.path.dirname(offset_file), 't-inf_gpu{0:}.txt'.format(gpu)), 'w') as f: f.write("Inference with gpu {0:} in {1:} s\n".format(gpu, t_predict))
def single_gpu_inference(sample, gpu, iteration): raw_path = '/groups/saalfeld/home/papec/Work/neurodata_hdd/cremi_warped/sample%s.n5' % sample assert os.path.exists(raw_path), raw_path out_file = '/groups/saalfeld/home/papec/Work/neurodata_hdd/cremi_warped/sample%s.n5' % sample assert os.path.exists(out_file), out_file net_folder = '/groups/saalfeld/home/papec/Work/my_projects/nnets/gunpowder-experiments/experiments/cremi-tf' graph_weights = os.path.join(net_folder, 'unet_default/unet_checkpoint_%i' % iteration) graph_inference = os.path.join(net_folder, 'unet_default/unet_inference') net_io_json = os.path.join(net_folder, 'unet_default/net_io_names.json') with open(net_io_json, 'r') as f: net_io_names = json.load(f) offset_file = './offsets_sample%s/list_gpu_%i.json' % (sample, gpu) with open(offset_file, 'r') as f: offset_list = json.load(f) input_key = net_io_names["raw"] output_key = net_io_names["affs"] input_shape = (88, 808, 808) output_shape = (60, 596, 596) prediction = TensorflowPredict(graph_weights, graph_inference, input_key=input_key, output_key=output_key) t_predict = time.time() run_inference_n5(prediction, preprocess, float_to_uint8, raw_path, out_file, offset_list, input_key='raw', target_keys='predictions/full_affs', input_shape_wc=input_shape, output_shape_wc=output_shape, channel_order=[list(range(12))]) t_predict = time.time() - t_predict with open(os.path.join(out_file, 't-inf_gpu%i.txt' % gpu), 'w') as f: f.write("Inference with gpu %i in %f s" % (gpu, t_predict))
def single_gpu_inference(path, gpu, iteration): assert os.path.exists(path), path out_file = '/data/heinrichl/sample_E.n5' weight_meta_graph = '/nrs/saalfeld/heinrichl/synapses/cremi_all_0116_01/unet_checkpoint_%i' % iteration inference_meta_graph = '/nrs/saalfeld/heinrichl/synapses/cremi_all_0116_01/unet_inference' net_io_json = '/nrs/saalfeld/heinrichl/synapses/cremi_all_0116_01/net_io_names.json' with open(net_io_json, 'r') as f: net_io_names = json.load(f) offset_file = '/nrs/saalfeld/heinrichl/synapses/sampleE_DTU2_offsets_update/list_gpu_{0:}.json'.format( gpu) with open(offset_file, 'r') as f: offset_list = json.load(f) input_key = net_io_names["raw"] output_key = net_io_names["dist"] input_shape = (91, 862, 862) output_shape = (71, 650, 650) prediction = TensorflowPredict(weight_meta_graph, inference_meta_graph, input_key=input_key, output_key=output_key) t_predict = time.time() run_inference_n5( prediction, preprocess, path, out_file, offset_list, input_shape_wc=input_shape, output_shape_wc=output_shape, target_keys='syncleft_dist_DTU-2_{0:}'.format(iteration), input_key='volumes/raw/s0', log_processed= '/nrs/saalfeld/heinrichl/synapses/sampleE_DTU2_offsets_update/list_gpu_{' '0:}_processed.txt'.format(gpu)) t_predict = time.time() - t_predict with open(os.path.join(out_file, 't-inf_gpu%i.txt' % gpu), 'w') as f: f.write("Inference with gpu %i in %f s" % (gpu, t_predict))
def single_gpu_inference(path, gpu, iteration): assert os.path.exists(path), path net_folder = '/groups/saalfeld/home/papec/Work/my_projects/nnets/gunpowder-experiments/experiments/cremi-tf' graph_weights = os.path.join(net_folder, 'unet_default/unet_checkpoint_%i' % iteration) graph_inference = os.path.join(net_folder, 'unet_default/unet_inference') net_io_json = os.path.join(net_folder, 'unet_default/net_io_names.json') with open(net_io_json, 'r') as f: net_io_names = json.load(f) mhash = hashlib.md5(path.encode('utf-8')).hexdigest() offset_file = './offsets_%s/list_gpu_%i.json' % (mhash, gpu) with open(offset_file, 'r') as f: offset_list = json.load(f) input_key = net_io_names["raw"] output_key = net_io_names["affs"] input_shape = (88, 808, 808) output_shape = (60, 596, 596) prediction = TensorflowPredict(graph_weights, graph_inference, input_key=input_key, output_key=output_key) t_predict = time.time() run_inference_n5(prediction, preprocess, path, path, offset_list, input_shape_wc=input_shape, output_shape_wc=output_shape, input_key='gray', target_keys='predictions/full_affs', full_affinities=True) t_predict = time.time() - t_predict with open(os.path.join(path, 't-inf_gpu%i.txt' % gpu), 'w') as f: f.write("Inference with gpu %i in %f s" % (gpu, t_predict))
def single_gpu_inference(data_train, augmentation, data_eval, samples, gpu, iteration): path = os.path.join( config_loader.get_config()["synapses"]["training_setups_path"], "data_and_augmentations/{0:}/{1:}".format(data_train, augmentation)) weight_meta_graph = os.path.join(path, "unet_checkpoint_{0:}".format(iteration)) inference_meta_graph = os.path.join(path, "unet_inference") net_io_json = os.path.join(path, "net_io_names.json") with open(net_io_json, "r") as f: net_io_names = json.load(f) input_key = net_io_names["raw"] output_key = [ net_io_names["pre_dist"], net_io_names["post_dist"], net_io_names["cleft_dist"], ] input_shape = (91 * 40, 862 * 4, 862 * 4) output_shape = (71 * 40, 650 * 4, 650 * 4) prediction = TensorflowPredict( weight_meta_graph, inference_meta_graph, input_keys=input_key, output_keys=output_key, ) t_predict = time.time() for k, de in enumerate(data_eval): for s in samples: print("{0:} ({1:}/{2:}), {3:}".format(de, k, len(data_eval), s)) raw_file = os.path.join( config_loader.get_config()["synapses"]["cremieval_path"], "{0:}/{1:}.n5".format(de, s)) out_file = os.path.join( config_loader.get_config()["synapses"]["cremieval_path"], "data_and_augmentations/{0:}/{1:}/evaluation/{2:}/{3:}/{4:}.n5" .format(data_train, augmentation, iteration, de, s)) offset_file = os.path.join(out_file, "list_gpu_{0:}.json".format(gpu)) with open(offset_file, "r") as f: offset_list = json.load(f) run_inference_zarr(prediction, preprocess, partial(clip_float_to_uint8, safe_scale=False, float_range=(-1, 1)), raw_file, out_file, offset_list, input_shape_wc=input_shape, output_shape_wc=output_shape, target_keys=("pre_dist", "post_dist", "clefts"), input_key="volumes/raw", input_resolution=(40, 4, 4), target_resolution=(40, 4, 4), log_processed=os.path.join( out_file, "list_gpu_{0:}processed.txt".format(gpu))) t_predict = time.time() - t_predict with open( os.path.join( os.path.dirname(offset_file), "t-inf_gpu_{0:}_{1:}.txt".format(gpu, iteration), ), "w", ) as f: f.write("Inference with gpu %i in %f s\n" % (gpu, t_predict))
def single_gpu_inference(data_train, augmentation, data_eval, samples, gpu, iteration): path = "/nrs/saalfeld/heinrichl/synapses/data_and_augmentations/{0:}/{1:}".format( data_train, augmentation) weight_meta_graph = os.path.join(path, "unet_checkpoint_{0:}".format(iteration)) inference_meta_graph = os.path.join(path, "unet_inference") net_io_json = os.path.join(path, "net_io_names.json") with open(net_io_json, "r") as f: net_io_names = json.load(f) input_key = net_io_names["raw"] output_key = [ net_io_names["pre_dist"], net_io_names["post_dist"], net_io_names["cleft_dist"], ] input_shape = (91, 862, 862) output_shape = (71, 650, 650) prediction = TensorflowPredict( weight_meta_graph, inference_meta_graph, input_key=input_key, output_key=output_key, ) t_predict = time.time() for k, de in enumerate(data_eval): for s in samples: print("{0:} ({1:}/{2:}), {3:}".format(de, k, len(data_eval), s)) raw_file = "/groups/saalfeld/saalfeldlab/larissa/data/cremieval/{0:}/{1:}.n5".format( de, s) out_file = ( "/nrs/saalfeld/heinrichl/synapses/data_and_augmentations/{0:}/{" "1:}/evaluation/{2:}/{3:}/{4:}.n5".format( data_train, augmentation, iteration, de, s)) offset_file = os.path.join(out_file, "list_gpu_{0:}.json".format(gpu)) with open(offset_file, "r") as f: offset_list = json.load(f) run_inference_n5( prediction, preprocess, partial(clip_float_to_uint8, safe_scale=False, float_range=(-1, 1)), raw_file, out_file, offset_list, input_shape=input_shape, output_shape=output_shape, target_keys=("pre_dist", "post_dist", "clefts"), input_key="volumes/raw", log_processed=os.path.join( out_file, "list_gpu_{0:}processed.txt".format(gpu)), ) t_predict = time.time() - t_predict with open( os.path.join( os.path.dirname(offset_file), "t-inf_gpu_{0:}_{1:}.txt".format(gpu, iteration), ), "w", ) as f: f.write("Inference with gpu %i in %f s\n" % (gpu, t_predict))
def single_job_inference(job_no, raw_data_path, sigma, raw_ds, setup_path, output_path=None, factor=None, min_sc=None, max_sc=None, float_range=(-1, 1), safe_scale=False, n_cpus=5): output_dir, out_file = get_output_paths(raw_data_path, setup_path, output_path) offset_file = os.path.join(out_file, "list_gpu_{0:}.json".format(job_no)) if not os.path.exists(offset_file): return with open(offset_file, 'r') as f: offset_list = json.load(f) rf = zarr.open(raw_data_path, mode="r") shape_vc = rf[raw_ds].shape weight_meta_graph = os.path.join(setup_path, "blur_{0:}".format(float(sigma))) inference_meta_graph = os.path.join(setup_path, "blur_{0:}".format(float(sigma))) net_io_json = os.path.join(setup_path, "net_io_names.json") with open(net_io_json, "r") as f: net_io_names = json.load(f) shapes_file = os.path.join( setup_path, "shapes_steps_{0:}x{1:}x{2:}.json".format(*size)) with open(shapes_file, "r") as f: shapes = json.load(f) input_shape_vc, output_shape_vc, chunk_shape_vc = \ shapes["input_shape_vc"], shapes["output_shape_vc"], shapes["chunk_shape_vc"] input_key = net_io_names["raw_input"] network_output_keys = net_io_names["output"] dataset_target_keys = ["raw_blurred"] input_shape_wc = Coordinate(input_shape_vc) * voxel_size output_shape_wc = Coordinate(output_shape_vc) * voxel_size chunk_shape_wc = Coordinate(chunk_shape_vc) * voxel_size prediction = TensorflowPredict(weight_meta_graph, inference_meta_graph, input_keys=input_key, output_keys=network_output_keys, has_trained_variables=False) t_predict = time.time() factor, scale, shift = get_contrast_adjustment(rf, raw_ds, factor, min_sc, max_sc) run_inference_zarr_multi_crop( prediction, functools.partial(preprocess, factor=1. / factor, scale=scale, shift=shift), functools.partial(clip_float_to_uint8, float_range=float_range, safe_scale=safe_scale), raw_data_path, out_file, offset_list, network_input_shapes_wc=[ input_shape_wc, ], network_output_shape_wc=output_shape_wc, chunk_shape_wc=chunk_shape_wc, input_keys=[ raw_ds, ], target_keys=dataset_target_keys, input_resolutions=[ tuple(voxel_size), ], target_resolutions=[ tuple(voxel_size), ] * len(dataset_target_keys), log_processed=os.path.join( os.path.dirname(offset_file), "list_gpu_{0:}_{1:}_processed.txt".format(job_no, sigma)), pad_value=int(round(-factor * (shift / scale))), num_cpus=n_cpus) t_predict = time.time() - t_predict
def single_job_inference(job_no: int, raw_data_path: str, iteration: int, raw_ds: str, setup_path: str, output_path: Optional[str] = None, factor: Optional[int] = None, min_sc: Optional[int] = None, max_sc: Optional[int] = None, float_range: Tuple[int, int] = (-1, 1), safe_scale: bool = False, n_cpus: int = 5) -> None: """ Run an inference job. Args: job_no: Id for the inference job. raw_data_path: Path to n5 container that contains raw data. iteration: Iteration to pull inference for. raw_ds: Dataset in n5 container (`raw_data_path`) for raw data. setup_path: Path containing setup. output_path: N5 container to save output to, autogenerated if None. factor: Factor to normalize raw data by, tried to infer from datatype if None. min_sc: Minimum intensity (mapped to -1) max_sc: Maximum intensity (mapped to 1) float_range: Range of output floats for conversion to uint8. safe_scale: If True, values are scaled such that all values within `float_range` fall within (0, 255). and are not cropped. If False, values at the lower end of `float_range` may be scaled to < 0 and then cropped to 0. n_cpus: Number of cpus to use per job. """ sys.path.append(setup_path) import unet_template output_dir, out_file = get_output_paths(raw_data_path, setup_path, output_path, iteration) offset_file = os.path.join(out_file, "list_gpu_{0:}.json".format(job_no)) if not os.path.exists(offset_file): return with open(offset_file, 'r') as f: offset_list = json.load(f) rf = zarr.open(raw_data_path, mode="r") shape_vc = rf[raw_ds].shape weight_meta_graph = os.path.join( setup_path, "unet_train_checkpoint_{0:}".format(iteration)) inference_meta_graph = os.path.join(setup_path, "unet_inference") net_io_json = os.path.join(setup_path, "net_io_names.json") with open(net_io_json, "r") as f: net_io_names = json.load(f) shapes_file = os.path.join( setup_path, "shapes_steps{0:}.json".format(unet_template.steps_inference)) with open(shapes_file, "r") as f: shapes = json.load(f) input_shape_vc, output_shape_vc, chunk_shape_vc = \ shapes["input_shape_vc"], shapes["output_shape_vc"], shapes["chunk_shape_vc"] input_key = net_io_names["raw"] network_output_keys = [] dataset_target_keys = [] for label in unet_template.labels: network_output_keys.append(net_io_names[label.labelname]) dataset_target_keys.append(label.labelname) voxel_size_input = unet_template.voxel_size_input voxel_size_output = unet_template.voxel_size input_shape_wc = Coordinate(input_shape_vc) * voxel_size_input output_shape_wc = Coordinate(output_shape_vc) * voxel_size_output chunk_shape_wc = Coordinate(chunk_shape_vc) * voxel_size_output prediction = TensorflowPredict(weight_meta_graph, inference_meta_graph, input_keys=input_key, output_keys=network_output_keys) factor, scale, shift = get_contrast_adjustment(rf, raw_ds, factor, min_sc, max_sc) t_predict = time.time() run_inference_zarr_multi_crop( prediction, functools.partial(preprocess, factor=1. / factor, scale=scale, shift=shift), functools.partial(clip_float_to_uint8, float_range=float_range, safe_scale=safe_scale), raw_data_path, out_file, offset_list, network_input_shapes_wc=[ input_shape_wc, ], network_output_shape_wc=output_shape_wc, chunk_shape_wc=chunk_shape_wc, input_keys=[ raw_ds, ], target_keys=dataset_target_keys, input_resolutions=[ tuple(voxel_size_input), ], target_resolutions=[ tuple(voxel_size_output), ] * len(dataset_target_keys), log_processed=os.path.join( os.path.dirname(offset_file), "list_gpu_{0:}_{1:}_processed.txt".format(job_no, iteration)), pad_value=int(round(-factor * (shift / scale))), num_cpus=n_cpus) t_predict = time.time() - t_predict logging.info("completed {0:} blocks in {1:}s.".format( len(offset_list), t_predict))
def single_gpu_inference(gpu, iteration, list_extension=''): raw_path = '/groups/saalfeld/saalfeldlab/FAFB00/v14_align_tps_20170818_dmg.n5/volumes/raw/' assert os.path.exists(raw_path), raw_path weight_meta_graph = '/nrs/saalfeld/heinrichl/synapses/cremi_all_0116_01/unet_checkpoint_%i' % iteration inference_meta_graph = '/nrs/saalfeld/heinrichl/synapses/cremi_all_0116_01/unet_inference' net_io_json = '/nrs/saalfeld/heinrichl/synapses/cremi_all_0116_01/net_io_names.json' with open(net_io_json, 'r') as f: net_io_names = json.load(f) out_file = '/nrs/saalfeld/FAFB00/v14_align_tps_20170818_dmg.n5/volumes/predictions/synapses_dt' metadata_folder = '/nrs/saalfeld/heinrichl/fafb_meta/' offset_file = os.path.join( metadata_folder, 'list_gpu_{0:}{1:}.json'.format(gpu, list_extension)) processed_file = os.path.join( metadata_folder, 'list_gpu_{0:}_processed{1:}.txt'.format(gpu, list_extension)) with open(offset_file, 'r') as f: offset_list = json.load(f) if os.path.exists(processed_file): curated_processed_file = curate_processed_list(processed_file) with open(curated_processed_file, 'r') as f: processed_list = json.load(f)[:-1] processed_list_set = set(tuple(coo) for coo in processed_list) else: processed_list_set = set() offset_list_set = set(tuple(coo) for coo in offset_list) if processed_list_set == offset_list_set: print("processing was complete") return assert processed_list_set < offset_list_set offset_list = [list(coo) for coo in offset_list_set - processed_list_set] input_key = net_io_names["raw"] output_key = net_io_names["dist"] input_shape = (91, 862, 862) output_shape = (71, 650, 650) prediction = TensorflowPredict(weight_meta_graph, inference_meta_graph, input_key=input_key, output_key=output_key) t_predict = time.time() run_inference_n5(prediction, preprocess, partial(clip_float_to_uint8, float_range=(-1, 1), safe_scale=False), raw_path, out_file, offset_list, input_shape=input_shape, output_shape=output_shape, target_keys='s0', input_key='s0', log_processed=processed_file) t_predict = time.time() - t_predict with open( os.path.join(metadata_folder, 't-inf_gpu_{0:}{1:}.txt'.format(gpu, list_extension)), 'w') as f: f.write("Inference with gpu %i in %f s\n" % (gpu, t_predict))
def single_job_inference(job_no, raw_data_path, iteration, raw_ds, setup_path, output_path=None, factor=None, min_sc=None, max_sc=None, float_range=(-1, 1), safe_scale=False, n_cpus=5): sys.path.append(setup_path) import setup_config output_dir, out_file = get_output_paths(raw_data_path, setup_path, output_path) offset_file = os.path.join(out_file, "list_gpu_{0:}.json".format(job_no)) if not os.path.exists(offset_file): return with open(offset_file, 'r') as f: offset_list = json.load(f) rf = zarr.open(raw_data_path, mode="r") shape_vc = rf[raw_ds].shape weight_meta_graph = os.path.join( setup_path, "{0:}_train_checkpoint_{1:}".format(setup_config.network_name, iteration)) inference_meta_graph = os.path.join( setup_path, "{0:}_inference".format(setup_config.network_name)) net_io_json = os.path.join( setup_path, "{0:}_io_names.json".format(setup_config.network_name)) with open(net_io_json, "r") as f: net_io_names = json.load(f) shapes_file = os.path.join( setup_path, "shapes_steps{0:}.json".format(setup_config.steps_inference)) with open(shapes_file, "r") as f: shapes = json.load(f) input_shape_vc, output_shape_vc, chunk_shape_vc = \ shapes["input_shape_vc"], shapes["output_shape_vc"], shapes["chunk_shape_vc"] input_key = net_io_names[setup_config.input_name] network_output_keys = [] dataset_target_keys = [] for out_name in setup_config.output_names: network_output_keys.append(net_io_names[out_name + "_predicted"]) dataset_target_keys.append(out_name + "_predicted") voxel_size_input = setup_config.voxel_size voxel_size_output = setup_config.voxel_size input_shape_wc = Coordinate(input_shape_vc) * voxel_size_input output_shape_wc = Coordinate(output_shape_vc) * voxel_size_output chunk_shape_wc = Coordinate(chunk_shape_vc) * voxel_size_output prediction = TensorflowPredict(weight_meta_graph, inference_meta_graph, input_keys=input_key, output_keys=network_output_keys) t_predict = time.time() factor, scale, shift = get_contrast_adjustment(rf, raw_ds, factor, min_sc, max_sc) run_inference_zarr_multi_crop( prediction, functools.partial(preprocess, factor=1. / factor, scale=scale, shift=shift), functools.partial(clip_float_to_uint8, float_range=float_range, safe_scale=safe_scale), raw_data_path, out_file, offset_list, network_input_shapes_wc=[ input_shape_wc, ], network_output_shape_wc=output_shape_wc, chunk_shape_wc=chunk_shape_wc, input_keys=[ raw_ds, ], target_keys=dataset_target_keys, input_resolutions=[ tuple(voxel_size_input), ], target_resolutions=[ tuple(voxel_size_output), ] * len(dataset_target_keys), log_processed=os.path.join( os.path.dirname(offset_file), "list_gpu_{0:}_{1:}_processed.txt".format(job_no, iteration)), pad_value=int(round(-factor * (shift / scale))), num_cpus=n_cpus) t_predict = time.time() - t_predict