def complete_inference(sample, gpu_list):
# path to the raw data
raw_path = '/groups/saalfeld/home/papec/Work/neurodata_hdd/cremi_warped/sample%s_inference.n5' % sample
assert os.path.exists(raw_path), raw_path
rf = z5py.File(raw_path, use_zarr_format=False)
shape = rf['data'].shape
# create the datasets
output_shape = (32, 320, 320)
out_file = '/groups/saalfeld/home/papec/Work/neurodata_hdd/networks/neurofire/mws/unet-1/Predictions/prediction_sample%s.n5' % sample
# the n5 file might exist already
f = z5py.File(out_file, use_zarr_format=False)
if 'full_affs' not in f:
# chunks = (3,) + tuple(outs // 2 for outs in output_shape)
chunks = (3,) + output_shape
aff_shape = (19,) + shape
f.create_dataset('full_affs', shape=aff_shape,
compression='gzip', dtype='float32', chunks=chunks)
# make the offset files, that assign blocks to gpus
save_folder = './offsets_sample%s' % sample
get_offset_lists(shape, gpu_list, save_folder, output_shape=output_shape)
# run multiprocessed inference
with ProcessPoolExecutor(max_workers=len(gpu_list)) as pp:
tasks = [pp.submit(single_inference, sample, gpu) for gpu in gpu_list]
result = [t.result() for t in tasks]
if all(result):
print("All gpu's finished inference properly.")
else:
print("WARNING: at least one process didn't finish properly.")
def run_inference(input_file, input_key, output_file, output_key, checkpoint,
n_gpus):
out_blocks = (65, 675, 675)
chunks = (1, ) + out_blocks
with h5py.File(input_file) as f:
shape = f[input_key].shape
aff_shape = (3, ) + shape
f = z5py.N5File(output_file)
f.require_dataset(output_key,
shape=aff_shape,
chunks=chunks,
compression='gzip',
dtype='uint8')
get_offset_lists(shape,
list(range(n_gpus)),
'./offsets',
output_shape=out_blocks)
with futures.ProcessPoolExecutor(n_gpus) as pp:
tasks = [
pp.submit(single_inference, input_file, input_key, output_file,
output_key, checkpoint, gpu_id)
for gpu_id in range(n_gpus)
]
[t.result() for t in tasks]
evaluate_bench()
def single_gpu_inference(raw_path,
out_path,
net_folder,
gpu,
iteration):
assert os.path.exists(raw_path)
assert os.path.exists(net_folder)
rf = z5py.File(raw_path, use_zarr_format=False)
shape = rf['data'].shape
input_shape = (84, 268, 268)
output_shape = (56, 56, 56)
# the n5 file might exist already
if not os.path.exists(out_path):
f = z5py.File(out_path, use_zarr_format=False)
f.create_dataset('affs_xy', shape=shape,
compressor='gzip',
dtype='float32',
chunks=output_shape)
f.create_dataset('affs_z', shape=shape,
compressor='gzip',
dtype='float32',
chunks=output_shape)
# make offset list
get_offset_lists(shape, [gpu], './offsets', output_shape=output_shape)
meta_graph = os.path.join(net_folder, 'unet_checkpoint_%i' % iteration)
net_io_json = os.path.join(net_folder, 'net_io_names.json')
with open(net_io_json, 'r') as f:
net_io_names = json.load(f)
offset_file = './offsets/list_gpu_%i.json' % gpu
with open(offset_file, 'r') as f:
offset_list = json.load(f)
input_key = net_io_names["raw"]
output_key = net_io_names["affs"]
prediction = TensorflowPredict(meta_graph,
input_key=input_key,
output_key=output_key)
t_predict = time.time()
run_inference_n5(prediction,
preprocess,
raw_path,
out_path,
offset_list,
input_shape_wc=input_shape,
output_shape_wc=output_shape)
t_predict = time.time() - t_predict
print("Running inference in %f s" % t_predict)
def complete_inference(sample, gpu_list, iteration):
# path to the raw data
raw_path = '/groups/saalfeld/home/papec/Work/neurodata_hdd/cremi_warped/n5/cremi_warped_sample%s.n5' % sample
rf = z5py.File(raw_path, use_zarr_format=False)
shape = rf['data'].shape
# create the datasets
out_shape = (56,) *3
out_file = '/groups/saalfeld/home/papec/Work/neurodata_hdd/cremi_warped/gp_caffe_predictions_iter_%i' % iteration
if not os.path.exists(out_file):
os.mkdir(out_file)
out_file = os.path.join(out_file, 'cremi_warped_sample%s_predictions_blosc.n5' % sample)
# the n5 file might exist already
if not os.path.exists(out_file):
f = z5py.File(out_file, use_zarr_format=False)
f.create_dataset('affs_xy',
dtype='float32',
shape=shape,
chunks=out_shape,
compression='blosc',
codec='lz4',
shuffle=1)
f.create_dataset('affs_z',
dtype='float32',
shape=shape,
chunks=out_shape,
compression='blosc',
codec='lz4',
shuffle=1)
# make the offset files, that assign blocks to gpus
save_folder = './offsets_sample%s' % sample
output_shape = (56, 56, 56)
get_offset_lists(shape, gpu_list, save_folder, output_shape=output_shape)
# run multiprocessed inference
with ProcessPoolExecutor(max_workers=len(gpu_list)) as pp:
tasks = [pp.submit(single_inference, sample, gpu, iteration) for gpu in gpu_list]
result = [t.result() for t in tasks]
if all(result):
print("All gpu's finished inference properly.")
else:
print("WARNING: at least one process didn't finish properly.")
def complete_inference(sample, gpu_list, iteration):
# path to the raw data
raw_path = '/groups/saalfeld/saalfeldlab/larissa/data/cremi/cremi_warped_sample{0:}.n5/volumes/'.format(
sample)
assert os.path.exists(
raw_path), "Path to N5 dataset with raw data and mask does not exist"
rf = z5py.File(raw_path, use_zarr_format=False)
shape = rf['raw'].shape
# create the datasets
out_shape = (71, 650, 650)
out_file = '/nrs/saalfeld/heinrichl/synapses/cremi_all_0116_01/'
if not os.path.exists(out_file):
os.mkdir(out_file)
out_file = os.path.join(
out_file,
'prediction_cremi_warped_sample{0:}_{1:}.n5'.format(sample, iteration))
f = z5py.File(out_file, use_zarr_format=False)
f.create_dataset('syncleft_dist',
shape=shape,
compression='gzip',
dtype='uint8',
chunks=out_shape)
# make the offset files, that assign blocks to gpus
offset_folder = '/nrs/saalfeld/heinrichl/synapses/cremi_all_0116_01/offsets_{0:}_{1:}_x{2:}_y{3:}_z{4:}/'.format(
sample, iteration, out_shape[0], out_shape[1], out_shape[2])
if not os.path.exists(offset_folder):
os.mkdir(offset_folder)
get_offset_lists(shape, gpu_list, offset_folder, output_shape=out_shape)
# run multiprocessed inference
with ProcessPoolExecutor(max_workers=len(gpu_list)) as pp:
tasks = [
pp.submit(single_inference, sample, gpu, iteration)
for gpu in gpu_list
]
result = [t.result() for t in tasks]
if all(result):
print("All gpu's finished inference properly.")
else:
print("WARNING: at least one process didn't finish properly.")
def run(self):
src = '/groups/saalfeld/saalfeldlab/larissa/data/cremieval/{0:}/{1:}.n5'
tgt = os.path.join(os.path.dirname(self.input().fn), '{0:}', '{1:}.n5')
output_shape = (71, 650, 650)
gpu_list = []
for i in range(8):
nvsmi = subprocess.Popen("nvidia-smi -d PIDS -q -i {0:}".format(i),
shell=True,
stdout=subprocess.PIPE).stdout.read()
if 'None' in nvsmi:
gpu_list.append(i)
completed = []
for de in self.data_eval:
for s in self.samples:
srcf = z5py.File(src.format(de, s), use_zarr_format=False)
shape = srcf['volumes/raw'].shape
tgtf = z5py.File(tgt.format(de, s), use_zarr_format=False)
if not os.path.exists(os.path.join(tgt.format(de, s),
'clefts')):
tgtf.create_dataset('clefts',
shape=shape,
compression='gzip',
dtype='uint8',
chunks=output_shape)
completed.append(False)
else:
if self.check_completeness()[0]:
completed.append(True)
else:
completed.append(False)
if not os.path.exists(
os.path.join(tgt.format(de, s), 'pre_dist')):
tgtf.create_dataset('pre_dist',
shape=shape,
compression='gzip',
dtype='uint8',
chunks=output_shape)
completed.append(False)
else:
if self.check_completeness()[0]:
completed.append(True)
else:
completed.append(False)
if not os.path.exists(
os.path.join(tgt.format(de, s), 'post_dist')):
tgtf.create_dataset('post_dist',
shape=shape,
compression='gzip',
dtype='uint8',
chunks=output_shape)
completed.append(False)
else:
if self.check_completeness()[0]:
completed.append(True)
else:
completed.append(False)
get_offset_lists(shape,
gpu_list,
tgt.format(de, s),
output_shape=output_shape)
if all(completed):
self.finish()
return
self.submit_inference(self.data_eval, gpu_list)
reprocess_attempts = 0
while reprocess_attempts < 4:
complete, reprocess_list = self.check_completeness(gpu_list)
if complete:
self.finish()
return
else:
self.set_status_message("Reprocessing {0:}, try {1:}".format(
list(reprocess_list), reprocess_attempts))
self.submit_inference(tuple(reprocess_list), gpu_list)
reprocess_attempts += 1
if reprocess_attempts >= 4:
raise AssertionError
def run(self):
src = os.path.join(
config_loader.get_config()["synapses"]["cremieval_path"],
"{0:}/{1:}.n5")
tgt = os.path.join(os.path.dirname(self.input().fn), "{0:}", "{1:}.n5")
output_shape = (71, 650, 650)
gpu_list = []
for i in range(8):
nvsmi = subprocess.Popen(
"nvidia-smi -d PIDS -q -i {0:}".format(i),
shell=True,
stdout=subprocess.PIPE,
).stdout.read()
if "None" in nvsmi:
gpu_list.append(i)
completed = []
for de in self.data_eval:
for s in self.samples:
srcf = zarr.open(src.format(de, s), mode="r")
shape = srcf["volumes/raw"].shape
tgtf = zarr.open(tgt.format(de, s), mode="a")
if not os.path.exists(os.path.join(tgt.format(de, s),
"clefts")):
tgtf.empty(
name="clefts",
shape=shape,
compressor=numcodecs.GZip(6),
dtype="uint8",
chunks=output_shape,
)
completed.append(False)
else:
if self.check_completeness()[0]:
completed.append(True)
else:
completed.append(False)
if not os.path.exists(
os.path.join(tgt.format(de, s), "pre_dist")):
tgtf.empty(
name="pre_dist",
shape=shape,
compressor=numcodecs.GZip(6),
dtype="uint8",
chunks=output_shape,
)
completed.append(False)
else:
if self.check_completeness()[0]:
completed.append(True)
else:
completed.append(False)
if not os.path.exists(
os.path.join(tgt.format(de, s), "post_dist")):
tgtf.empty(
name="post_dist",
shape=shape,
compressor=numcodecs.GZip(6),
dtype="uint8",
chunks=output_shape,
)
completed.append(False)
else:
if self.check_completeness()[0]:
completed.append(True)
else:
completed.append(False)
get_offset_lists(shape,
gpu_list,
tgt.format(de, s),
output_shape=output_shape)
if all(completed):
self.finish()
return
self.submit_inference(self.data_eval, gpu_list)
reprocess_attempts = 0
while reprocess_attempts < 4:
complete, reprocess_list = self.check_completeness(gpu_list)
if complete:
self.finish()
return
else:
self.set_status_message("Reprocessing {0:}, try {1:}".format(
list(reprocess_list), reprocess_attempts))
self.submit_inference(tuple(reprocess_list), gpu_list)
reprocess_attempts += 1
if reprocess_attempts >= 4:
raise AssertionError