def skeletonize_in_subprocess(config, id_box_mask_factor_err):
"""
Execute skeletonize() in a subprocess, and handle TimeoutErrors.
"""
logger = logging.getLogger(__name__ + '.skeletonize')
logger.setLevel(logging.WARN)
timeout = config['options']['analysis-timeout']
body_id, combined_box, combined_mask, downsample_factor, _err_msg = id_box_mask_factor_err
try:
func = execute_in_subprocess(timeout, logger)(skeletonize)
body_id, swc = func(config, body_id, combined_box, combined_mask, downsample_factor)
return (body_id, swc, None)
except TimeoutError:
err_msg = f"Timeout ({timeout}) while skeletonizing body: id={body_id} box={combined_box.tolist()}"
logger.error(err_msg)
output_dir = config['options']['failed-mask-dir']
if output_dir:
os.makedirs(output_dir, exist_ok=True)
output_path = output_dir + f'/failed-skeleton-{body_id}.h5'
logger.error(f"Writing mask to {output_path}")
import h5py
with h5py.File(output_path, 'w') as f:
f["downsample_factor"] = downsample_factor
f["box"] = combined_box
f.create_dataset("mask", data=combined_mask, chunks=True)
return (body_id, None, err_msg)
def test_timeout(self):
try:
_result = execute_in_subprocess(1.0, logger)(_test_helper)(1,2,3.0)
except TimeoutError:
pass
else:
assert False, "Expected a timeout error."
def test_timeout_in_C_function(self):
try:
_result = execute_in_subprocess(1.0, logger)(c_sleep)()
except TimeoutError:
pass
else:
assert False, "Expected a timeout error."
def combine_masks_in_subprocess(config, ids_and_boxes_and_compressed_masks):
"""
Execute combine_masks() in a subprocess, and handle TimeoutErrors.
"""
logger = logging.getLogger(__name__ + '.combine_masks')
logger.setLevel(logging.WARN)
timeout = config['options']['downsample-timeout']
body_id, boxes_and_compressed_masks = ids_and_boxes_and_compressed_masks
if config["options"]["downsample-in-subprocess"]:
func = execute_in_subprocess(timeout, logger)(combine_masks)
else:
func = combine_masks
try:
body_id, combined_box, combined_mask_downsampled, chosen_downsample_factor = func(config, body_id, boxes_and_compressed_masks)
return (body_id, combined_box, combined_mask_downsampled, chosen_downsample_factor, None)
except TimeoutError: # fyi: can't be raised unless a subprocessed is used
boxes, _compressed_masks, counts = zip(*boxes_and_compressed_masks)
total_count = sum(counts)
boxes = np.asarray(boxes)
combined_box = np.zeros((2,3), dtype=np.int64)
combined_box[0] = boxes[:, 0, :].min(axis=0)
combined_box[1] = boxes[:, 1, :].max(axis=0)
err_msg = f"Timeout ({timeout}) while downsampling/assembling body: id={body_id} size={total_count} box={combined_box.tolist()}"
logger.error(err_msg)
return (body_id, combined_box, None, None, err_msg)
def skeletonize_in_subprocess(config, id_box_mask_factor_err):
"""
Execute skeletonize() in a subprocess, and handle TimeoutErrors.
"""
logger = logging.getLogger(__name__ + '.skeletonize')
logger.setLevel(logging.WARN)
timeout = config['options']['analysis-timeout']
body_id, combined_box, combined_mask, downsample_factor, _err_msg = id_box_mask_factor_err
try:
func = execute_in_subprocess(timeout, logger)(skeletonize)
body_id, swc = func(config, body_id, combined_box, combined_mask,
downsample_factor)
return (body_id, swc, None)
except TimeoutError:
err_msg = f"Timeout ({timeout}) while skeletonizing body: id={body_id} box={combined_box.tolist()}"
logger.error(err_msg)
output_dir = config['options']['failed-mask-dir']
if output_dir:
os.makedirs(output_dir, exist_ok=True)
output_path = output_dir + f'/failed-skeleton-{body_id}.h5'
logger.error(f"Writing mask to {output_path}")
import h5py
with h5py.File(output_path, 'w') as f:
f["downsample_factor"] = downsample_factor
f["box"] = combined_box
f.create_dataset("mask", data=combined_mask, chunks=True)
return (body_id, None, err_msg)
def combine_masks_in_subprocess(config, ids_and_boxes_and_compressed_masks):
"""
Execute combine_masks() in a subprocess, and handle TimeoutErrors.
"""
logger = logging.getLogger(__name__ + '.combine_masks')
logger.setLevel(logging.WARN)
timeout = config['options']['downsample-timeout']
body_id, boxes_and_compressed_masks = ids_and_boxes_and_compressed_masks
if config["options"]["downsample-in-subprocess"]:
func = execute_in_subprocess(timeout, logger)(combine_masks)
else:
func = combine_masks
try:
body_id, combined_box, combined_mask_downsampled, chosen_downsample_factor = func(
config, body_id, boxes_and_compressed_masks)
return (body_id, combined_box, combined_mask_downsampled,
chosen_downsample_factor, None)
except TimeoutError: # fyi: can't be raised unless a subprocessed is used
boxes, _compressed_masks, counts = zip(*boxes_and_compressed_masks)
total_count = sum(counts)
boxes = np.asarray(boxes)
combined_box = np.zeros((2, 3), dtype=np.int64)
combined_box[0] = boxes[:, 0, :].min(axis=0)
combined_box[1] = boxes[:, 1, :].max(axis=0)
err_msg = f"Timeout ({timeout}) while downsampling/assembling body: id={body_id} size={total_count} box={combined_box.tolist()}"
logger.error(err_msg)
return (body_id, combined_box, None, None, err_msg)
def test_error(self):
"""
Generate an exception in the subprocess and verify that it appears in the parent.
"""
# Too many arguments; should fail
try:
_result = execute_in_subprocess(1.0, logger)(_test_helper)(1,2,3,4,5)
except:
pass
else:
raise RuntimeError("Expected to see an exception in the subprocess.")
def test_timeout_in_skeletonize(self):
"""
Verify that the subprocess_decorator works to kill the skeletonize function.
"""
a = np.ones((100,1000,1000), dtype=np.uint8)
try:
_result = execute_in_subprocess(1.0, logger)(skeletonize_array)(a)
except TimeoutError:
pass
else:
assert False, "Expected a timeout error."
def test_basic(self):
"""
Execute a well-behaved function in a subprocess and verify the result.
"""
handler = MessageCollector()
logging.getLogger().addHandler(handler)
try:
result = execute_in_subprocess(1.0, logger)(_test_helper)(1,2,0)
assert result == 1+2+0, f"Wrong result: {result}"
assert handler.collected_messages['INFO'] == ['1', '0'], f"Got: {handler.collected_messages['INFO']}"
assert handler.collected_messages['ERROR'] == ['2'], f"Got: {handler.collected_messages['ERROR']}"
finally:
logging.getLogger().removeHandler(handler)
def test_segfault(self):
"""
Test the behavior of a subprocess that segfaults.
Note: If the process dies via a signal (SEGFAULT or SIGKILL), the error is not
propagated immediately and eventually an ordinary TimeoutError is raised.
This is due to a limitation in the Python multiprocessing module:
https://bugs.python.org/issue22393
"""
try:
with pause_faulthandler():
_result = execute_in_subprocess(1.0, logger)(generate_segfault)()
except TimeoutError:
pass
else:
raise RuntimeError("Expected to see a TimeoutError exception.")
def test_no_logger(self):
"""
If no logger is specified to the decorator, then standard out and
standard error are sent to the parent process stdout/stderr as usual.
"""
with open('/tmp/captured-stdout.txt', 'w') as f_out, \
open('/tmp/captured-stderr.txt', 'w') as f_err:
with stdout_redirected( f_out.fileno(), sys.stdout ), \
stdout_redirected( f_err.fileno(), sys.stderr ):
result = execute_in_subprocess(1.0)(_test_helper)(1,2,0)
assert result == 1+2+0, "Wrong result: {}".format(result)
with open('/tmp/captured-stdout.txt', 'r') as f_out, \
open('/tmp/captured-stderr.txt', 'r') as f_err:
out = f_out.read()
err = f_err.read()
assert out == '1\n0', f"Got: {out}"
assert err == '2\n', f"Got: {err}"
def generate_mesh_in_subprocess(config, id_box_mask_factor_err):
"""
If use_subprocess is True, execute generate_mesh() in a subprocess, and handle TimeoutErrors.
Otherwise, just call generate_mesh() directly, with the appropriate parameters
"""
logger = logging.getLogger(__name__ + '.generate_mesh')
logger.setLevel(logging.WARN)
timeout = config['options']['analysis-timeout']
body_id, combined_box, combined_mask, downsample_factor, _err_msg = id_box_mask_factor_err
if config["mesh-config"]["use-subprocesses"]:
func = execute_in_subprocess(timeout, logger)(generate_mesh)
else:
func = generate_mesh
try:
_body_id, mesh_obj = func(config, body_id, combined_box, combined_mask,
downsample_factor)
return (body_id, mesh_obj, None)
except TimeoutError: # fyi: can't be raised unless a subprocessed is used
err_msg = f"Timeout ({timeout}) while meshifying body: id={body_id} box={combined_box.tolist()}"
logger.error(err_msg)
output_dir = config['options']['failed-mask-dir']
if output_dir:
os.makedirs(output_dir, exist_ok=True)
output_path = output_dir + f'/failed-mesh-{body_id}.h5'
logger.error(f"Writing mask to {output_path}")
import h5py
with h5py.File(output_path, 'w') as f:
f["downsample_factor"] = downsample_factor
f["box"] = combined_box
f.create_dataset("mask", data=combined_mask, chunks=True)
return (body_id, None, err_msg)
def generate_mesh_in_subprocess(config, id_box_mask_factor_err):
"""
If use_subprocess is True, execute generate_mesh() in a subprocess, and handle TimeoutErrors.
Otherwise, just call generate_mesh() directly, with the appropriate parameters
"""
logger = logging.getLogger(__name__ + '.generate_mesh')
logger.setLevel(logging.WARN)
timeout = config['options']['analysis-timeout']
body_id, combined_box, combined_mask, downsample_factor, _err_msg = id_box_mask_factor_err
if config["mesh-config"]["use-subprocesses"]:
func = execute_in_subprocess(timeout, logger)(generate_mesh)
else:
func = generate_mesh
try:
_body_id, mesh_obj = func(config, body_id, combined_box, combined_mask, downsample_factor)
return (body_id, mesh_obj, None)
except TimeoutError: # fyi: can't be raised unless a subprocessed is used
err_msg = f"Timeout ({timeout}) while meshifying body: id={body_id} box={combined_box.tolist()}"
logger.error(err_msg)
output_dir = config['options']['failed-mask-dir']
if output_dir:
os.makedirs(output_dir, exist_ok=True)
output_path = output_dir + f'/failed-mesh-{body_id}.h5'
logger.error(f"Writing mask to {output_path}")
import h5py
with h5py.File(output_path, 'w') as f:
f["downsample_factor"] = downsample_factor
f["box"] = combined_box
f.create_dataset("mask", data=combined_mask, chunks=True)
return (body_id, None, err_msg)