def test_run_external_command_where_tmpdir_is_none_or_not_a_dir(self):
ecode, out, err = core.run_external_command('foo', None)
self.assertEqual(ecode, 255)
self.assertEqual(out, '')
self.assertEqual(err, 'Tmpdir must be set')
temp_dir = tempfile.mkdtemp()
try:
notdir = os.path.join(temp_dir, 'blah')
ecode, out, err = core.run_external_command('foo', notdir)
self.assertEqual(ecode, 254)
self.assertEqual(out, '')
self.assertEqual(err, 'Tmpdir must be a directory')
finally:
shutil.rmtree(temp_dir)
def _run_single_merge_job(theargs, taskid):
"""runs CHM Job
:param theargs: list of arguments obtained from _parse_arguments()
:returns: exit code for program. 0 success otherwise failure
"""
# TODO REFACTOR THIS INTO FACTORY CLASS TO GET CONFIG
# TODO REFACTOR THIS INTO CLASS TO GENERATE CHM JOB COMMAND
out_dir = None
try:
out_dir = os.path.join(theargs.scratchdir,
str(taskid) + '.' + uuid.uuid4().hex)
config = configparser.ConfigParser()
config.read(
os.path.join(theargs.jobdir, CHMJobCreator.MERGE_CONFIG_FILE_NAME))
thebin = config.get(taskid, CHMJobCreator.MERGE_MERGETILES_BIN)
input_dir = config.get(taskid, CHMJobCreator.MERGE_INPUT_IMAGE_DIR)
# TODO TEST that relative paths work with MERGE phase
if not input_dir.startswith('/'):
input_dir = os.path.join(theargs.jobdir, CHMJobCreator.RUN_DIR,
input_dir)
out_file = config.get(taskid, CHMJobCreator.MERGE_OUTPUT_IMAGE)
if not out_file.startswith('/'):
out_file = os.path.join(theargs.jobdir, CHMJobCreator.RUN_DIR,
out_file)
logger.debug('Creating directory ' + out_dir)
os.makedirs(out_dir, mode=0o775)
cmd = (thebin + ' ' + input_dir + ' ' + out_file +
' --suffix png --log DEBUG')
exitcode, out, err = core.run_external_command(cmd, out_dir)
sys.stdout.write(out)
sys.stderr.write(err)
sys.stdout.flush()
sys.stderr.flush()
return exitcode
except Exception:
logger.exception("Error caught exception")
return 2
finally:
if out_dir is not None:
if os.path.isdir(out_dir):
shutil.rmtree(out_dir)
def test_run_external_command_fail_no_output(self):
temp_dir = tempfile.mkdtemp()
try:
fakecmd = os.path.join(temp_dir, 'fake.py')
f = open(fakecmd, 'w')
f.write('#!/usr/bin/env python\n\n')
f.write('import sys\n')
f.write('sys.exit(1)\n')
f.flush()
f.close()
os.chmod(fakecmd, stat.S_IRWXU)
ecode, out, err = core.run_external_command(fakecmd,
temp_dir,
polling_sleep_time=0.1)
self.assertEqual(ecode, 1)
self.assertEqual(out, '')
self.assertEqual(err, '')
finally:
shutil.rmtree(temp_dir)
def _run_single_chm_job(jobdir, scratchdir, taskid, config):
"""runs CHM Job
:param scratchdir: temp directory
:returns: exit code for program. 0 success otherwise failure
"""
# TODO REFACTOR THIS INTO FACTORY CLASS TO GET CONFIG
# TODO REFACTOR THIS INTO CLASS TO GENERATE CHM JOB COMMAND
out_dir = None
try:
out_dir = os.path.join(scratchdir,
str(taskid) + '.' + uuid.uuid4().hex)
input_image = config.get(taskid, CHMJobCreator.CONFIG_INPUT_IMAGE)
if not input_image.startswith('/'):
logger.debug('Prepending images dir to path: ' + input_image)
input_image = os.path.join(
config.get(taskid, CHMJobCreator.CONFIG_IMAGES), input_image)
logger.debug('Creating directory ' + out_dir)
os.makedirs(out_dir, mode=0o775)
if config.get(taskid,
CHMJobCreator.CONFIG_DISABLE_HISTEQ_IMAGES) == 'True':
histeq_flag = ' -h '
else:
histeq_flag = ' '
cmd = ('"' + config.get('DEFAULT', CHMJobCreator.CONFIG_CHM_BIN) +
'" test "' + input_image + '" ' + out_dir + ' -m "' +
config.get(taskid, CHMJobCreator.CONFIG_MODEL) + '" -b ' +
config.get(taskid, CHMJobCreator.CONFIG_TILE_SIZE) + ' -o ' +
config.get(taskid, CHMJobCreator.CONFIG_OVERLAP_SIZE) +
histeq_flag + ' ' +
config.get(taskid, CHMJobCreator.CONFIG_ARGS))
exitcode, out, err = core.run_external_command(cmd, out_dir)
sys.stdout.write(out)
sys.stderr.write(err)
sys.stdout.flush()
sys.stderr.flush()
logger.info('Job has completed with exit code: ' + str(exitcode))
prob_map = os.path.join(out_dir, os.path.basename(input_image))
if os.path.isfile(prob_map) is False:
logger.error('Result file missing : ' + prob_map)
# this handles case where singularity pukes cause the
# directory under /tmp already exists
if 'ABORT: Could not create temporary directory /tmp' in err:
raise SingularityAbortError(err)
if 'ABORT: Could not create directory /tmp' in err:
raise SingularityAbortError(err)
return 3
out_image = config.get(taskid, CHMJobCreator.CONFIG_OUTPUT_IMAGE)
if not out_image.startswith('/'):
logger.debug('Prepending rundir to out image path' + out_image)
out_image = os.path.join(jobdir, CHMJobCreator.RUN_DIR, out_image)
logger.debug('Copying image ' + prob_map + ' to final destination: ' +
out_image)
shutil.move(prob_map, out_image)
return exitcode
finally:
if out_dir is not None:
if os.path.isdir(out_dir):
logger.debug('Removing directory: ' + out_dir)
shutil.rmtree(out_dir)
def _convert_mod_mrc_files(theargs):
"""Check if images and labels are mrc and mod files, if yes
use IMOD to convert them to images and labels
:param theargs: Parameters from _parse_arguments function
:raises OSError: If there is an error creating images or labels directory
under theargs.outdir
:raises IMODConversionError: If either input is invalid or invocations to
mrc2tif or imodmop fail
:raises InvalidInputDataError: If either input images does not exist on
filesystem
:returns: tuple (updated theargs.images value,
updated theargs.labels value)
"""
if not os.path.exists(theargs.images):
raise InvalidInputDataError(theargs.images + ' does not exist')
if os.path.isdir(theargs.images):
logger.debug('Images path is a directory, no conversion needed')
return theargs.images, theargs.labels
tmp_dir = os.path.join(theargs.outdir, TMP_DIR)
images_dir = os.path.join(theargs.outdir, IMAGES_DIR)
logger.info(theargs.images + ' is a file, assume its an mrc and'
' convert it')
logger.debug('Creating directory ' + images_dir)
os.makedirs(images_dir, mode=0o755)
mrc2tif = os.path.join(theargs.imodbindir, 'mrc2tif')
logger.debug('Running ' + mrc2tif)
imageoutpath = os.path.join(images_dir, 'x')
ecode, out, err = core.run_external_command(
(mrc2tif + ' -p ' + theargs.images + ' ' + imageoutpath), tmp_dir)
logger.debug('Output from ' + mrc2tif + ': ' + str(out) + ':' + str(err))
if ecode is not 0:
raise IMODConversionError('Non zero exit code from mrc2tif: ' +
str(ecode) + ' : ' + str(out) + ' : ' +
str(err))
if not os.path.isfile(theargs.labels):
raise IMODConversionError(theargs.labels +
' is not a file, cannot convert')
labels_dir = os.path.join(theargs.outdir, LABELS_DIR)
tmp_mrc = os.path.join(theargs.outdir, TMP_DIR, 'tmp.mrc')
logger.info(theargs.images + ' is a file, assume its a mod '
'file and convert it')
logger.debug('Creating directory ' + labels_dir)
os.makedirs(labels_dir, mode=0o755)
imodmop = os.path.join(theargs.imodbindir, 'imodmop')
logger.debug('Running ' + imodmop)
ecode, out, err = core.run_external_command(
(imodmop + ' -mask 1 ' + theargs.labels + ' ' + theargs.images + ' ' +
tmp_mrc), tmp_dir)
logger.info('Output from imodmop: ' + str(out) + ':' + str(err))
if ecode is not 0:
raise IMODConversionError('Non zero exit code from imodmop: ' +
str(ecode) + ' : ' + str(out) + ' : ' +
str(err))
ecode, out, err = core.run_external_command(
(mrc2tif + ' -p ' + tmp_mrc + ' ' + os.path.join(labels_dir, 'x')),
tmp_dir)
logger.debug('Output from mrc2tif: ' + str(out) + ':' + str(err))
if ecode is not 0:
raise IMODConversionError('Non zero exit code from mrc2tif: ' +
str(ecode) + ' : ' + str(out) + ' : ' +
str(err))
return images_dir, labels_dir
def _create_mrc_stack(image_dir, num_tiles, dest_file, theargs):
"""Convert image
"""
img_list = image.get_image_path_list(image_dir, theargs.suffix)
if len(img_list) is 0:
logger.error('No images found in ' + image_dir)
return 1
try:
logger.debug('--useconfig set to ' + str(theargs.useconfig))
if theargs.useconfig is None:
parse_config = False
else:
parse_config = True
except AttributeError:
parse_config = False
if parse_config is True:
logger.debug('Using config for tiles')
tile_tuple_list = _get_tiles_from_tuple_list(img_list,
theargs.useconfig)
else:
logger.debug('Generating random tiles')
random.seed(theargs.seed)
tsize = core.parse_width_and_height_from_str(theargs.tilesize)
tile_tuple_list = _pick_random_tiles(img_list,
num_tiles,
tile_width=tsize[0],
tile_height=tsize[1])
if tile_tuple_list is None:
logger.error('Unable to generate random tiles')
return 1
temp_dir = tempfile.mkdtemp(dir=theargs.scratchdir)
curdir = os.getcwd()
try:
counter = 0
for entry in tile_tuple_list:
_extract_and_save_tile(temp_dir, entry, counter)
counter += 1
logger.info('Changing to ' + temp_dir + 'directory to run newstack')
os.chdir(temp_dir)
tif_list = []
for entry in os.listdir(temp_dir):
if entry.endswith('.tif'):
tif_list.append(entry)
cmd = 'newstack ' + ' '.join(tif_list) + ' "' + dest_file + '"'
exit, out, err = core.run_external_command(cmd, temp_dir)
sys.stdout.write(out)
sys.stderr.write(err)
_save_tile_tuple_list_as_config_file(tile_tuple_list,
dest_file + '.tile.list.config')
return exit
finally:
os.chdir(curdir)
if theargs.dontdeletescratch is True:
logger.info('Skipping delete of scratchdir cause '
'--dontdeletescratch was set')
else:
shutil.rmtree(temp_dir)
def test_run_external_command_where_command_is_none(self):
ecode, out, err = core.run_external_command(None, None)
self.assertEqual(ecode, 256)
self.assertEqual(out, '')
self.assertEqual(err, 'Command must be set')