def test_get_chmconfig_skip_loading_config_true_and_skip_merge_false(self):
temp_dir = tempfile.mkdtemp()
try:
cfile = os.path.join(temp_dir,
CHMJobCreator.MERGE_CONFIG_FILE_NAME)
config = configparser.ConfigParser()
config.set('', CHMJobCreator.CONFIG_IMAGES, 'images')
config.set('', CHMJobCreator.CONFIG_CLUSTER, 'yocluster')
config.set('', CHMJobCreator.MERGE_TASKS_PER_NODE, '4')
f = open(cfile, 'w')
config.write(f)
f.flush()
f.close()
fac = CHMConfigFromConfigFactory(temp_dir)
chmconfig = fac.get_chmconfig(skip_loading_config=True,
skip_loading_mergeconfig=False)
self.assertEqual(chmconfig.get_out_dir(), temp_dir)
mcon = chmconfig.get_merge_config()
self.assertEqual(mcon.get(CHMJobCreator.CONFIG_DEFAULT,
CHMJobCreator.CONFIG_IMAGES),
'images')
self.assertEqual(mcon.get(CHMJobCreator.CONFIG_DEFAULT,
CHMJobCreator.CONFIG_CLUSTER),
'yocluster')
self.assertEqual(mcon.getint(CHMJobCreator.CONFIG_DEFAULT,
CHMJobCreator.MERGE_TASKS_PER_NODE),
4)
self.assertEqual(chmconfig.get_cluster(), 'yocluster')
finally:
shutil.rmtree(temp_dir)
def test_check_chm_job_all_tasks_complete_with_submit(self):
temp_dir = tempfile.mkdtemp()
try:
out = create_successful_job(temp_dir)
pargs = checkchmjob._parse_arguments('hi', [out, '--submit'])
pargs.program = 'foo'
pargs.version = '1.0.0'
img_tile = os.path.join(out, CHMJobCreator.RUN_DIR,
CHMJobCreator.TILES_DIR, 'foo.png',
'001.foo.png')
size = 800, 800
myimg = Image.new('L', size)
myimg.save(img_tile, 'PNG')
probmap = os.path.join(out, CHMJobCreator.RUN_DIR,
CHMJobCreator.PROBMAPS_DIR, 'foo.png')
myimg.save(probmap)
val = checkchmjob._check_chm_job(pargs)
self.assertEqual(val, 0)
cfac = CHMConfigFromConfigFactory(out)
chmconfig = cfac.get_chmconfig()
path = chmconfig.get_batchedjob_config_file_path()
self.assertEqual(os.path.isfile(path), False)
mpath = chmconfig.get_batched_mergejob_config_file_path()
self.assertEqual(os.path.isfile(mpath), False)
finally:
shutil.rmtree(temp_dir)
def test_create_chm_job_success_withgentifs_set(self):
temp_dir = tempfile.mkdtemp()
try:
images = os.path.join(temp_dir, 'images')
os.makedirs(images, mode=0o755)
# add a fake png image
pngfile = os.path.join(images, 'foo.png')
size = 800, 800
myimg = Image.new('L', size)
myimg.save(pngfile, 'PNG')
model = os.path.join(temp_dir, 'model')
os.makedirs(model, mode=0o755)
p_mat = os.path.join(model, 'param.mat')
open(p_mat, 'a').close()
out = os.path.join(temp_dir, 'out')
pargs = createchmjob._parse_arguments(
'hi',
[images, model, out, '--tilesize', '520x520', '--gentifs'])
pargs.program = 'foo'
pargs.version = '0.1.2'
pargs.rawargs = 'hi how are you'
val = createchmjob._create_chm_job(pargs)
self.assertEqual(val, 0)
fac = CHMConfigFromConfigFactory(out)
chmconfig = fac.get_chmconfig(skip_loading_mergeconfig=False)
mcon = chmconfig.get_merge_config()
self.assertEqual(
mcon.getboolean(CHMJobCreator.CONFIG_DEFAULT,
CHMJobCreator.MERGE_GENTIFS), True)
finally:
shutil.rmtree(temp_dir)
def test_check_chm_job_invalid_cluster(self):
temp_dir = tempfile.mkdtemp()
try:
out = create_successful_job(temp_dir)
cfac = CHMConfigFromConfigFactory(out)
# alter cluster name to be an invalid one
#
chmconfig = cfac.get_chmconfig()
con = chmconfig.get_config()
con.set(CHMJobCreator.CONFIG_DEFAULT, CHMJobCreator.CONFIG_CLUSTER,
'doesnotexist')
f = open(chmconfig.get_job_config(), 'w')
con.write(f)
f.flush()
f.close()
#
pargs = checkchmjob._parse_arguments('hi', [out, '--submit'])
pargs.program = 'foo'
pargs.version = '1.0.0'
img_tile = os.path.join(out, CHMJobCreator.RUN_DIR,
CHMJobCreator.TILES_DIR, 'foo.png',
'001.foo.png')
size = 800, 800
myimg = Image.new('L', size)
myimg.save(img_tile, 'PNG')
val = checkchmjob._check_chm_job(pargs)
self.assertEqual(val, 2)
finally:
shutil.rmtree(temp_dir)
def _run_chm_job(theargs):
"""Runs all jobs for task
:raises LoadConfigError: if no config is found in job dir
:returns: status of `_run_jobs` call 0 for success otherwise error
"""
cfac = CHMConfigFromConfigFactory(theargs.jobdir)
chmconfig = cfac.get_chmconfig()
return _run_jobs(chmconfig, theargs, theargs.taskid)
def test_constructor(self):
try:
CHMConfigFromConfigFactory(None)
self.fail('Expected InvalidJobDirError')
except InvalidJobDirError as e:
self.assertEqual(str(e),
'job directory passed in cannot be null')
fac = CHMConfigFromConfigFactory('foo')
self.assertEqual(fac._job_dir, 'foo')
def _run_merge_job(theargs):
"""Runs all jobs for task
"""
cfac = CHMConfigFromConfigFactory(theargs.jobdir)
chmconfig = cfac.get_chmconfig(skip_loading_config=True,
skip_loading_mergeconfig=False)
if not os.path.isfile(chmconfig.get_batched_mergejob_config_file_path()):
logger.error('No batched merge config found: ' +
chmconfig.get_batched_mergejob_config_file_path())
return 1
return _run_jobs(chmconfig, theargs, theargs.taskid)
def test_get_chmconfig_skip_all_configs(self):
temp_dir = tempfile.mkdtemp()
try:
fac = CHMConfigFromConfigFactory(temp_dir)
chmconfig = fac.get_chmconfig(skip_loading_config=True,
skip_loading_mergeconfig=True)
self.assertEqual(chmconfig.get_images(), None)
self.assertEqual(chmconfig.get_model(), None)
self.assertEqual(chmconfig.get_out_dir(), temp_dir)
finally:
shutil.rmtree(temp_dir)
def test_get_chmconfig_default_values_with_account_set(self):
temp_dir = tempfile.mkdtemp()
try:
cfile = os.path.join(temp_dir,
CHMJobCreator.CONFIG_FILE_NAME)
config = configparser.ConfigParser()
config.set('', CHMJobCreator.CONFIG_IMAGES, 'images')
config.set('', CHMJobCreator.CONFIG_MODEL, 'model')
config.set('', CHMJobCreator.CONFIG_TILE_SIZE, '500x600')
config.set('', CHMJobCreator.CONFIG_OVERLAP_SIZE, '10x20')
config.set('', CHMJobCreator.CONFIG_TILES_PER_TASK, 'tilesperjob')
config.set('', CHMJobCreator.CONFIG_TASKS_PER_NODE, 'jobspernode')
config.set('', CHMJobCreator.CONFIG_DISABLE_HISTEQ_IMAGES, 'True')
config.set('', CHMJobCreator.CONFIG_CHM_BIN, 'chmbin')
config.set('', CHMJobCreator.CONFIG_CLUSTER, 'mycluster')
config.set('', CHMJobCreator.CONFIG_ACCOUNT, 'gg123')
f = open(cfile, 'w')
config.write(f)
f.flush()
f.close()
fac = CHMConfigFromConfigFactory(temp_dir)
chmconfig = fac.get_chmconfig()
self.assertEqual(chmconfig.get_out_dir(), temp_dir)
self.assertEqual(chmconfig.get_chm_binary(), 'chmbin')
self.assertEqual(chmconfig.get_script_bin(), '')
self.assertEqual(chmconfig.get_disable_histogram_eq_val(), True)
self.assertEqual(chmconfig.get_images(), 'images')
self.assertEqual(chmconfig.get_model(), 'model')
self.assertEqual(chmconfig.get_number_tasks_per_node(),
'jobspernode')
self.assertEqual(chmconfig.get_number_tiles_per_task(),
'tilesperjob')
self.assertEqual(chmconfig.get_tile_height(), 600)
self.assertEqual(chmconfig.get_tile_width(), 500)
self.assertEqual(chmconfig.get_tile_size(), '500x600')
self.assertEqual(chmconfig.get_overlap_height(), 20)
self.assertEqual(chmconfig.get_overlap_width(), 10)
self.assertEqual(chmconfig.get_overlap_size(), '10x20')
self.assertEqual(chmconfig.get_cluster(), 'mycluster')
self.assertEqual(chmconfig.get_account(), 'gg123')
config.set('', CHMJobCreator.CONFIG_DISABLE_HISTEQ_IMAGES, 'False')
f = open(cfile, 'w')
config.write(f)
f.flush()
f.close()
chmconfig = fac.get_chmconfig()
self.assertEqual(chmconfig.get_disable_histogram_eq_val(), False)
finally:
shutil.rmtree(temp_dir)
def test_get_chmconfig_skip_config_with_missing_mergeconfig(self):
temp_dir = tempfile.mkdtemp()
try:
fac = CHMConfigFromConfigFactory(temp_dir)
chmconfig = fac.get_chmconfig(skip_loading_config=True,
skip_loading_mergeconfig=False)
self.assertEqual(chmconfig.get_images(), None)
self.assertEqual(chmconfig.get_model(), None)
self.assertEqual(chmconfig.get_out_dir(), temp_dir)
self.assertEqual(chmconfig.get_number_merge_tasks_per_node(), 1)
except LoadConfigError as e:
self.assertTrue('configuration file does not exist' in str(e))
finally:
shutil.rmtree(temp_dir)
def test_get_chmconfig_raise_exception(self):
temp_dir = tempfile.mkdtemp()
try:
fac = CHMConfigFromConfigFactory(temp_dir)
try:
fac.get_chmconfig()
self.fail('expected LoadConfigError')
except LoadConfigError as e:
cfile = os.path.join(temp_dir,
CHMJobCreator.CONFIG_FILE_NAME)
self.assertEqual(str(e), cfile +
' configuration file does not exist')
finally:
shutil.rmtree(temp_dir)
def test_check_chm_job_one_chm_job_to_run(self):
temp_dir = tempfile.mkdtemp()
try:
out = create_successful_job(temp_dir)
pargs = checkchmjob._parse_arguments('hi', [out, '--submit'])
pargs.program = 'foo'
pargs.version = '1.0.0'
val = checkchmjob._check_chm_job(pargs)
self.assertEqual(val, 0)
cfac = CHMConfigFromConfigFactory(out)
chmconfig = cfac.get_chmconfig()
path = chmconfig.get_batchedjob_config_file_path()
self.assertTrue(os.path.isfile(path))
finally:
shutil.rmtree(temp_dir)
def test_get_chmconfig_skip_config_but_not_merge_with_no_tasks_per(self):
temp_dir = tempfile.mkdtemp()
try:
cfile = os.path.join(temp_dir,
CHMJobCreator.MERGE_CONFIG_FILE_NAME)
config = configparser.ConfigParser()
config.set('', CHMJobCreator.CONFIG_IMAGES, 'images')
config.set('', CHMJobCreator.CONFIG_CLUSTER, 'yocluster')
f = open(cfile, 'w')
config.write(f)
f.flush()
f.close()
fac = CHMConfigFromConfigFactory(temp_dir)
chmconfig = fac.get_chmconfig(skip_loading_config=True,
skip_loading_mergeconfig=False)
self.assertEqual(chmconfig.get_images(), None)
self.assertEqual(chmconfig.get_model(), None)
self.assertEqual(chmconfig.get_out_dir(), temp_dir)
self.assertEqual(chmconfig.get_number_merge_tasks_per_node(), 1)
finally:
shutil.rmtree(temp_dir)
def _get_chmconfig(jobdir):
"""Gets chmconfig
"""
cfac = CHMConfigFromConfigFactory(os.path.abspath(jobdir))
return cfac.get_chmconfig(skip_loading_mergeconfig=False)