def test_create_job_three_images_five_tiles_per_job(self):
temp_dir = tempfile.mkdtemp()
try:
image_dir = os.path.join(temp_dir, 'images')
os.makedirs(image_dir, mode=0o775)
foo1img = os.path.join(image_dir, 'foo1.png')
self._create_png_image(foo1img, (400, 300))
foo2img = os.path.join(image_dir, 'foo2.png')
self._create_png_image(foo2img, (200, 100))
foo3img = os.path.join(image_dir, 'foo3.png')
self._create_png_image(foo3img, (800, 400))
opts = CHMConfig(image_dir,
'model',
temp_dir,
'200x100',
'0x0',
number_tiles_per_task=5)
creator = CHMJobCreator(opts)
opts = creator.create_job()
self.assertEqual(opts.get_out_dir(), temp_dir)
self.assertEqual(
opts.get_job_config(),
os.path.join(temp_dir, CHMJobCreator.CONFIG_FILE_NAME))
config = configparser.ConfigParser()
config.read(opts.get_job_config())
self.assertEqual(len(config.sections()), 7)
o_img_list = []
for x in range(1, 8):
o_img = config.get(str(x), CHMJobCreator.CONFIG_OUTPUT_IMAGE)
o_img_list.append(o_img)
o_img_list.sort()
self.assertEqual(o_img_list, [
'tiles/foo1.png/001.foo1.png', 'tiles/foo1.png/002.foo1.png',
'tiles/foo2.png/001.foo2.png', 'tiles/foo3.png/001.foo3.png',
'tiles/foo3.png/002.foo3.png', 'tiles/foo3.png/003.foo3.png',
'tiles/foo3.png/004.foo3.png'
])
self.assertTrue(
os.path.isdir(
os.path.join(temp_dir, CHMJobCreator.RUN_DIR,
CHMJobCreator.TILES_DIR, 'foo1.png')))
self.assertTrue(
os.path.isdir(
os.path.join(temp_dir, CHMJobCreator.RUN_DIR,
CHMJobCreator.TILES_DIR, 'foo2.png')))
self.assertTrue(
os.path.isdir(
os.path.join(temp_dir, CHMJobCreator.RUN_DIR,
CHMJobCreator.TILES_DIR, 'foo3.png')))
finally:
shutil.rmtree(temp_dir)
def test_extract_width_and_height(self):
opts = CHMConfig(None, None, None, None, None)
w, h = opts._extract_width_and_height(None)
self.assertEqual(w, '')
self.assertEqual(h, '')
w, h = opts._extract_width_and_height('')
self.assertEqual(w, '')
self.assertEqual(h, '')
w, h = opts._extract_width_and_height(50)
self.assertEqual(w, 50)
self.assertEqual(h, 50)
w, h = opts._extract_width_and_height('300')
self.assertEqual(w, 300)
self.assertEqual(h, 300)
w, h = opts._extract_width_and_height('10x20')
self.assertEqual(w, 10)
self.assertEqual(h, 20)
w, h = opts._extract_width_and_height('10x20x')
self.assertEqual(w, 10)
self.assertEqual(h, 20)
try:
w, h = opts._extract_width_and_height('x')
self.fail('Expected ValueError')
except ValueError:
pass
def test_get_merge_submit_script_path(self):
opts = CHMConfig('images', 'model', 'out', '500x500', '20x20')
rc = GordonCluster(None)
self.assertEqual(rc._get_merge_submit_script_path(),
GordonCluster.MERGE_SUBMIT_SCRIPT_NAME)
opts = CHMConfig('images', 'model', 'out', '500x500', '20x20')
rc = GordonCluster(opts)
self.assertEqual(
rc._get_merge_submit_script_path(),
os.path.join('out', GordonCluster.MERGE_SUBMIT_SCRIPT_NAME))
def test_get_submit_script_path(self):
opts = CHMConfig('images', 'model', 'out', '500x500', '20x20')
gen = GordonCluster(None)
val = gen._get_submit_script_path()
self.assertEqual(val, GordonCluster.SUBMIT_SCRIPT_NAME)
gen = GordonCluster(opts)
val = gen._get_submit_script_path()
script = os.path.join(opts.get_out_dir(),
GordonCluster.SUBMIT_SCRIPT_NAME)
self.assertEqual(val, script)
def test_get_number_of_tiles_tuple_no_overlap(self):
# no overlap test with various image_stats sizes
opts = CHMConfig('/foo', 'model', 'outdir', '100x200', '0x0')
gen = CHMArgGenerator(opts)
# 100x200 image
im_stats = ImageStats('f', 100, 200, 'PNG')
w, h = gen._get_number_of_tiles_tuple(im_stats)
self.assertEqual(w, 1)
self.assertEqual(h, 1)
# 101x201 image
im_stats = ImageStats('f', 101, 201, 'PNG')
w, h = gen._get_number_of_tiles_tuple(im_stats)
self.assertEqual(w, 2)
self.assertEqual(h, 2)
# 199x299 image
im_stats = ImageStats('f', 199, 299, 'PNG')
w, h = gen._get_number_of_tiles_tuple(im_stats)
self.assertEqual(w, 2)
self.assertEqual(h, 2)
# 505x650 image
im_stats = ImageStats('f', 505, 650, 'PNG')
w, h = gen._get_number_of_tiles_tuple(im_stats)
self.assertEqual(w, 6)
self.assertEqual(h, 4)
# 32000x24000 image
im_stats = ImageStats('f', 32000, 24000, 'PNG')
w, h = gen._get_number_of_tiles_tuple(im_stats)
self.assertEqual(w, 320)
self.assertEqual(h, 120)
def test_get_merge_submit_command(self):
opts = CHMConfig('images', 'model', 'out', '500x500', '20x20')
rc = RocceCluster(opts)
self.assertEqual(
rc.get_merge_submit_command(100),
'cd "out";qsub -t 1-100 ' + RocceCluster.MERGE_SUBMIT_SCRIPT_NAME)
def test_add_mergetask_for_image_to_config(self):
temp_dir = tempfile.mkdtemp()
try:
opts = CHMConfig(os.path.join(temp_dir, 'images'), 'model',
temp_dir, '200x100', '20x20')
creator = CHMJobCreator(opts)
config = creator._create_config()
run_dir = creator._create_run_dir()
self.assertTrue(os.path.isdir(run_dir))
creator._add_mergetask_for_image_to_config(config, "1", "foo",
None)
res = config.get("1", CHMJobCreator.MERGE_INPUT_IMAGE_DIR)
self.assertEqual(res, os.path.join(CHMJobCreator.TILES_DIR, 'foo'))
res = config.get("1", CHMJobCreator.MERGE_OUTPUT_IMAGE)
self.assertEqual(res,
os.path.join(CHMJobCreator.PROBMAPS_DIR, 'foo'))
tif = CHMJobCreator.PMAP_SUFFIX
creator._add_mergetask_for_image_to_config(config, "2", "bar", tif)
res = config.get("2", CHMJobCreator.MERGE_INPUT_IMAGE_DIR)
self.assertEqual(res, os.path.join(CHMJobCreator.TILES_DIR, 'bar'))
res = config.get("2", CHMJobCreator.MERGE_OUTPUT_IMAGE)
self.assertEqual(
res, os.path.join(CHMJobCreator.PROBMAPS_DIR, 'bar' + tif))
finally:
shutil.rmtree(temp_dir)
def test_write_readme_no_args(self):
temp_dir = tempfile.mkdtemp()
try:
opts = CHMConfig('/foo',
'model',
temp_dir,
'200x100',
'20x20',
tasks_per_node=20)
creator = CHMJobCreator(opts)
con = creator._create_config()
creator._write_readme(con)
readme = os.path.join(temp_dir, CHMJobCreator.README_TXT_FILE)
self.assertTrue(os.path.isfile(readme))
f = open(readme, 'r')
data = f.read()
f.close()
self.assertTrue('command line:\n\nUnknown' in data)
self.assertTrue('Chmutil version: unknown' in data)
# verifies issue #20 has been fixed
self.assertTrue('checkjobstatus.py' not in data)
finally:
shutil.rmtree(temp_dir)
def test_get_chm_submit_command(self):
opts = CHMConfig('images', 'model', 'out', '500x500', '20x20')
rc = CometCluster(opts)
self.assertEqual(
rc.get_chm_submit_command(5),
'cd "out";sbatch -a 1-5 ' + CometCluster.SUBMIT_SCRIPT_NAME)
def test_create_config_and_write_config(self):
temp_dir = tempfile.mkdtemp()
try:
opts = CHMConfig('/foo',
'model',
temp_dir,
'200x100',
'20x20',
tasks_per_node=20)
creator = CHMJobCreator(opts)
con = creator._create_config()
self.assertEqual(con.get('DEFAULT', 'model'), 'model')
self.assertEqual(
con.get('DEFAULT', CHMJobCreator.CONFIG_TILES_PER_TASK), '1')
self.assertEqual(con.get('DEFAULT', 'tilesize'), '200x100')
self.assertEqual(con.get('DEFAULT', 'overlapsize'), '20x20')
self.assertEqual(con.get('DEFAULT', 'disablehisteqimages'), 'True')
self.assertEqual(
con.get('DEFAULT', CHMJobCreator.CONFIG_TASKS_PER_NODE), '20')
cfile = creator._write_config(con)
self.assertEqual(os.path.isfile(cfile), True)
finally:
shutil.rmtree(temp_dir)
def test_add_job_for_image_to_config(self):
temp_dir = tempfile.mkdtemp()
try:
opts = CHMConfig(os.path.join(temp_dir, 'images'), 'model',
temp_dir, '200x100', '20x20')
creator = CHMJobCreator(opts)
config = creator._create_config()
run_dir = creator._create_run_dir()
self.assertTrue(os.path.isdir(run_dir))
iis = ImageStats(os.path.join(temp_dir, 'images', 'foo123.png'),
500, 400, 'PNG')
i_name = creator._create_output_image_dir(iis, run_dir)
self.assertEqual(i_name, 'foo123.png')
creator._add_task_for_image_to_config(config, '12', i_name, 2,
['-t 1,1'])
self.assertEqual(
config.get('12', CHMJobCreator.CONFIG_INPUT_IMAGE),
'foo123.png')
self.assertEqual(config.get('12', CHMJobCreator.CONFIG_ARGS),
'-t 1,1')
self.assertEqual(
config.get('12', CHMJobCreator.CONFIG_OUTPUT_IMAGE),
os.path.join(CHMJobCreator.TILES_DIR, i_name, '002.' + i_name))
finally:
shutil.rmtree(temp_dir)
def test_get_number_of_tiles_tuple_with_overlap(self):
try:
opts = CHMConfig('/foo', 'model', 'outdir', '100x200', '50x20')
CHMArgGenerator(opts)
except OverlapTooLargeForTileSizeError as e:
self.assertEqual(str(e), 'Overlap width too large for tile')
try:
opts = CHMConfig('/foo', 'model', 'outdir', '100x200', '40x110')
CHMArgGenerator(opts)
except OverlapTooLargeForTileSizeError as e:
self.assertEqual(str(e), 'Overlap height too large for tile')
# overlap test with various image_stats sizes
opts = CHMConfig('/foo', 'model', 'outdir', '100x200', '20x20')
gen = CHMArgGenerator(opts)
# 100x200 image
im_stats = ImageStats('f', 100, 200, 'PNG')
w, h = gen._get_number_of_tiles_tuple(im_stats)
self.assertEqual(w, 2)
self.assertEqual(h, 2)
# 101x201 image
im_stats = ImageStats('f', 101, 201, 'PNG')
w, h = gen._get_number_of_tiles_tuple(im_stats)
self.assertEqual(w, 2)
self.assertEqual(h, 2)
# 199x299 image
im_stats = ImageStats('f', 199, 299, 'PNG')
w, h = gen._get_number_of_tiles_tuple(im_stats)
self.assertEqual(w, 4)
self.assertEqual(h, 2)
# 505x650 image
im_stats = ImageStats('f', 505, 650, 'PNG')
w, h = gen._get_number_of_tiles_tuple(im_stats)
self.assertEqual(w, 9)
self.assertEqual(h, 5)
# 32000x24000 image
im_stats = ImageStats('f', 32000, 24000, 'PNG')
w, h = gen._get_number_of_tiles_tuple(im_stats)
self.assertEqual(w, 534)
self.assertEqual(h, 150)
def test_get_chm_runner_path(self):
gen = GordonCluster(None)
self.assertEqual(gen._get_chm_runner_path(), CHMJobCreator.CHMRUNNER)
opts = CHMConfig('images', 'model', 'out', '500x500', '20x20')
gen = GordonCluster(opts)
self.assertEqual(gen._get_chm_runner_path(), CHMJobCreator.CHMRUNNER)
opts = CHMConfig('images',
'model',
'out',
'500x500',
'20x20',
scriptbin='/home/foo/.local/bin')
gen = GordonCluster(opts)
spath = os.path.join('/home/foo/.local/bin', CHMJobCreator.CHMRUNNER)
self.assertEqual(gen._get_chm_runner_path(), spath)
def test_get_args_one_tile(self):
opts = CHMConfig('/foo', 'model', 'outdir', '100x200', '0x0')
gen = CHMArgGenerator(opts)
# 100x200 image
im_stats = ImageStats('f', 100, 200, 'PNG')
tlist = gen.get_args(im_stats)
self.assertEqual(len(tlist), 1)
self.assertEqual(tlist, [['-t 1,1']])
def test_get_chm_submit_command_too_many_jobs(self):
temp_dir = tempfile.mkdtemp()
try:
opts = CHMConfig('images', 'model', temp_dir, '500x500', '20x20')
rc = GordonCluster(opts)
self.assertTrue(GordonCluster.WARNING_MESSAGE + 'cd "' + temp_dir +
'";qsub ' + GordonCluster.SUBMIT_SCRIPT_NAME in
rc.get_chm_submit_command(1001))
finally:
shutil.rmtree(temp_dir)
def test_get_chm_submit_command(self):
temp_dir = tempfile.mkdtemp()
try:
opts = CHMConfig('images', 'model', temp_dir, '500x500', '20x20')
rc = GordonCluster(opts)
self.assertEqual(
rc.get_chm_submit_command(5), 'cd "' + temp_dir + '";qsub ' +
GordonCluster.SUBMIT_SCRIPT_NAME)
finally:
shutil.rmtree(temp_dir)
def test_get_chm_task_stats(self):
con = CHMConfig('./images', './model', './outdir', '500x500', '20x20')
cfig = configparser.ConfigParser()
cfig.add_section('1')
cfig.set('1', 'hi', 'val')
cfig.add_section('2')
cfig.set('2', 'hi', 'val')
cfig.add_section('3')
cfig.set('3', 'hi', 'val')
cfig.add_section('4')
cfig.set('4', 'hi', 'val')
# try with none for lists
con.set_config(cfig)
tsf = TaskSummaryFactory(con)
ts = tsf._get_chm_task_stats()
self.assertEqual(ts.get_completed_task_count(), 0)
self.assertEqual(ts.get_total_task_count(), 4)
# try with empty lists
tsf = TaskSummaryFactory(con,
chm_incomplete_tasks=[],
merge_incomplete_tasks=[])
ts = tsf._get_chm_task_stats()
self.assertEqual(ts.get_completed_task_count(), 4)
self.assertEqual(ts.get_total_task_count(), 4)
# try with lists with elements
tsf = TaskSummaryFactory(con,
chm_incomplete_tasks=['hi'],
merge_incomplete_tasks=['a', 'b'])
ts = tsf._get_chm_task_stats()
self.assertEqual(ts.get_completed_task_count(), 3)
self.assertEqual(ts.get_total_task_count(), 4)
# try with lists with elements 2 incomplete
tsf = TaskSummaryFactory(con,
chm_incomplete_tasks=['hi', 'bye'],
merge_incomplete_tasks=['a', 'b'])
ts = tsf._get_chm_task_stats()
self.assertEqual(ts.get_completed_task_count(), 2)
self.assertEqual(ts.get_total_task_count(), 4)
def test_get_checkchmjob_command(self):
temp_dir = tempfile.mkdtemp()
try:
opts = CHMConfig('images', 'model', temp_dir, '500x500', '20x20')
rc = GordonCluster(opts)
self.assertEqual(
rc.get_checkchmjob_command(),
CHMJobCreator.CHECKCHMJOB + ' "' + temp_dir + '" --submit')
finally:
shutil.rmtree(temp_dir)
def test_generate_merge_submit_script(self):
temp_dir = tempfile.mkdtemp()
try:
opts = CHMConfig('images', 'model', temp_dir, '500x500', '20x20')
gen = CometCluster(opts)
script = gen._get_merge_submit_script_path()
self.assertEqual(os.path.isfile(script), False)
gen.generate_merge_submit_script()
self.assertEqual(os.path.isfile(script), True)
# TODO Test qsub script file has correct data in it
finally:
shutil.rmtree(temp_dir)
def test_create_run_dir(self):
temp_dir = tempfile.mkdtemp()
try:
opts = CHMConfig(os.path.join(temp_dir, 'images'), 'model',
temp_dir, '200x100', '20x20')
creator = CHMJobCreator(opts)
run_dir = creator._create_run_dir()
self.assertTrue(os.path.isdir(run_dir))
stdout_dir = os.path.join(run_dir, CHMJobCreator.STDOUT_DIR)
self.assertTrue(os.path.isdir(stdout_dir))
finally:
shutil.rmtree(temp_dir)
def test_get_image_stats_summary_output_compute_true_one_image(self):
temp_dir = tempfile.mkdtemp()
try:
im = Image.new('L', (10, 10))
iname = os.path.join(temp_dir, '1.png')
im.save(iname)
expsize = os.path.getsize(iname)
con = CHMConfig(temp_dir, './model', './outdir', '500x500',
'20x20')
cfig = configparser.ConfigParser()
cfig.add_section('1')
cfig.set('1', 'hi', 'val')
cfig.add_section('2')
cfig.set('2', 'hi', 'val')
con.set_config(cfig)
mfig = configparser.ConfigParser()
mfig.add_section('3')
mfig.set('3', 'hi', 'val')
con.set_merge_config(mfig)
tsf = TaskSummaryFactory(con,
chm_incomplete_tasks=[],
merge_incomplete_tasks=[],
output_compute=True)
isum = tsf._get_image_stats_summary()
self.assertEqual(isum.get_image_count(), 1)
self.assertEqual(isum.get_total_pixels(), 100)
self.assertEqual(isum.get_total_size_of_images_in_bytes(), expsize)
self.assertEqual(isum.get_image_dimensions_as_dict(),
{(10, 10): 1})
finally:
shutil.rmtree(temp_dir)
def test_create_job_one_image_one_tile_per_job_and_gentifs_true(self):
temp_dir = tempfile.mkdtemp()
try:
image_dir = os.path.join(temp_dir, 'images')
os.makedirs(image_dir, mode=0o775)
fooimg = os.path.join(image_dir, 'foo1.png')
self._create_png_image(fooimg, (400, 300))
opts = CHMConfig(image_dir,
'model',
temp_dir,
'200x100',
'0x0',
gentifs=True)
creator = CHMJobCreator(opts)
opts = creator.create_job()
self.assertEqual(opts.get_out_dir(), temp_dir)
self.assertEqual(
opts.get_job_config(),
os.path.join(temp_dir, CHMJobCreator.CONFIG_FILE_NAME))
tmpdir = os.path.join(opts.get_run_dir(), CHMJobCreator.STDOUT_DIR)
self.assertTrue(os.path.isdir(tmpdir))
config = configparser.ConfigParser()
config.read(opts.get_job_config())
self.assertEqual(config.sections(), ['1', '2', '3', '4', '5', '6'])
self.assertEqual(config.get('1', CHMJobCreator.CONFIG_INPUT_IMAGE),
'foo1.png')
self.assertEqual(config.get('1', CHMJobCreator.CONFIG_ARGS),
'-t 1,1')
self.assertEqual(
config.get('1', CHMJobCreator.CONFIG_OUTPUT_IMAGE),
os.path.join(CHMJobCreator.TILES_DIR, 'foo1.png',
'001.foo1.png'))
self.assertEqual(config.get('1', CHMJobCreator.CONFIG_MODEL),
'model')
self.assertEqual(
config.get('1', CHMJobCreator.CONFIG_DISABLE_HISTEQ_IMAGES),
'True')
self.assertEqual(
config.get('1', CHMJobCreator.CONFIG_TASKS_PER_NODE), '1')
config = opts.get_merge_config()
res = config.getboolean(CHMJobCreator.CONFIG_DEFAULT,
CHMJobCreator.MERGE_GENTIFS)
self.assertEqual(res, True)
res = config.get('1', CHMJobCreator.MERGE_OUTPUT_IMAGE)
self.assertEqual(
res,
os.path.join(CHMJobCreator.PROBMAPS_DIR,
'foo1.png' + CHMJobCreator.PMAP_SUFFIX))
finally:
shutil.rmtree(temp_dir)
def test_get_task_summary(self):
con = CHMConfig('./images', './model', './outdir', '500x500', '20x20')
cfig = configparser.ConfigParser()
cfig.add_section('1')
cfig.set('1', 'hi', 'val')
cfig.add_section('2')
cfig.set('2', 'hi', 'val')
con.set_config(cfig)
mfig = configparser.ConfigParser()
mfig.add_section('3')
mfig.set('3', 'hi', 'val')
con.set_merge_config(mfig)
tsf = TaskSummaryFactory(con,
chm_incomplete_tasks=[],
merge_incomplete_tasks=['a'])
ts = tsf.get_task_summary()
self.assertEqual(
ts.get_summary(), 'chmutil version: unknown\n'
'Tiles: 500x500 with 20x20 '
'overlap\nDisable histogram '
'equalization in CHM: True\n'
'Tasks: 1 tiles per task, 1 '
'tasks(s) per node\nTrained '
'CHM model: ./model\nCHM binary: '
'./chm-0.1.0.img\n\nCHM tasks: '
'100% complete (2 of 2 completed)'
'\nMerge tasks: 0% complete (0 of '
'1 completed)\n')
def test_get_image_stats_summary_output_compute_true_no_images(self):
temp_dir = tempfile.mkdtemp()
try:
con = CHMConfig(temp_dir, './model', './outdir', '500x500',
'20x20')
cfig = configparser.ConfigParser()
cfig.add_section('1')
cfig.set('1', 'hi', 'val')
cfig.add_section('2')
cfig.set('2', 'hi', 'val')
con.set_config(cfig)
mfig = configparser.ConfigParser()
mfig.add_section('3')
mfig.set('3', 'hi', 'val')
con.set_merge_config(mfig)
tsf = TaskSummaryFactory(con,
chm_incomplete_tasks=[],
merge_incomplete_tasks=[],
output_compute=True)
isum = tsf._get_image_stats_summary()
self.assertEqual(isum.get_image_count(), 0)
finally:
shutil.rmtree(temp_dir)
def test_create_job_one_image_five_tiles_per_job(self):
temp_dir = tempfile.mkdtemp()
try:
image_dir = os.path.join(temp_dir, 'images')
os.makedirs(image_dir, mode=0o775)
fooimg = os.path.join(image_dir, 'foo1.png')
self._create_png_image(fooimg, (400, 300))
opts = CHMConfig(image_dir,
'model',
temp_dir,
'200x100',
'0x0',
number_tiles_per_task=5)
creator = CHMJobCreator(opts)
opts = creator.create_job()
self.assertEqual(opts.get_out_dir(), temp_dir)
self.assertEqual(
opts.get_job_config(),
os.path.join(temp_dir, CHMJobCreator.CONFIG_FILE_NAME))
self.assertTrue(
os.path.isdir(
os.path.join(opts.get_out_dir(), CHMJobCreator.RUN_DIR)))
config = configparser.ConfigParser()
config.read(opts.get_job_config())
self.assertEqual(config.sections(), ['1', '2'])
self.assertEqual(config.get('1', CHMJobCreator.CONFIG_INPUT_IMAGE),
'foo1.png')
self.assertEqual(config.get('1', CHMJobCreator.CONFIG_ARGS),
'-t 1,1 -t 1,2 -t 1,3 -t 2,1 -t 2,2')
self.assertEqual(
config.get('1', CHMJobCreator.CONFIG_OUTPUT_IMAGE),
os.path.join(CHMJobCreator.TILES_DIR, 'foo1.png',
'001.foo1.png'))
self.assertTrue(
os.path.isdir(
os.path.join(temp_dir, CHMJobCreator.RUN_DIR,
CHMJobCreator.TILES_DIR, 'foo1.png')))
self.assertEqual(config.get('2', CHMJobCreator.CONFIG_INPUT_IMAGE),
'foo1.png')
self.assertEqual(config.get('2', CHMJobCreator.CONFIG_ARGS),
'-t 2,3')
self.assertEqual(
config.get('2', CHMJobCreator.CONFIG_OUTPUT_IMAGE),
os.path.join(CHMJobCreator.TILES_DIR, 'foo1.png',
'002.foo1.png'))
self.assertEqual(config.get('2', CHMJobCreator.CONFIG_MODEL),
'model')
self.assertEqual(
config.get('2', CHMJobCreator.CONFIG_DISABLE_HISTEQ_IMAGES),
'True')
self.assertEqual(
config.get('2', CHMJobCreator.CONFIG_TASKS_PER_NODE), '1')
finally:
shutil.rmtree(temp_dir)
def test_get_merge_runner_path(self):
rc = GordonCluster(None)
self.assertEqual(rc._get_merge_runner_path(),
CHMJobCreator.MERGERUNNER)
opts = CHMConfig('images',
'model',
'out',
'500x500',
'20x20',
scriptbin='foo')
rc = GordonCluster(opts)
self.assertEqual(rc._get_merge_runner_path(),
os.path.join('foo', CHMJobCreator.MERGERUNNER))
def test_generate_submit_script(self):
temp_dir = tempfile.mkdtemp()
try:
opts = CHMConfig('images', 'model', temp_dir, '500x500', '20x20')
gen = RocceCluster(opts)
script = gen._get_submit_script_path()
self.assertEqual(os.path.isfile(script), False)
gen.generate_submit_script()
self.assertEqual(os.path.isfile(script), True)
# TODO Test qsub script file has correct data in it
f = open(script, 'r')
data = f.read()
f.close()
self.assertTrue('\n#$ -l h_rt=12:00:00,h_vmem=10G\n' in data)
finally:
shutil.rmtree(temp_dir)
def test_generate_submit_script(self):
temp_dir = tempfile.mkdtemp()
try:
opts = CHMConfig('images', 'model', temp_dir, '500x500', '20x20')
gen = CometCluster(None)
gen.set_chmconfig(opts)
script = gen._get_submit_script_path()
self.assertEqual(os.path.isfile(script), False)
gen.generate_submit_script()
self.assertEqual(os.path.isfile(script), True)
# TODO Test qsub script file has correct data in it
with open(script, 'r') as f:
data = f.read()
self.assertTrue('\nmodule load ' 'singularity/2.3.2\n' in data)
finally:
shutil.rmtree(temp_dir)
def test_generate_merge_submit_script(self):
temp_dir = tempfile.mkdtemp()
try:
opts = CHMConfig('images', 'model', temp_dir, '500x500', '20x20')
gen = GordonCluster(opts)
script = gen._get_merge_submit_script_path()
self.assertEqual(os.path.isfile(script), False)
gen.generate_merge_submit_script()
self.assertEqual(os.path.isfile(script), True)
# TODO Test qsub script file has correct data in it
# TODO Test qsub script file has correct data in it
f = open(script, 'r')
script_data = f.read()
f.close()
self.assertTrue('#PBS -A \n' in script_data)
finally:
shutil.rmtree(temp_dir)
def test_create_output_image_dir(self):
temp_dir = tempfile.mkdtemp()
try:
opts = CHMConfig(os.path.join(temp_dir, 'images'), 'model',
temp_dir, '200x100', '20x20')
creator = CHMJobCreator(opts)
run_dir = creator._create_run_dir()
self.assertTrue(os.path.isdir(run_dir))
iis = ImageStats(os.path.join(temp_dir, 'images', 'foo123.png'),
500, 400, 'PNG')
i_name = creator._create_output_image_dir(iis, run_dir)
self.assertTrue(
os.path.isdir(
os.path.join(run_dir, CHMJobCreator.TILES_DIR,
'foo123.png')))
self.assertEqual(i_name, 'foo123.png')
finally:
shutil.rmtree(temp_dir)