def test_create_job_one_image_five_tiles_per_job(self):
temp_dir = tempfile.mkdtemp()
try:
image_dir = os.path.join(temp_dir, 'images')
os.makedirs(image_dir, mode=0o775)
fooimg = os.path.join(image_dir, 'foo1.png')
self._create_png_image(fooimg, (400, 300))
opts = CHMConfig(image_dir,
'model',
temp_dir,
'200x100',
'0x0',
number_tiles_per_task=5)
creator = CHMJobCreator(opts)
opts = creator.create_job()
self.assertEqual(opts.get_out_dir(), temp_dir)
self.assertEqual(
opts.get_job_config(),
os.path.join(temp_dir, CHMJobCreator.CONFIG_FILE_NAME))
self.assertTrue(
os.path.isdir(
os.path.join(opts.get_out_dir(), CHMJobCreator.RUN_DIR)))
config = configparser.ConfigParser()
config.read(opts.get_job_config())
self.assertEqual(config.sections(), ['1', '2'])
self.assertEqual(config.get('1', CHMJobCreator.CONFIG_INPUT_IMAGE),
'foo1.png')
self.assertEqual(config.get('1', CHMJobCreator.CONFIG_ARGS),
'-t 1,1 -t 1,2 -t 1,3 -t 2,1 -t 2,2')
self.assertEqual(
config.get('1', CHMJobCreator.CONFIG_OUTPUT_IMAGE),
os.path.join(CHMJobCreator.TILES_DIR, 'foo1.png',
'001.foo1.png'))
self.assertTrue(
os.path.isdir(
os.path.join(temp_dir, CHMJobCreator.RUN_DIR,
CHMJobCreator.TILES_DIR, 'foo1.png')))
self.assertEqual(config.get('2', CHMJobCreator.CONFIG_INPUT_IMAGE),
'foo1.png')
self.assertEqual(config.get('2', CHMJobCreator.CONFIG_ARGS),
'-t 2,3')
self.assertEqual(
config.get('2', CHMJobCreator.CONFIG_OUTPUT_IMAGE),
os.path.join(CHMJobCreator.TILES_DIR, 'foo1.png',
'002.foo1.png'))
self.assertEqual(config.get('2', CHMJobCreator.CONFIG_MODEL),
'model')
self.assertEqual(
config.get('2', CHMJobCreator.CONFIG_DISABLE_HISTEQ_IMAGES),
'True')
self.assertEqual(
config.get('2', CHMJobCreator.CONFIG_TASKS_PER_NODE), '1')
finally:
shutil.rmtree(temp_dir)
def test_create_job_three_images_five_tiles_per_job(self):
temp_dir = tempfile.mkdtemp()
try:
image_dir = os.path.join(temp_dir, 'images')
os.makedirs(image_dir, mode=0o775)
foo1img = os.path.join(image_dir, 'foo1.png')
self._create_png_image(foo1img, (400, 300))
foo2img = os.path.join(image_dir, 'foo2.png')
self._create_png_image(foo2img, (200, 100))
foo3img = os.path.join(image_dir, 'foo3.png')
self._create_png_image(foo3img, (800, 400))
opts = CHMConfig(image_dir,
'model',
temp_dir,
'200x100',
'0x0',
number_tiles_per_task=5)
creator = CHMJobCreator(opts)
opts = creator.create_job()
self.assertEqual(opts.get_out_dir(), temp_dir)
self.assertEqual(
opts.get_job_config(),
os.path.join(temp_dir, CHMJobCreator.CONFIG_FILE_NAME))
config = configparser.ConfigParser()
config.read(opts.get_job_config())
self.assertEqual(len(config.sections()), 7)
o_img_list = []
for x in range(1, 8):
o_img = config.get(str(x), CHMJobCreator.CONFIG_OUTPUT_IMAGE)
o_img_list.append(o_img)
o_img_list.sort()
self.assertEqual(o_img_list, [
'tiles/foo1.png/001.foo1.png', 'tiles/foo1.png/002.foo1.png',
'tiles/foo2.png/001.foo2.png', 'tiles/foo3.png/001.foo3.png',
'tiles/foo3.png/002.foo3.png', 'tiles/foo3.png/003.foo3.png',
'tiles/foo3.png/004.foo3.png'
])
self.assertTrue(
os.path.isdir(
os.path.join(temp_dir, CHMJobCreator.RUN_DIR,
CHMJobCreator.TILES_DIR, 'foo1.png')))
self.assertTrue(
os.path.isdir(
os.path.join(temp_dir, CHMJobCreator.RUN_DIR,
CHMJobCreator.TILES_DIR, 'foo2.png')))
self.assertTrue(
os.path.isdir(
os.path.join(temp_dir, CHMJobCreator.RUN_DIR,
CHMJobCreator.TILES_DIR, 'foo3.png')))
finally:
shutil.rmtree(temp_dir)
def test_create_job_one_image_one_tile_per_job_and_gentifs_true(self):
temp_dir = tempfile.mkdtemp()
try:
image_dir = os.path.join(temp_dir, 'images')
os.makedirs(image_dir, mode=0o775)
fooimg = os.path.join(image_dir, 'foo1.png')
self._create_png_image(fooimg, (400, 300))
opts = CHMConfig(image_dir,
'model',
temp_dir,
'200x100',
'0x0',
gentifs=True)
creator = CHMJobCreator(opts)
opts = creator.create_job()
self.assertEqual(opts.get_out_dir(), temp_dir)
self.assertEqual(
opts.get_job_config(),
os.path.join(temp_dir, CHMJobCreator.CONFIG_FILE_NAME))
tmpdir = os.path.join(opts.get_run_dir(), CHMJobCreator.STDOUT_DIR)
self.assertTrue(os.path.isdir(tmpdir))
config = configparser.ConfigParser()
config.read(opts.get_job_config())
self.assertEqual(config.sections(), ['1', '2', '3', '4', '5', '6'])
self.assertEqual(config.get('1', CHMJobCreator.CONFIG_INPUT_IMAGE),
'foo1.png')
self.assertEqual(config.get('1', CHMJobCreator.CONFIG_ARGS),
'-t 1,1')
self.assertEqual(
config.get('1', CHMJobCreator.CONFIG_OUTPUT_IMAGE),
os.path.join(CHMJobCreator.TILES_DIR, 'foo1.png',
'001.foo1.png'))
self.assertEqual(config.get('1', CHMJobCreator.CONFIG_MODEL),
'model')
self.assertEqual(
config.get('1', CHMJobCreator.CONFIG_DISABLE_HISTEQ_IMAGES),
'True')
self.assertEqual(
config.get('1', CHMJobCreator.CONFIG_TASKS_PER_NODE), '1')
config = opts.get_merge_config()
res = config.getboolean(CHMJobCreator.CONFIG_DEFAULT,
CHMJobCreator.MERGE_GENTIFS)
self.assertEqual(res, True)
res = config.get('1', CHMJobCreator.MERGE_OUTPUT_IMAGE)
self.assertEqual(
res,
os.path.join(CHMJobCreator.PROBMAPS_DIR,
'foo1.png' + CHMJobCreator.PMAP_SUFFIX))
finally:
shutil.rmtree(temp_dir)
def test_get_submit_script_path(self):
opts = CHMConfig('images', 'model', 'out', '500x500', '20x20')
gen = GordonCluster(None)
val = gen._get_submit_script_path()
self.assertEqual(val, GordonCluster.SUBMIT_SCRIPT_NAME)
gen = GordonCluster(opts)
val = gen._get_submit_script_path()
script = os.path.join(opts.get_out_dir(),
GordonCluster.SUBMIT_SCRIPT_NAME)
self.assertEqual(val, script)
def test_constructor(self):
opts = CHMConfig(None, None, None, None, None)
self.assertEqual(opts.get_images(), None)
self.assertEqual(opts.get_model(), None)
self.assertEqual(opts.get_out_dir(), None)
self.assertEqual(opts.get_tile_size(), None)
self.assertEqual(opts.get_overlap_size(), None)
self.assertEqual(opts.get_tile_height(), '')
self.assertEqual(opts.get_tile_width(), '')
self.assertEqual(opts.get_overlap_height(), '')
self.assertEqual(opts.get_overlap_width(), '')
self.assertEqual(opts.get_number_tasks_per_node(), 1)
self.assertEqual(opts.get_number_tiles_per_task(), 1)
self.assertEqual(opts.get_disable_histogram_eq_val(), True)
self.assertEqual(opts.get_config(), None)
self.assertEqual(opts.get_job_config(), CHMJobCreator.CONFIG_FILE_NAME)
self.assertEqual(opts.get_batchedjob_config_file_path(),
CHMJobCreator.CONFIG_BATCHED_TASKS_FILE_NAME)
self.assertEqual(opts.get_batched_mergejob_config_file_path(),
CHMJobCreator.MERGE_CONFIG_BATCHED_TASKS_FILE_NAME)
self.assertEqual(opts.get_run_dir(), CHMJobCreator.RUN_DIR)
self.assertEqual(opts.get_script_bin(), '')
self.assertEqual(opts.get_job_name(), 'chmjob')
self.assertEqual(opts.get_walltime(), '12:00:00')
self.assertEqual(opts.get_max_image_pixels(), 768000000)
self.assertEqual(opts.get_merge_walltime(), '12:00:00')
self.assertEqual(opts.get_mergejob_name(), 'mergechmjob')
self.assertEqual(opts.get_max_chm_memory_in_gb(), 10)
self.assertEqual(opts.get_max_merge_memory_in_gb(), 20)
self.assertEqual(opts.get_account(), '')
opts = CHMConfig('images',
'model',
'out',
'500x600',
'20x30',
number_tiles_per_task=122,
tasks_per_node=12,
disablehisteq=False,
config='hi',
scriptbin='/foo',
jobname='yo',
walltime='1:2:3',
mergewalltime='4:5:6',
mergejobname='mergy',
max_image_pixels=10,
max_chm_memory_in_gb=5,
max_merge_memory_in_gb=7,
account='yo12')
self.assertEqual(opts.get_images(), 'images')
self.assertEqual(opts.get_model(), 'model')
self.assertEqual(opts.get_out_dir(), 'out')
self.assertEqual(opts.get_tile_size(), '500x600')
self.assertEqual(opts.get_overlap_size(), '20x30')
self.assertEqual(opts.get_tile_height(), 600)
self.assertEqual(opts.get_tile_width(), 500)
self.assertEqual(opts.get_overlap_height(), 30)
self.assertEqual(opts.get_overlap_width(), 20)
self.assertEqual(opts.get_number_tasks_per_node(), 12)
self.assertEqual(opts.get_number_tiles_per_task(), 122)
self.assertEqual(opts.get_disable_histogram_eq_val(), False)
self.assertEqual(opts.get_config(), 'hi')
self.assertEqual(opts.get_job_config(),
os.path.join('out', CHMJobCreator.CONFIG_FILE_NAME))
self.assertEqual(
opts.get_batchedjob_config_file_path(),
os.path.join('out', CHMJobCreator.CONFIG_BATCHED_TASKS_FILE_NAME))
self.assertEqual(
opts.get_batched_mergejob_config_file_path(),
os.path.join('out',
CHMJobCreator.MERGE_CONFIG_BATCHED_TASKS_FILE_NAME))
self.assertEqual(
opts.get_run_dir(),
os.path.join(opts.get_out_dir(), CHMJobCreator.RUN_DIR))
self.assertEqual(opts.get_script_bin(), '/foo')
self.assertEqual(opts.get_job_name(), 'yo')
self.assertEqual(opts.get_walltime(), '1:2:3')
self.assertEqual(opts.get_max_image_pixels(), 10)
self.assertEqual(opts.get_merge_walltime(), '4:5:6')
self.assertEqual(opts.get_mergejob_name(), 'mergy')
self.assertEqual(
opts.get_stdout_dir(),
os.path.join(opts.get_run_dir(), CHMJobCreator.STDOUT_DIR))
self.assertEqual(
opts.get_merge_stdout_dir(),
os.path.join(opts.get_run_dir(), CHMJobCreator.MERGE_STDOUT_DIR))
self.assertEqual(opts.get_max_chm_memory_in_gb(), 5)
self.assertEqual(opts.get_max_merge_memory_in_gb(), 7)
opts.set_config('bye')
self.assertEqual(opts.get_config(), 'bye')
self.assertEqual(opts.get_account(), 'yo12')