def test_get_task_summary(self):
con = CHMConfig('./images', './model', './outdir', '500x500', '20x20')
cfig = configparser.ConfigParser()
cfig.add_section('1')
cfig.set('1', 'hi', 'val')
cfig.add_section('2')
cfig.set('2', 'hi', 'val')
con.set_config(cfig)
mfig = configparser.ConfigParser()
mfig.add_section('3')
mfig.set('3', 'hi', 'val')
con.set_merge_config(mfig)
tsf = TaskSummaryFactory(con,
chm_incomplete_tasks=[],
merge_incomplete_tasks=['a'])
ts = tsf.get_task_summary()
self.assertEqual(
ts.get_summary(), 'chmutil version: unknown\n'
'Tiles: 500x500 with 20x20 '
'overlap\nDisable histogram '
'equalization in CHM: True\n'
'Tasks: 1 tiles per task, 1 '
'tasks(s) per node\nTrained '
'CHM model: ./model\nCHM binary: '
'./chm-0.1.0.img\n\nCHM tasks: '
'100% complete (2 of 2 completed)'
'\nMerge tasks: 0% complete (0 of '
'1 completed)\n')
def test_get_image_stats_summary_output_compute_true_one_image(self):
temp_dir = tempfile.mkdtemp()
try:
im = Image.new('L', (10, 10))
iname = os.path.join(temp_dir, '1.png')
im.save(iname)
expsize = os.path.getsize(iname)
con = CHMConfig(temp_dir, './model', './outdir', '500x500',
'20x20')
cfig = configparser.ConfigParser()
cfig.add_section('1')
cfig.set('1', 'hi', 'val')
cfig.add_section('2')
cfig.set('2', 'hi', 'val')
con.set_config(cfig)
mfig = configparser.ConfigParser()
mfig.add_section('3')
mfig.set('3', 'hi', 'val')
con.set_merge_config(mfig)
tsf = TaskSummaryFactory(con,
chm_incomplete_tasks=[],
merge_incomplete_tasks=[],
output_compute=True)
isum = tsf._get_image_stats_summary()
self.assertEqual(isum.get_image_count(), 1)
self.assertEqual(isum.get_total_pixels(), 100)
self.assertEqual(isum.get_total_size_of_images_in_bytes(), expsize)
self.assertEqual(isum.get_image_dimensions_as_dict(),
{(10, 10): 1})
finally:
shutil.rmtree(temp_dir)
def test_get_image_stats_summary_output_compute_true_no_images(self):
temp_dir = tempfile.mkdtemp()
try:
con = CHMConfig(temp_dir, './model', './outdir', '500x500',
'20x20')
cfig = configparser.ConfigParser()
cfig.add_section('1')
cfig.set('1', 'hi', 'val')
cfig.add_section('2')
cfig.set('2', 'hi', 'val')
con.set_config(cfig)
mfig = configparser.ConfigParser()
mfig.add_section('3')
mfig.set('3', 'hi', 'val')
con.set_merge_config(mfig)
tsf = TaskSummaryFactory(con,
chm_incomplete_tasks=[],
merge_incomplete_tasks=[],
output_compute=True)
isum = tsf._get_image_stats_summary()
self.assertEqual(isum.get_image_count(), 0)
finally:
shutil.rmtree(temp_dir)
def test_get_image_stats_summary_output_compute_false(self):
con = CHMConfig('./images', './model', './outdir', '500x500', '20x20')
cfig = configparser.ConfigParser()
cfig.add_section('1')
cfig.set('1', 'hi', 'val')
cfig.add_section('2')
cfig.set('2', 'hi', 'val')
con.set_config(cfig)
mfig = configparser.ConfigParser()
mfig.add_section('3')
mfig.set('3', 'hi', 'val')
con.set_merge_config(mfig)
tsf = TaskSummaryFactory(con,
chm_incomplete_tasks=[],
merge_incomplete_tasks=[])
isum = tsf._get_image_stats_summary()
self.assertEqual(isum, None)
def test_get_chm_task_stats(self):
con = CHMConfig('./images', './model', './outdir', '500x500', '20x20')
cfig = configparser.ConfigParser()
cfig.add_section('1')
cfig.set('1', 'hi', 'val')
cfig.add_section('2')
cfig.set('2', 'hi', 'val')
cfig.add_section('3')
cfig.set('3', 'hi', 'val')
cfig.add_section('4')
cfig.set('4', 'hi', 'val')
# try with none for lists
con.set_config(cfig)
tsf = TaskSummaryFactory(con)
ts = tsf._get_chm_task_stats()
self.assertEqual(ts.get_completed_task_count(), 0)
self.assertEqual(ts.get_total_task_count(), 4)
# try with empty lists
tsf = TaskSummaryFactory(con,
chm_incomplete_tasks=[],
merge_incomplete_tasks=[])
ts = tsf._get_chm_task_stats()
self.assertEqual(ts.get_completed_task_count(), 4)
self.assertEqual(ts.get_total_task_count(), 4)
# try with lists with elements
tsf = TaskSummaryFactory(con,
chm_incomplete_tasks=['hi'],
merge_incomplete_tasks=['a', 'b'])
ts = tsf._get_chm_task_stats()
self.assertEqual(ts.get_completed_task_count(), 3)
self.assertEqual(ts.get_total_task_count(), 4)
# try with lists with elements 2 incomplete
tsf = TaskSummaryFactory(con,
chm_incomplete_tasks=['hi', 'bye'],
merge_incomplete_tasks=['a', 'b'])
ts = tsf._get_chm_task_stats()
self.assertEqual(ts.get_completed_task_count(), 2)
self.assertEqual(ts.get_total_task_count(), 4)
def test_constructor(self):
opts = CHMConfig(None, None, None, None, None)
self.assertEqual(opts.get_images(), None)
self.assertEqual(opts.get_model(), None)
self.assertEqual(opts.get_out_dir(), None)
self.assertEqual(opts.get_tile_size(), None)
self.assertEqual(opts.get_overlap_size(), None)
self.assertEqual(opts.get_tile_height(), '')
self.assertEqual(opts.get_tile_width(), '')
self.assertEqual(opts.get_overlap_height(), '')
self.assertEqual(opts.get_overlap_width(), '')
self.assertEqual(opts.get_number_tasks_per_node(), 1)
self.assertEqual(opts.get_number_tiles_per_task(), 1)
self.assertEqual(opts.get_disable_histogram_eq_val(), True)
self.assertEqual(opts.get_config(), None)
self.assertEqual(opts.get_job_config(), CHMJobCreator.CONFIG_FILE_NAME)
self.assertEqual(opts.get_batchedjob_config_file_path(),
CHMJobCreator.CONFIG_BATCHED_TASKS_FILE_NAME)
self.assertEqual(opts.get_batched_mergejob_config_file_path(),
CHMJobCreator.MERGE_CONFIG_BATCHED_TASKS_FILE_NAME)
self.assertEqual(opts.get_run_dir(), CHMJobCreator.RUN_DIR)
self.assertEqual(opts.get_script_bin(), '')
self.assertEqual(opts.get_job_name(), 'chmjob')
self.assertEqual(opts.get_walltime(), '12:00:00')
self.assertEqual(opts.get_max_image_pixels(), 768000000)
self.assertEqual(opts.get_merge_walltime(), '12:00:00')
self.assertEqual(opts.get_mergejob_name(), 'mergechmjob')
self.assertEqual(opts.get_max_chm_memory_in_gb(), 10)
self.assertEqual(opts.get_max_merge_memory_in_gb(), 20)
self.assertEqual(opts.get_account(), '')
opts = CHMConfig('images',
'model',
'out',
'500x600',
'20x30',
number_tiles_per_task=122,
tasks_per_node=12,
disablehisteq=False,
config='hi',
scriptbin='/foo',
jobname='yo',
walltime='1:2:3',
mergewalltime='4:5:6',
mergejobname='mergy',
max_image_pixels=10,
max_chm_memory_in_gb=5,
max_merge_memory_in_gb=7,
account='yo12')
self.assertEqual(opts.get_images(), 'images')
self.assertEqual(opts.get_model(), 'model')
self.assertEqual(opts.get_out_dir(), 'out')
self.assertEqual(opts.get_tile_size(), '500x600')
self.assertEqual(opts.get_overlap_size(), '20x30')
self.assertEqual(opts.get_tile_height(), 600)
self.assertEqual(opts.get_tile_width(), 500)
self.assertEqual(opts.get_overlap_height(), 30)
self.assertEqual(opts.get_overlap_width(), 20)
self.assertEqual(opts.get_number_tasks_per_node(), 12)
self.assertEqual(opts.get_number_tiles_per_task(), 122)
self.assertEqual(opts.get_disable_histogram_eq_val(), False)
self.assertEqual(opts.get_config(), 'hi')
self.assertEqual(opts.get_job_config(),
os.path.join('out', CHMJobCreator.CONFIG_FILE_NAME))
self.assertEqual(
opts.get_batchedjob_config_file_path(),
os.path.join('out', CHMJobCreator.CONFIG_BATCHED_TASKS_FILE_NAME))
self.assertEqual(
opts.get_batched_mergejob_config_file_path(),
os.path.join('out',
CHMJobCreator.MERGE_CONFIG_BATCHED_TASKS_FILE_NAME))
self.assertEqual(
opts.get_run_dir(),
os.path.join(opts.get_out_dir(), CHMJobCreator.RUN_DIR))
self.assertEqual(opts.get_script_bin(), '/foo')
self.assertEqual(opts.get_job_name(), 'yo')
self.assertEqual(opts.get_walltime(), '1:2:3')
self.assertEqual(opts.get_max_image_pixels(), 10)
self.assertEqual(opts.get_merge_walltime(), '4:5:6')
self.assertEqual(opts.get_mergejob_name(), 'mergy')
self.assertEqual(
opts.get_stdout_dir(),
os.path.join(opts.get_run_dir(), CHMJobCreator.STDOUT_DIR))
self.assertEqual(
opts.get_merge_stdout_dir(),
os.path.join(opts.get_run_dir(), CHMJobCreator.MERGE_STDOUT_DIR))
self.assertEqual(opts.get_max_chm_memory_in_gb(), 5)
self.assertEqual(opts.get_max_merge_memory_in_gb(), 7)
opts.set_config('bye')
self.assertEqual(opts.get_config(), 'bye')
self.assertEqual(opts.get_account(), 'yo12')
def test_get_chm_task_stats_outputcompute_true(self):
temp_dir = tempfile.mkdtemp()
try:
con = CHMConfig('./images', './model', temp_dir, '500x500',
'20x20')
run_dir = os.path.join(temp_dir, CHMJobCreator.RUN_DIR)
stdout_dir = os.path.join(run_dir, CHMJobCreator.STDOUT_DIR)
os.makedirs(stdout_dir, mode=0o755)
new_format_file2 = os.path.join(stdout_dir, '778786.2')
f = open(new_format_file2, 'w')
f.write('a\n\b\nc\n')
f.write(' User time (seconds): 100.00\n')
f.write(' System time (seconds): 200.00\n')
f.write(' Percent of CPU this job got: 1591%\n')
f.write(' Elapsed (wall clock) time (h:mm:ss or m:ss): 20:15\n')
f.write('Average shared text size (kbytes): 0\n')
f.write('Average unshared data size (kbytes): 0\n')
f.write('Average stack size (kbytes): 0\n')
f.write(' Maximum resident set size (kbytes): 5287148\n')
f.flush()
f.close()
cfig = configparser.ConfigParser()
cfig.add_section('1')
cfig.set('1', 'hi', 'val')
cfig.add_section('2')
cfig.set('2', 'hi', 'val')
cfig.add_section('3')
cfig.set('3', 'hi', 'val')
cfig.add_section('4')
cfig.set('4', 'hi', 'val')
con.set_config(cfig)
# try with lists with elements
tsf = TaskSummaryFactory(con,
chm_incomplete_tasks=['hi'],
merge_incomplete_tasks=['a', 'b'],
output_compute=True)
ts = tsf._get_chm_task_stats()
self.assertEqual(ts.get_completed_task_count(), 3)
self.assertEqual(ts.get_total_task_count(), 4)
self.assertEqual(ts.get_total_tasks_with_cputimes(), 1)
self.assertEqual(ts.get_max_memory_in_kb(), 5287148)
self.assertEqual(ts.get_total_cpu_usertime(), 100)
self.assertEqual(ts.get_total_cpu_walltime(), 1215)
# try with a second output file
oldformatfile = os.path.join(stdout_dir, '1234.1')
f = open(oldformatfile, 'w')
f.write('HOST: comet-22-63\nDATE: blah\ns\n\n')
f.write('real 150.0\nuser 250.0\nsys 60.0\n')
f.write('chmrunner.py exited with code: 0\n')
f.flush()
f.close()
tsf = TaskSummaryFactory(con,
chm_incomplete_tasks=['hi'],
merge_incomplete_tasks=['a', 'b'],
output_compute=True)
ts = tsf._get_chm_task_stats()
self.assertEqual(ts.get_completed_task_count(), 3)
self.assertEqual(ts.get_total_task_count(), 4)
self.assertEqual(ts.get_total_tasks_with_cputimes(), 2)
self.assertEqual(ts.get_max_memory_in_kb(), 5287148)
self.assertEqual(ts.get_total_cpu_usertime(), 350)
self.assertEqual(ts.get_total_cpu_walltime(), 1365)
finally:
shutil.rmtree(temp_dir)
