def test_get_task_summary(self): con = CHMConfig('./images', './model', './outdir', '500x500', '20x20') cfig = configparser.ConfigParser() cfig.add_section('1') cfig.set('1', 'hi', 'val') cfig.add_section('2') cfig.set('2', 'hi', 'val') con.set_config(cfig) mfig = configparser.ConfigParser() mfig.add_section('3') mfig.set('3', 'hi', 'val') con.set_merge_config(mfig) tsf = TaskSummaryFactory(con, chm_incomplete_tasks=[], merge_incomplete_tasks=['a']) ts = tsf.get_task_summary() self.assertEqual( ts.get_summary(), 'chmutil version: unknown\n' 'Tiles: 500x500 with 20x20 ' 'overlap\nDisable histogram ' 'equalization in CHM: True\n' 'Tasks: 1 tiles per task, 1 ' 'tasks(s) per node\nTrained ' 'CHM model: ./model\nCHM binary: ' './chm-0.1.0.img\n\nCHM tasks: ' '100% complete (2 of 2 completed)' '\nMerge tasks: 0% complete (0 of ' '1 completed)\n')
def test_get_image_stats_summary_output_compute_true_one_image(self): temp_dir = tempfile.mkdtemp() try: im = Image.new('L', (10, 10)) iname = os.path.join(temp_dir, '1.png') im.save(iname) expsize = os.path.getsize(iname) con = CHMConfig(temp_dir, './model', './outdir', '500x500', '20x20') cfig = configparser.ConfigParser() cfig.add_section('1') cfig.set('1', 'hi', 'val') cfig.add_section('2') cfig.set('2', 'hi', 'val') con.set_config(cfig) mfig = configparser.ConfigParser() mfig.add_section('3') mfig.set('3', 'hi', 'val') con.set_merge_config(mfig) tsf = TaskSummaryFactory(con, chm_incomplete_tasks=[], merge_incomplete_tasks=[], output_compute=True) isum = tsf._get_image_stats_summary() self.assertEqual(isum.get_image_count(), 1) self.assertEqual(isum.get_total_pixels(), 100) self.assertEqual(isum.get_total_size_of_images_in_bytes(), expsize) self.assertEqual(isum.get_image_dimensions_as_dict(), {(10, 10): 1}) finally: shutil.rmtree(temp_dir)
def test_get_image_stats_summary_output_compute_true_no_images(self): temp_dir = tempfile.mkdtemp() try: con = CHMConfig(temp_dir, './model', './outdir', '500x500', '20x20') cfig = configparser.ConfigParser() cfig.add_section('1') cfig.set('1', 'hi', 'val') cfig.add_section('2') cfig.set('2', 'hi', 'val') con.set_config(cfig) mfig = configparser.ConfigParser() mfig.add_section('3') mfig.set('3', 'hi', 'val') con.set_merge_config(mfig) tsf = TaskSummaryFactory(con, chm_incomplete_tasks=[], merge_incomplete_tasks=[], output_compute=True) isum = tsf._get_image_stats_summary() self.assertEqual(isum.get_image_count(), 0) finally: shutil.rmtree(temp_dir)
def test_get_image_stats_summary_output_compute_false(self): con = CHMConfig('./images', './model', './outdir', '500x500', '20x20') cfig = configparser.ConfigParser() cfig.add_section('1') cfig.set('1', 'hi', 'val') cfig.add_section('2') cfig.set('2', 'hi', 'val') con.set_config(cfig) mfig = configparser.ConfigParser() mfig.add_section('3') mfig.set('3', 'hi', 'val') con.set_merge_config(mfig) tsf = TaskSummaryFactory(con, chm_incomplete_tasks=[], merge_incomplete_tasks=[]) isum = tsf._get_image_stats_summary() self.assertEqual(isum, None)
def test_get_chm_task_stats(self): con = CHMConfig('./images', './model', './outdir', '500x500', '20x20') cfig = configparser.ConfigParser() cfig.add_section('1') cfig.set('1', 'hi', 'val') cfig.add_section('2') cfig.set('2', 'hi', 'val') cfig.add_section('3') cfig.set('3', 'hi', 'val') cfig.add_section('4') cfig.set('4', 'hi', 'val') # try with none for lists con.set_config(cfig) tsf = TaskSummaryFactory(con) ts = tsf._get_chm_task_stats() self.assertEqual(ts.get_completed_task_count(), 0) self.assertEqual(ts.get_total_task_count(), 4) # try with empty lists tsf = TaskSummaryFactory(con, chm_incomplete_tasks=[], merge_incomplete_tasks=[]) ts = tsf._get_chm_task_stats() self.assertEqual(ts.get_completed_task_count(), 4) self.assertEqual(ts.get_total_task_count(), 4) # try with lists with elements tsf = TaskSummaryFactory(con, chm_incomplete_tasks=['hi'], merge_incomplete_tasks=['a', 'b']) ts = tsf._get_chm_task_stats() self.assertEqual(ts.get_completed_task_count(), 3) self.assertEqual(ts.get_total_task_count(), 4) # try with lists with elements 2 incomplete tsf = TaskSummaryFactory(con, chm_incomplete_tasks=['hi', 'bye'], merge_incomplete_tasks=['a', 'b']) ts = tsf._get_chm_task_stats() self.assertEqual(ts.get_completed_task_count(), 2) self.assertEqual(ts.get_total_task_count(), 4)
def test_constructor(self): opts = CHMConfig(None, None, None, None, None) self.assertEqual(opts.get_images(), None) self.assertEqual(opts.get_model(), None) self.assertEqual(opts.get_out_dir(), None) self.assertEqual(opts.get_tile_size(), None) self.assertEqual(opts.get_overlap_size(), None) self.assertEqual(opts.get_tile_height(), '') self.assertEqual(opts.get_tile_width(), '') self.assertEqual(opts.get_overlap_height(), '') self.assertEqual(opts.get_overlap_width(), '') self.assertEqual(opts.get_number_tasks_per_node(), 1) self.assertEqual(opts.get_number_tiles_per_task(), 1) self.assertEqual(opts.get_disable_histogram_eq_val(), True) self.assertEqual(opts.get_config(), None) self.assertEqual(opts.get_job_config(), CHMJobCreator.CONFIG_FILE_NAME) self.assertEqual(opts.get_batchedjob_config_file_path(), CHMJobCreator.CONFIG_BATCHED_TASKS_FILE_NAME) self.assertEqual(opts.get_batched_mergejob_config_file_path(), CHMJobCreator.MERGE_CONFIG_BATCHED_TASKS_FILE_NAME) self.assertEqual(opts.get_run_dir(), CHMJobCreator.RUN_DIR) self.assertEqual(opts.get_script_bin(), '') self.assertEqual(opts.get_job_name(), 'chmjob') self.assertEqual(opts.get_walltime(), '12:00:00') self.assertEqual(opts.get_max_image_pixels(), 768000000) self.assertEqual(opts.get_merge_walltime(), '12:00:00') self.assertEqual(opts.get_mergejob_name(), 'mergechmjob') self.assertEqual(opts.get_max_chm_memory_in_gb(), 10) self.assertEqual(opts.get_max_merge_memory_in_gb(), 20) self.assertEqual(opts.get_account(), '') opts = CHMConfig('images', 'model', 'out', '500x600', '20x30', number_tiles_per_task=122, tasks_per_node=12, disablehisteq=False, config='hi', scriptbin='/foo', jobname='yo', walltime='1:2:3', mergewalltime='4:5:6', mergejobname='mergy', max_image_pixels=10, max_chm_memory_in_gb=5, max_merge_memory_in_gb=7, account='yo12') self.assertEqual(opts.get_images(), 'images') self.assertEqual(opts.get_model(), 'model') self.assertEqual(opts.get_out_dir(), 'out') self.assertEqual(opts.get_tile_size(), '500x600') self.assertEqual(opts.get_overlap_size(), '20x30') self.assertEqual(opts.get_tile_height(), 600) self.assertEqual(opts.get_tile_width(), 500) self.assertEqual(opts.get_overlap_height(), 30) self.assertEqual(opts.get_overlap_width(), 20) self.assertEqual(opts.get_number_tasks_per_node(), 12) self.assertEqual(opts.get_number_tiles_per_task(), 122) self.assertEqual(opts.get_disable_histogram_eq_val(), False) self.assertEqual(opts.get_config(), 'hi') self.assertEqual(opts.get_job_config(), os.path.join('out', CHMJobCreator.CONFIG_FILE_NAME)) self.assertEqual( opts.get_batchedjob_config_file_path(), os.path.join('out', CHMJobCreator.CONFIG_BATCHED_TASKS_FILE_NAME)) self.assertEqual( opts.get_batched_mergejob_config_file_path(), os.path.join('out', CHMJobCreator.MERGE_CONFIG_BATCHED_TASKS_FILE_NAME)) self.assertEqual( opts.get_run_dir(), os.path.join(opts.get_out_dir(), CHMJobCreator.RUN_DIR)) self.assertEqual(opts.get_script_bin(), '/foo') self.assertEqual(opts.get_job_name(), 'yo') self.assertEqual(opts.get_walltime(), '1:2:3') self.assertEqual(opts.get_max_image_pixels(), 10) self.assertEqual(opts.get_merge_walltime(), '4:5:6') self.assertEqual(opts.get_mergejob_name(), 'mergy') self.assertEqual( opts.get_stdout_dir(), os.path.join(opts.get_run_dir(), CHMJobCreator.STDOUT_DIR)) self.assertEqual( opts.get_merge_stdout_dir(), os.path.join(opts.get_run_dir(), CHMJobCreator.MERGE_STDOUT_DIR)) self.assertEqual(opts.get_max_chm_memory_in_gb(), 5) self.assertEqual(opts.get_max_merge_memory_in_gb(), 7) opts.set_config('bye') self.assertEqual(opts.get_config(), 'bye') self.assertEqual(opts.get_account(), 'yo12')
def test_get_chm_task_stats_outputcompute_true(self): temp_dir = tempfile.mkdtemp() try: con = CHMConfig('./images', './model', temp_dir, '500x500', '20x20') run_dir = os.path.join(temp_dir, CHMJobCreator.RUN_DIR) stdout_dir = os.path.join(run_dir, CHMJobCreator.STDOUT_DIR) os.makedirs(stdout_dir, mode=0o755) new_format_file2 = os.path.join(stdout_dir, '778786.2') f = open(new_format_file2, 'w') f.write('a\n\b\nc\n') f.write(' User time (seconds): 100.00\n') f.write(' System time (seconds): 200.00\n') f.write(' Percent of CPU this job got: 1591%\n') f.write(' Elapsed (wall clock) time (h:mm:ss or m:ss): 20:15\n') f.write('Average shared text size (kbytes): 0\n') f.write('Average unshared data size (kbytes): 0\n') f.write('Average stack size (kbytes): 0\n') f.write(' Maximum resident set size (kbytes): 5287148\n') f.flush() f.close() cfig = configparser.ConfigParser() cfig.add_section('1') cfig.set('1', 'hi', 'val') cfig.add_section('2') cfig.set('2', 'hi', 'val') cfig.add_section('3') cfig.set('3', 'hi', 'val') cfig.add_section('4') cfig.set('4', 'hi', 'val') con.set_config(cfig) # try with lists with elements tsf = TaskSummaryFactory(con, chm_incomplete_tasks=['hi'], merge_incomplete_tasks=['a', 'b'], output_compute=True) ts = tsf._get_chm_task_stats() self.assertEqual(ts.get_completed_task_count(), 3) self.assertEqual(ts.get_total_task_count(), 4) self.assertEqual(ts.get_total_tasks_with_cputimes(), 1) self.assertEqual(ts.get_max_memory_in_kb(), 5287148) self.assertEqual(ts.get_total_cpu_usertime(), 100) self.assertEqual(ts.get_total_cpu_walltime(), 1215) # try with a second output file oldformatfile = os.path.join(stdout_dir, '1234.1') f = open(oldformatfile, 'w') f.write('HOST: comet-22-63\nDATE: blah\ns\n\n') f.write('real 150.0\nuser 250.0\nsys 60.0\n') f.write('chmrunner.py exited with code: 0\n') f.flush() f.close() tsf = TaskSummaryFactory(con, chm_incomplete_tasks=['hi'], merge_incomplete_tasks=['a', 'b'], output_compute=True) ts = tsf._get_chm_task_stats() self.assertEqual(ts.get_completed_task_count(), 3) self.assertEqual(ts.get_total_task_count(), 4) self.assertEqual(ts.get_total_tasks_with_cputimes(), 2) self.assertEqual(ts.get_max_memory_in_kb(), 5287148) self.assertEqual(ts.get_total_cpu_usertime(), 350) self.assertEqual(ts.get_total_cpu_walltime(), 1365) finally: shutil.rmtree(temp_dir)