def test_generate_batched_tasks_list_none_for_configfile(self):
gen = BatchedTasksListGenerator(1)
try:
gen.write_batched_config(None, None)
self.fail('Expected InvalidConfigFileError')
except InvalidConfigFileError as e:
self.assertEqual(str(e), 'configfile passed in cannot be null')
def test_generate_batched_tasks_list_no_tasks(self):
temp_dir = tempfile.mkdtemp()
try:
gen = BatchedTasksListGenerator(1)
cfile = os.path.join(temp_dir, 'foo.config')
self.assertEqual(gen.write_batched_config(cfile, []), 0)
finally:
shutil.rmtree(temp_dir)
def test_generate_batched_tasks_list_none_for_task_lists(self):
temp_dir = tempfile.mkdtemp()
try:
gen = BatchedTasksListGenerator(1)
cfile = os.path.join(temp_dir, 'foo.config')
try:
gen.write_batched_config(cfile, None)
self.fail("Expected InvalidTaskListError")
except InvalidTaskListError as e:
self.assertEqual(str(e), 'task list cannot be None')
finally:
shutil.rmtree(temp_dir)
def test_write_batched_task_config_no_preexisting_config(self):
temp_dir = tempfile.mkdtemp()
try:
gen = BatchedTasksListGenerator(1)
cfile = os.path.join(temp_dir, 'foo.config')
bconfig = configparser.ConfigParser()
bconfig.set('', 'somekey', 'val')
gen._write_batched_task_config(bconfig, cfile)
self.assertTrue(os.path.isfile(cfile))
finally:
shutil.rmtree(temp_dir)
def test_generate_batched_tasks_list_one_task_one_task_per_node(self):
temp_dir = tempfile.mkdtemp()
try:
gen = BatchedTasksListGenerator(1)
cfile = os.path.join(temp_dir, 'foo.config')
self.assertEqual(gen.write_batched_config(cfile, ['1']), 1)
self.assertTrue(os.path.isfile(cfile))
bconfig = configparser.ConfigParser()
bconfig.read(cfile)
self.assertEqual(bconfig.get('1', CHMJobCreator.BCONFIG_TASK_ID),
'1')
self.assertEqual(bconfig.sections(), ['1'])
finally:
shutil.rmtree(temp_dir)
def _submit(chmconfig, chm_task_list, merge_task_list):
"""Generates new configuration files and outputs commands
to submit incomplete CHM and merge tasks
"""
cfac = ClusterFactory()
clust = cfac.get_cluster_by_name(chmconfig.get_cluster())
if clust is None:
logger.error('Cluster not supported: ' + chmconfig.get_cluster())
return 2
clust.set_chmconfig(chmconfig)
num_chm_tasks = len(chm_task_list)
if num_chm_tasks > 0:
batcher = BatchedTasksListGenerator(
chmconfig.get_number_tasks_per_node())
logger.info('Found ' + str(num_chm_tasks) +
' CHM tasks that need submission')
chm_con_file = chmconfig.get_batchedjob_config_file_path()
logger.info('Batched config file path: ' + chm_con_file)
return _submit_chm_tasks(batcher, chm_con_file, chm_task_list, clust)
num_merge_tasks = len(merge_task_list)
if num_merge_tasks > 0:
# TODO modify code to write these out even on incomplete job
# TODO with any merge jobs that CAN be safely run
batcher = BatchedTasksListGenerator(
chmconfig.get_number_merge_tasks_per_node())
logger.info('Found ' + str(num_merge_tasks) +
' Merge tasks that need submission')
mer_con_file = chmconfig.get_batched_mergejob_config_file_path()
logger.info('Batched config file path: ' + mer_con_file)
return _submit_merge_tasks(batcher, mer_con_file, merge_task_list,
clust)
sys.stdout.write('\nAll jobs completed. Have a nice day!\n\n')
return 0
def test_write_batched_task_config_with_preexisting_config(self):
temp_dir = tempfile.mkdtemp()
try:
gen = BatchedTasksListGenerator(1)
cfile = os.path.join(temp_dir, 'foo.config')
bconfig = configparser.ConfigParser()
bconfig.set('', 'somekey', 'val')
gen._write_batched_task_config(bconfig, cfile)
self.assertTrue(os.path.isfile(cfile))
bconfig.set('', 'somekey', 'anotherval')
gen._write_batched_task_config(bconfig, cfile)
ocfile = cfile + BatchedTasksListGenerator.OLD_SUFFIX
self.assertTrue(os.path.isfile(ocfile))
bconfig.read(ocfile)
self.assertEqual(bconfig.get('DEFAULT', 'somekey'), 'val')
bconfig.read(cfile)
self.assertEqual(bconfig.get('DEFAULT', 'somekey'), 'anotherval')
finally:
shutil.rmtree(temp_dir)