def test_mini_readme_with_non_int_base_runs(self):
decorated = add_content_to_temp_inputfilepath(get_job_info_from_readme)
content = """
base-run (job-id): this is some accident text
"""
readme_info = decorated(content)
self.assertEqual(readme_info["base_runs"], [])
def test_existing_directory(self):
decorated = add_content_to_temp_inputfilepath(
get_cluster_scratch_dir_from_script)
with tempfile.TemporaryDirectory() as tempdir:
scratch_dir = os.path.join(tempdir, "1234")
os.makedirs(scratch_dir)
content = "cd {}*".format(scratch_dir)
self.assertEqual(decorated(content), scratch_dir)
def test_mini_readme_with_two_comma_sep_base_runs(self):
decorated = add_content_to_temp_inputfilepath(get_job_info_from_readme)
base_run_1 = 1234567
base_run_2 = 7654321
content = f"""
base-run (job-id): {base_run_1}, {base_run_2}
"""
readme_info = decorated(content)
self.assertEqual(readme_info["base_runs"], [base_run_1, base_run_2])
def test_existing_directory_with_following_newline(self):
logger.info("Testing existing directory with following newline")
decorated = add_content_to_temp_inputfilepath(
get_cluster_scratch_dir_from_script)
with tempfile.TemporaryDirectory() as tempdir:
scratch_dir = os.path.join(tempdir, "1234")
os.makedirs(scratch_dir)
content = "cd {}*\n".format(scratch_dir)
self.assertEqual(decorated(content), scratch_dir)
def test_mini_readme_with_empty_info(self):
decorated = add_content_to_temp_inputfilepath(get_job_info_from_readme)
content = """
information :
********Header********
"""
readme_info = decorated(content)
self.assertEqual(readme_info["info_block"], "")
def test_warning_log_when_empty_joblogfile(self):
"""Test logging of warning when file is empty"""
logger.info("Test warning log when empty joblogfile")
decorated = add_content_to_temp_inputfilepath(
get_job_info_from_joblogfile)
content = ""
with self.assertLogs(logger="utils.caefileio.joblogfile",
level=logging.WARNING) as cm:
job_id, sub_dir, log_date = decorated(content)
logger.info("Logs of required level: {}".format(cm.output))
def test_read_empty_joblogfile(self):
"""Test output for info missing from file"""
logger.info("Test empty joblogfile")
decorated = add_content_to_temp_inputfilepath(
get_job_info_from_joblogfile)
content = ""
job_id, sub_dir, log_date = decorated(content)
self.assertIsNone(job_id)
self.assertIsNone(sub_dir)
self.assertIsNone(log_date)
def test_non_existing_directory(self):
"""
The cluster scratch directory that is contained in the file should be
returned even if it is not existing. The handling/check if the
directory exists should be handled on the caller level.
The purpose of this function is merely to read the available
information from the clusterscript.
"""
decorated = add_content_to_temp_inputfilepath(
get_cluster_scratch_dir_from_script)
scratch_dir = "/this/is/not/existing/1234"
content = "cd {}*".format(scratch_dir)
self.assertEqual(decorated(content), scratch_dir)
def test_warning_log_when_useless_content_joblogfile(self):
"""Test logging of warning when file contains useless text"""
logger.info("Test warning log when joblogfile contains useless text")
decorated = add_content_to_temp_inputfilepath(
get_job_info_from_joblogfile)
content = """This is just some text.
It is completely useless to derive information from.
This is not what a joblogfile should look like.
If this happens... something when wrong.
"""
with self.assertLogs(logger="utils.caefileio.joblogfile",
level=logging.WARNING) as cm:
job_id, sub_dir, log_date = decorated(content)
logger.info("Logs of required level: {}".format(cm.output))
def test_sub_dir_in_content_not_exsits(self):
logger.info("Test sub dir does not exist")
decorated = add_content_to_temp_inputfilepath(
get_job_info_from_joblogfile)
given_job_id = 1234567
content = """
job_number: {}
sge_o_workdir: /this/sub_dir/does/not/exsit
submission_time: Fri Jul 27 08:28:38 2018
""".format(given_job_id)
extracted_job_id, sub_dir, log_date = decorated(content)
self.assertEqual(extracted_job_id, given_job_id)
self.assertEqual(sub_dir, "/this/sub_dir/does/not/exsit")
self.assertEqual(log_date, datetime(
year=2018, month=7, day=27, hour=8, minute=28, second=38))
def test_sub_dir_value_missing_in_content(self):
"""
Test sub dir value missing from content
"""
logger.info("Test sge_o_workdir value missing")
decorated = add_content_to_temp_inputfilepath(
get_job_info_from_joblogfile)
given_job_id = 1234567
content = """
job_number: {}
sge_o_workdir:
submission_time: Fri Jul 27 08:28:38 2018
""".format(given_job_id)
extracted_job_id, sub_dir, log_date = decorated(content)
self.assertEqual(extracted_job_id, given_job_id)
self.assertIsNone(sub_dir)
self.assertEqual(log_date, datetime(
year=2018, month=7, day=27, hour=8, minute=28, second=38))
def test_read_existing_info_id_first(self):
"""
Test output for info existing in file (job number then sub dir)
"""
logger.info("Test joblogfile with expected content -- number first")
decorated = add_content_to_temp_inputfilepath(
get_job_info_from_joblogfile)
with tempfile.TemporaryDirectory() as tempdir:
given_job_id = 1234567
content = """
job_number: {}
sge_o_workdir: {}
submission_time: Fri Jul 27 08:28:38 2018
""".format(given_job_id, tempdir)
extracted_job_id, sub_dir, log_date = decorated(content)
self.assertEqual(extracted_job_id, given_job_id)
self.assertEqual(sub_dir, tempdir)
self.assertEqual(log_date, datetime(
year=2018, month=7, day=27, hour=8, minute=28, second=38))
def test_time_value_missing_in_content(self):
"""
Test job id values missing from content.
Once the missing job id is discovered, the processing should end and
the sub_dir should not be found e.g. stay empty.
"""
logger.info("Test submission_time value missing")
decorated = add_content_to_temp_inputfilepath(
get_job_info_from_joblogfile)
with tempfile.TemporaryDirectory() as tempdir:
given_job_id = 1234567
content = """
job_number: {}
sge_o_workdir: {}
submission_time:
""".format(given_job_id, tempdir)
extracted_job_id, sub_dir, log_date = decorated(content)
self.assertEqual(extracted_job_id, given_job_id)
self.assertEqual(sub_dir, tempdir)
self.assertIsNone(log_date)
def test_job_id_value_missing_in_content(self):
"""
Test job id values missing from content.
Once the missing job id is discovered, the processing should end and
the sub_dir should not be found e.g. stay empty.
"""
logger.info("Test job_number value missing")
decorated = add_content_to_temp_inputfilepath(
get_job_info_from_joblogfile)
with tempfile.TemporaryDirectory() as tempdir:
content = """
job_number:
sge_o_workdir: {}
submission_time: Fri Jul 27 08:28:38 2018
""".format(tempdir)
job_id, sub_dir, log_date = decorated(content)
self.assertIsNone(job_id)
self.assertEqual(sub_dir, tempdir)
self.assertEqual(log_date, datetime(
year=2018, month=7, day=27, hour=8, minute=28, second=38))
def test_existing_readme_with_real_content(self):
decorated = add_content_to_temp_inputfilepath(get_job_info_from_readme)
main_name = "0696_AD_W222_SLD_FRB_56_TH_p1_SIB_relax45_noEnv_.key"
base_run = 7654321
info_block = """Dissipate more energy
------
New load limiter level
Load limiter level: x = 5.2 kN
Changed tension flag in spring material.
The spring did not develop any force when streched!"""
username = "******"
email = "*****@*****.**"
solver = "dyn"
content = f"""
README for {main_name}
base-run (job-id): {base_run}
information :
{info_block}
********Header********
SOLVERVER: mpp.s.R9.1.0.113698.113698_dmp_sp
FEMZIPCOMPRESS: y
FEMZIP: /share/sge/femzip/scripts/femzip_sge_W04_dyn_crash_mmkgms
SGESCHED: i
CORES: 16
SGERES: none=1
SOLVER: dyn
SOLPAR: MEMORY=512M MEMORY2=256M
SGECMD: /some/directory/path/with/multiple-levels/{main_name}/sge_post_script___master-file.sh >> sge_post_script___master-file.log 2>&1
UNIT: mm-kg-ms
******Environment******
Sub-User: {username}
EMail: {email}
Sub-Date: 2018-06-07__17:21:21
Sub-Host: x01xx001
Queue: dyn
Qsub: /usr/ge-6.1u6/bin/lx24-amd64/qsub -S /bin/ksh -M {email} -o /tmp -e /tmp -m b -p -101 -N i_{main_name} -V -pe dmp_16 16 -q dyn -l none=1,dyn=1,sensor_dyn=1 /clients/bin/_sge/_dyn-dmp
OS: CentOS release 6.6 (Final)
Cores: 16
CPUSUM: 1
R_COUNT: 0
P_COUNT: 0
PRIO: -101
RES: none=1,dyn=1,sensor_dyn=1
Solver: {solver}
SolverVer: mpp.s.R9.1.0.113698.113698_dmp_sp
SolverPath: /share/sge/dyn/mpp.s.R9.1.0.113698.113698_dmp_sp/x86_64/bin/dyna.bin
DMP/SMP: dmp
Sched: i
Scheddef: i
SubDir: /some/directory/path/with/multiple-levels/{main_name}
FILE: {main_name}
Femzip: y
FemzipConf: /share/sge/femzip/scripts/femzip_sge_W04_dyn_crash_mmkgms
SOLPAR: MEMORY=512M MEMORY2=256M
SGECMD: /some/directory/path/with/multiple-levels/{main_name}/sge_post_script___master-file.sh >> sge_post_script___master-file.log 2>&1
SGERES: none=1
JOBID: 1234567
Exec-Host: x99xx123.example.com 16 [email protected] <NULL>
"""
readme_info = decorated(content)
self.assertEqual(type(readme_info), type(dict()))
self.assertEqual(readme_info["main_name"], main_name)
self.assertEqual(readme_info["base_runs"], [base_run])
self.assertEqual(readme_info["info_block"], info_block)
self.assertEqual(readme_info["username"], username)
self.assertEqual(readme_info["email"], email)
self.assertEqual(
readme_info["sub_date"],
datetime(year=2018, month=6, day=7, hour=17, minute=21, second=21))
self.assertEqual(readme_info["solver"], "dyn")
def test_empty_script(self):
decorated = add_content_to_temp_inputfilepath(
get_cluster_scratch_dir_from_script)
content = ""
self.assertIsNone(decorated(content))
def test_missing_directory(self):
decorated = add_content_to_temp_inputfilepath(
get_cluster_scratch_dir_from_script)
content = "cd "
self.assertIsNone(decorated(content))
