def test_GIVEN_single_point_land_cover_edit_WHEN_submit_job_THEN_land_cover_editor_called_correctly(self):
land_cover_dict = {
JSON_LAND_COVER_BASE_FILE: "data/ancils/frac.nc",
JSON_LAND_COVER_BASE_KEY: "frac",
JSON_LAND_COVER_ACTIONS: [],
JSON_LAND_COVER_POINT_EDIT: {
JSON_LAND_COVER_LAT: 51.75,
JSON_LAND_COVER_LON: -0.25,
JSON_LAND_COVER_FRACTIONAL_VALS: 8 * [0.0] + [1.0],
},
}
model_run_json = self.model_run
model_run_json[JSON_LAND_COVER] = land_cover_dict
expected_copied_base_path = self.run_dir + "/user_edited_land_cover_fractional_file.nc"
land_cover_editor = LandCoverEditor()
land_cover_editor.copy_land_cover_base_map = MagicMock(return_value=expected_copied_base_path)
land_cover_editor.apply_land_cover_action = MagicMock()
land_cover_editor.apply_single_point_fractional_cover = MagicMock()
job_service = JobService(land_cover_editor=land_cover_editor)
job_service.submit(model_run_json)
c_path, c_vals, c_lat, c_lon, c_key = land_cover_editor.apply_single_point_fractional_cover.call_args_list[0][0]
assert_that(c_path, is_(expected_copied_base_path))
assert_that(c_lat, is_(51.75))
assert_that(c_lon, is_(-0.25))
assert_that(c_vals, is_(8 * [0.0] + [1.0]))
def test_GIVEN_code_version_script_doesnt_exist_WHEN_submit_job_is_submitted_THEN_error_thrown(self):
try:
VALID_CODE_VERSIONS["invalid_script"] = "doesnt exist.sh"
local_job_service = JobService(VALID_CODE_VERSIONS)
self.model_run["code_version"] = "invalid_script"
local_job_service.submit(self.model_run)
except ServiceException as ex:
assert_that(ex.message, is_("Unknown exception."), "error message does not contain details")
return
self.fail("Should have thrown an exception")
def setUp(self):
self.model_run_id = 201
self.run_dir = config["run_dir"] + "/run201"
self.user_id = 1
self.user_name = "test"
self.user_email = "*****@*****.**"
if os.path.exists(self.run_dir):
shutil.rmtree(self.run_dir)
self.job_service = JobService()
self.model_run = {
"model_run_id": self.model_run_id,
"code_version": "Jules v3.4.1",
JSON_USER_ID: self.user_id,
JSON_USER_EMAIL: self.user_email,
JSON_USER_NAME: self.user_name,
JSON_LAND_COVER: {},
"namelist_files": [
{
"filename": "timesteps.nml",
"namelists": [
{"name": "JULES_TIME", JSON_MODEL_NAMELIST_INDEX: 1, "parameters": {"timestep_len": "1800"}}
],
}
],
}
def test_GIVEN_land_cover_actions_WHEN_submit_job_THEN_land_cover_editor_called_correctly(self):
land_cover_dict = {
JSON_LAND_COVER_BASE_FILE: "data/ancils/frac.nc",
JSON_LAND_COVER_BASE_KEY: "frac",
JSON_LAND_COVER_ICE_INDEX: 9,
JSON_LAND_COVER_ACTIONS: [
{
JSON_LAND_COVER_MASK_FILE: "data/masks/mask1.nc",
JSON_LAND_COVER_VALUE: "9",
JSON_LAND_COVER_ORDER: "2",
},
{
JSON_LAND_COVER_MASK_FILE: "data/masks/mask2.nc",
JSON_LAND_COVER_VALUE: "5",
JSON_LAND_COVER_ORDER: "1",
},
],
JSON_LAND_COVER_POINT_EDIT: {},
}
model_run_json = self.model_run
model_run_json[JSON_LAND_COVER] = land_cover_dict
expected_copied_base_path = self.run_dir + "/user_edited_land_cover_fractional_file.nc"
land_cover_editor = LandCoverEditor()
land_cover_editor.copy_land_cover_base_map = Mock(return_value=expected_copied_base_path)
land_cover_editor.apply_land_cover_action = Mock()
job_service = JobService(land_cover_editor=land_cover_editor)
job_service.submit(model_run_json)
# Check that the file was called to copy correctly
called_base_file, called_run_directory = land_cover_editor.copy_land_cover_base_map.call_args_list[0][0]
assert_that(called_base_file, is_("data/ancils/frac.nc"))
assert_that(called_run_directory, is_(self.run_dir))
# Check that the apply action was called correctly the first time
called_base, called_mask, called_value, ice = land_cover_editor.apply_land_cover_action.call_args_list[0][0]
assert_that(called_base, is_(expected_copied_base_path))
assert_that(called_mask, is_(self.run_dir + "/data/masks/mask2.nc"))
assert_that(called_value, is_("5"))
# Check that the apply actions was called correctly the second time
called_base, called_mask, called_value, ice = land_cover_editor.apply_land_cover_action.call_args_list[1][0]
assert_that(called_base, is_(expected_copied_base_path))
assert_that(called_mask, is_(self.run_dir + "/data/masks/mask1.nc"))
assert_that(called_value, is_("9"))
def setUp(self):
self.job_service = JobService()
self.test_filename = 'user_uploaded_driving_data.dat'
self.model_run_id = 1
self.directory = self.job_service.get_run_dir(self.model_run_id)
if not os.path.exists(self.directory):
os.mkdir(self.directory)
self.path = os.path.join(self.directory, self.test_filename)
if os.path.exists(self.path):
os.remove(self.path)
class TestJobFileController(TestController):
"""
Test the functionality of the Job File controller
"""
def setUp(self):
self.job_service = JobService()
self.test_filename = 'user_uploaded_driving_data.dat'
self.model_run_id = 1
self.directory = self.job_service.get_run_dir(self.model_run_id)
if not os.path.exists(self.directory):
os.mkdir(self.directory)
self.path = os.path.join(self.directory, self.test_filename)
if os.path.exists(self.path):
os.remove(self.path)
def test_GIVEN_file_not_exist_WHEN_create_new_THEN_file_created(self):
json_dict = {constants.JSON_MODEL_FILENAME: self.test_filename,
constants.JSON_MODEL_RUN_ID: str(self.model_run_id)}
response = self.app.post_json(url(controller='job_file', action='new'),
params=json_dict)
assert_that(response.status_code, is_(200))
assert_that(os.path.isfile(self.path), is_(True))
def test_GIVEN_file_exists_WHEN_create_new_THEN_file_replaced(self):
test_contents = 'existing\nlines\nof\ndata\n'
# Set up file
f = open(self.path, 'a+')
f.write(test_contents)
f.close()
f = open(self.path, 'r')
assert_that(f.read(), is_(test_contents))
f.close()
# Call web service
json_dict = {constants.JSON_MODEL_FILENAME: self.test_filename,
constants.JSON_MODEL_RUN_ID: str(self.model_run_id)}
response = self.app.post_json(url(controller='job_file', action='new'),
params=json_dict)
assert_that(response.status_code, is_(200))
assert_that(os.path.isfile(self.path), is_(True))
f = open(self.path, 'r')
assert_that(f.read(), is_(''))
def test_GIVEN_file_exists_but_empty_WHEN_append_THEN_file_contains_text(self):
test_line = 'line of data into file\n'
# Set up file
f = open(self.path, 'w+')
# Call web service
json_dict = {constants.JSON_MODEL_FILENAME: self.test_filename,
constants.JSON_MODEL_RUN_ID: str(self.model_run_id),
constants.JSON_MODEL_FILE_LINE: test_line}
response = self.app.post_json(url(controller='job_file', action='append'),
params=json_dict)
assert_that(response.status_code, is_(200))
f = open(self.path, 'r')
assert_that(f.read(), is_(test_line))
def test_GIVEN_file_exists_with_lines_in_WHEN_append_THEN_file_contains_all_text(self):
test_contents = 'existing\nlines\nof\ndata\n'
test_line = 'line of data into file\n'
# Set up file
f = open(self.path, 'a+')
f.write(test_contents)
f.close()
# Call web service
json_dict = {constants.JSON_MODEL_FILENAME: self.test_filename,
constants.JSON_MODEL_RUN_ID: str(self.model_run_id),
constants.JSON_MODEL_FILE_LINE: test_line}
response = self.app.post_json(url(controller='job_file', action='append'),
params=json_dict)
assert_that(response.status_code, is_(200))
f = open(self.path, 'r')
assert_that(f.read(), is_(test_contents + test_line))
def test_GIVEN_file_not_exist_WHEN_append_THEN_file_created_and_text_added(self):
test_line = 'line of data into file\n'
# Call web service
json_dict = {constants.JSON_MODEL_FILENAME: self.test_filename,
constants.JSON_MODEL_RUN_ID: str(self.model_run_id),
constants.JSON_MODEL_FILE_LINE: test_line}
response = self.app.post_json(url(controller='job_file', action='append'),
params=json_dict)
assert_that(response.status_code, is_(200))
f = open(self.path, 'r')
assert_that(f.read(), is_(test_line))
def test_GIVEN_file_created_WHEN_delete_THEN_file_removed(self):
# Set up file
f = open(self.path, 'w+')
f.close()
# Call web service
json_dict = {constants.JSON_MODEL_FILENAME: self.test_filename,
constants.JSON_MODEL_RUN_ID: str(self.model_run_id)}
response = self.app.post_json(url(controller='job_file', action='delete'),
params=json_dict)
assert_that(response.status_code, is_(200))
assert_that(os.path.exists(self.path), is_(False))
def test_GIVEN_file_not_exist_WHEN_delete_THEN_nothing_bad_happens(self):
# Call web service
json_dict = {constants.JSON_MODEL_FILENAME: self.test_filename,
constants.JSON_MODEL_RUN_ID: str(self.model_run_id)}
response = self.app.post_json(url(controller='job_file', action='delete'),
params=json_dict)
assert_that(response.status_code, is_(200))
assert_that(os.path.exists(self.path), is_(False))
class TestJobService(TestController):
def setUp(self):
self.model_run_id = 201
self.run_dir = config["run_dir"] + "/run201"
self.user_id = 1
self.user_name = "test"
self.user_email = "*****@*****.**"
if os.path.exists(self.run_dir):
shutil.rmtree(self.run_dir)
self.job_service = JobService()
self.model_run = {
"model_run_id": self.model_run_id,
"code_version": "Jules v3.4.1",
JSON_USER_ID: self.user_id,
JSON_USER_EMAIL: self.user_email,
JSON_USER_NAME: self.user_name,
JSON_LAND_COVER: {},
"namelist_files": [
{
"filename": "timesteps.nml",
"namelists": [
{"name": "JULES_TIME", JSON_MODEL_NAMELIST_INDEX: 1, "parameters": {"timestep_len": "1800"}}
],
}
],
}
def test_GIVEN_model_run_WHEN_submit_job_is_submitted_THEN_directory_and_files_are_setup(self):
result = self.job_service.submit(self.model_run)
assert_that(os.path.exists(self.run_dir), is_(True), "directory '%s' exists" % self.run_dir)
assert_that(os.path.exists(self.run_dir + "/jules_run.sh"), is_(True), "jules_run.sh in run directory")
assert_that(os.path.lexists(self.run_dir + "/data"), is_(True), "softlink to data in run directory")
assert_that(os.path.exists(self.run_dir + "/timesteps.nml"), is_(True), "timesteps.nml in run directory")
assert_that(result, greater_than(0), "PID is greater than 0")
last_line = ""
f = file(os.path.join(config["run_dir"], "jobs_run.log"))
for line in f:
last_line = line
assert_that(last_line, contains_string(str(result)))
assert_that(last_line, contains_string(str(self.user_id)))
assert_that(last_line, contains_string(self.user_name))
assert_that(last_line, contains_string(str(self.model_run_id)))
assert_that(last_line, contains_string(self.user_email))
def test_GIVEN_model_id_is_not_an_integer_WHEN_submit_job_is_submitted_THEN_exception(self):
self.model_run["model_run_id"] = "../201"
with self.assertRaises(AssertionError):
self.job_service.submit(self.model_run)
def test_GIVEN_code_version_script_doesnt_submit_job_but_runs_WHEN_submit_job_is_submitted_THEN_error_thrown(self):
try:
VALID_CODE_VERSIONS["invalid_script"] = "invalid_script.sh"
local_job_service = JobService(VALID_CODE_VERSIONS)
self.model_run["code_version"] = "invalid_script"
local_job_service.submit(self.model_run)
except ServiceException as ex:
assert_that(ex.message, is_("Unexpected output."), "error message does not contain details")
return
self.fail("Should have thrown an exception")
def test_GIVEN_code_version_script_doesnt_exist_WHEN_submit_job_is_submitted_THEN_error_thrown(self):
try:
VALID_CODE_VERSIONS["invalid_script"] = "doesnt exist.sh"
local_job_service = JobService(VALID_CODE_VERSIONS)
self.model_run["code_version"] = "invalid_script"
local_job_service.submit(self.model_run)
except ServiceException as ex:
assert_that(ex.message, is_("Unknown exception."), "error message does not contain details")
return
self.fail("Should have thrown an exception")
def test_GIVEN_code_version_script_errors_WHEN_submit_job_is_submitted_THEN_error_thrown(self):
try:
VALID_CODE_VERSIONS["invalid_script"] = "error_submit.sh"
local_job_service = JobService(VALID_CODE_VERSIONS)
self.model_run["code_version"] = "invalid_script"
local_job_service.submit(self.model_run)
except ServiceException as ex:
assert_that(ex.message, is_("Unknown error."), "error message does not contain details")
return
self.fail("Should have thrown an exception")
def test_GIVEN_land_cover_actions_WHEN_submit_job_THEN_land_cover_editor_called_correctly(self):
land_cover_dict = {
JSON_LAND_COVER_BASE_FILE: "data/ancils/frac.nc",
JSON_LAND_COVER_BASE_KEY: "frac",
JSON_LAND_COVER_ICE_INDEX: 9,
JSON_LAND_COVER_ACTIONS: [
{
JSON_LAND_COVER_MASK_FILE: "data/masks/mask1.nc",
JSON_LAND_COVER_VALUE: "9",
JSON_LAND_COVER_ORDER: "2",
},
{
JSON_LAND_COVER_MASK_FILE: "data/masks/mask2.nc",
JSON_LAND_COVER_VALUE: "5",
JSON_LAND_COVER_ORDER: "1",
},
],
JSON_LAND_COVER_POINT_EDIT: {},
}
model_run_json = self.model_run
model_run_json[JSON_LAND_COVER] = land_cover_dict
expected_copied_base_path = self.run_dir + "/user_edited_land_cover_fractional_file.nc"
land_cover_editor = LandCoverEditor()
land_cover_editor.copy_land_cover_base_map = Mock(return_value=expected_copied_base_path)
land_cover_editor.apply_land_cover_action = Mock()
job_service = JobService(land_cover_editor=land_cover_editor)
job_service.submit(model_run_json)
# Check that the file was called to copy correctly
called_base_file, called_run_directory = land_cover_editor.copy_land_cover_base_map.call_args_list[0][0]
assert_that(called_base_file, is_("data/ancils/frac.nc"))
assert_that(called_run_directory, is_(self.run_dir))
# Check that the apply action was called correctly the first time
called_base, called_mask, called_value, ice = land_cover_editor.apply_land_cover_action.call_args_list[0][0]
assert_that(called_base, is_(expected_copied_base_path))
assert_that(called_mask, is_(self.run_dir + "/data/masks/mask2.nc"))
assert_that(called_value, is_("5"))
# Check that the apply actions was called correctly the second time
called_base, called_mask, called_value, ice = land_cover_editor.apply_land_cover_action.call_args_list[1][0]
assert_that(called_base, is_(expected_copied_base_path))
assert_that(called_mask, is_(self.run_dir + "/data/masks/mask1.nc"))
assert_that(called_value, is_("9"))
def test_GIVEN_single_point_land_cover_edit_WHEN_submit_job_THEN_land_cover_editor_called_correctly(self):
land_cover_dict = {
JSON_LAND_COVER_BASE_FILE: "data/ancils/frac.nc",
JSON_LAND_COVER_BASE_KEY: "frac",
JSON_LAND_COVER_ACTIONS: [],
JSON_LAND_COVER_POINT_EDIT: {
JSON_LAND_COVER_LAT: 51.75,
JSON_LAND_COVER_LON: -0.25,
JSON_LAND_COVER_FRACTIONAL_VALS: 8 * [0.0] + [1.0],
},
}
model_run_json = self.model_run
model_run_json[JSON_LAND_COVER] = land_cover_dict
expected_copied_base_path = self.run_dir + "/user_edited_land_cover_fractional_file.nc"
land_cover_editor = LandCoverEditor()
land_cover_editor.copy_land_cover_base_map = MagicMock(return_value=expected_copied_base_path)
land_cover_editor.apply_land_cover_action = MagicMock()
land_cover_editor.apply_single_point_fractional_cover = MagicMock()
job_service = JobService(land_cover_editor=land_cover_editor)
job_service.submit(model_run_json)
c_path, c_vals, c_lat, c_lon, c_key = land_cover_editor.apply_single_point_fractional_cover.call_args_list[0][0]
assert_that(c_path, is_(expected_copied_base_path))
assert_that(c_lat, is_(51.75))
assert_that(c_lon, is_(-0.25))
assert_that(c_vals, is_(8 * [0.0] + [1.0]))