def test_GIVEN_monthly_output_but_run_starts_mid_month_WHEN_alter_output_profiles_THEN_output_start_added(self):
job_runner_client = JobRunnerClient(config)
param1 = Parameter(name='main_run_start')
param1.namelist = Namelist(name='JULES_TIME')
param1_val = "'1901-6-25 12:00:00'"
param1.parameter_values = [ParameterValue(value=param1_val)]
param2 = Parameter(name='main_run_end')
param2.namelist = Namelist(name='JULES_TIME')
param2_val = "'1904-02-01 00:00:00'"
param2.parameter_values = [ParameterValue(value=param2_val)]
param3 = Parameter(name='output_period')
param3.namelist = Namelist(name='JULES_OUTPUT_PROFILE')
param3_val = "%s" % constants.JULES_MONTHLY_PERIOD
param3.parameter_values = [ParameterValue(value=param3_val, group_id=1)]
param4 = Parameter(name='output_start')
param4.namelist = Namelist(name='JULES_OUTPUT_PROFILE')
param4.parameter_values = []
parameters = [param1, param2, param3, param4]
parameters = job_runner_client.alter_yearly_monthly_output_profiles(parameters)
output_start = utils.get_first_parameter_value_from_parameter_list(parameters,
constants.JULES_PARAM_OUTPUT_START,
group_id=1)
assert_that(output_start, is_(datetime.datetime(1901, 7, 1)))
def _create_json_land_cover_with_land_cover_actions(self, land_cover_actions, parameters):
"""
Create JSON land cover object with land cover actions
:param land_cover_actions: List of land cover actions to be applied
:param parameters: List of parameters for model run
:return: JSON land cover object.
"""
# Identify the base land cover file, and rename the parameter to be the new file we'll create
# Identify the name of the base land cover variable
json_land_cover = self._create_json_land_cover(parameters)
# Add the ice index
ice_index_from_parameters = utils.get_first_parameter_value_from_parameter_list(
parameters,
constants.JULES_PARAM_MODEL_LEVELS_ICE_INDEX)
if ice_index_from_parameters is None:
ice_index_from_parameters = constants.JULES_PARAM_MODEL_LEVELS_ICE_INDEX_DEFAULT_VALUE
json_land_cover[constants.JSON_LAND_COVER_ICE_INDEX] = ice_index_from_parameters
# Add the land cover actions
json_actions = []
for action in land_cover_actions:
json_action = {constants.JSON_LAND_COVER_MASK_FILE: action.region.mask_file,
constants.JSON_LAND_COVER_VALUE: action.value_id,
constants.JSON_LAND_COVER_ORDER: action.order}
json_actions.append(json_action)
json_land_cover[constants.JSON_LAND_COVER_ACTIONS] = json_actions
json_land_cover[constants.JSON_LAND_COVER_POINT_EDIT] = {}
return json_land_cover
def alter_yearly_monthly_output_profiles(self, parameters):
"""
Adjusts the output profiles to avoid JULES errors like:
"Jules error:file_ts_open: When using data_period=-1, data must start at 00:00:00 on 1st of month"
caused by asking for output profiles that aren't valid for the selected run starts and ends
:param parameters: List of parameters
:return:
"""
run_start = utils.get_first_parameter_value_from_parameter_list(parameters,
constants.JULES_PARAM_RUN_START)
run_end = utils.get_first_parameter_value_from_parameter_list(parameters,
constants.JULES_PARAM_RUN_END)
output_starts_to_add = []
for parameter in parameters:
if parameter.namelist.name == constants.JULES_PARAM_OUTPUT_PERIOD[0]:
if parameter.name == constants.JULES_PARAM_OUTPUT_PERIOD[1]:
for pv_output_period in parameter.parameter_values:
period = pv_output_period.get_value_as_python()
group_id = pv_output_period.group_id
if period == constants.JULES_YEARLY_PERIOD:
if not utils.is_first_of_year(run_start):
next_year = utils.next_first_of_year(run_start)
if next_year <= run_end:
output_start = ParameterValue()
output_start.set_value_from_python(next_year)
output_start.group_id = group_id
output_starts_to_add.append(output_start)
elif period == constants.JULES_MONTHLY_PERIOD:
if not utils.is_first_of_month(run_start):
next_month = utils.next_first_of_month(run_start)
if next_month <= run_end:
output_start = ParameterValue()
output_start.set_value_from_python(next_month)
output_start.group_id = group_id
output_starts_to_add.append(output_start)
# Add parameter values (output_start)
for parameter in parameters:
if parameter.namelist.name == constants.JULES_PARAM_OUTPUT_START[0]:
if parameter.name == constants.JULES_PARAM_OUTPUT_START[1]:
parameter.parameter_values += output_starts_to_add
return parameters
def alter_driving_data_start(self, parameters):
"""
Moves the driving data start to be closer to the main run start
Due to a bug in JULES if the driving data start is too far before the main run start,
an integer overflows and causes an error. We therefore need to move the driving data
start close to the model run
:param parameters: List of parameters
:return:
"""
driving_start = utils.get_first_parameter_value_from_parameter_list(parameters,
constants.JULES_PARAM_DRIVE_DATA_START)
spinup_start = utils.get_first_parameter_value_from_parameter_list(parameters,
constants.JULES_PARAM_SPINUP_START)
time_gap = relativedelta(years=10)
new_driving_start = max(spinup_start - time_gap, driving_start)
utils.set_parameter_value_in_parameter_list(
parameters, constants.JULES_PARAM_DRIVE_DATA_START, new_driving_start)
return parameters
def _create_json_land_cover(self, parameters):
"""
Create a JSON land cover object with the land cover base filename, variable key and empty
actions and single point edits
:param parameters: Parameters for model run
:return: JSON land cover object
"""
json_land_cover = {}
fractional_base_filename = utils.get_first_parameter_value_from_parameter_list(
parameters, constants.JULES_PARAM_FRAC_FILE)
utils.set_parameter_value_in_parameter_list(
parameters, constants.JULES_PARAM_FRAC_FILE, constants.USER_EDITED_FRACTIONAL_FILENAME)
fractional_base_variable_key = utils.get_first_parameter_value_from_parameter_list(
parameters, constants.JULES_PARAM_FRAC_NAME)
# Create the land cover JSON object
json_land_cover[constants.JSON_LAND_COVER_BASE_FILE] = fractional_base_filename
json_land_cover[constants.JSON_LAND_COVER_BASE_KEY] = fractional_base_variable_key
json_land_cover[constants.JSON_LAND_COVER_ACTIONS] = []
json_land_cover[constants.JSON_LAND_COVER_POINT_EDIT] = {}
return json_land_cover
def _create_json_land_cover_with_single_point_edits(self, parameters, run_model):
"""
Create a JSON land cover object for single point land cover edits
:param parameters: List of parameters
:param run_model: Model run
:return: JSON land cover object
"""
json_land_cover = self._create_json_land_cover(parameters)
lat, lon = utils.get_first_parameter_value_from_parameter_list(
parameters, constants.JULES_PARAM_POINTS_FILE, is_list=True)
vals = [float(val) for val in run_model.land_cover_frac.split()]
land_cover_point_edit = {constants.JSON_LAND_COVER_LAT: lat,
constants.JSON_LAND_COVER_LON: lon,
constants.JSON_LAND_COVER_FRACTIONAL_VALS: vals}
json_land_cover[constants.JSON_LAND_COVER_POINT_EDIT] = land_cover_point_edit
return json_land_cover
def test_GIVEN_spinup_start_at_beginning_of_driving_data_WHEN_alter_driving_data_start_THEN_driving_data_start_unmoved(self):
job_runner_client = JobRunnerClient(config)
param1 = Parameter(name='data_start')
param1.namelist = Namelist(name='JULES_DRIVE')
param1_val = "'1901-01-01 00:00:00'"
param1.parameter_values = [ParameterValue(value=param1_val)]
param2 = Parameter(name='spinup_start')
param2.namelist = Namelist(name='JULES_SPINUP')
param2_val = "'1901-01-01 00:00:00'"
param2.parameter_values = [ParameterValue(value=param2_val)]
parameters = [param1, param2]
parameters = job_runner_client.alter_driving_data_start(parameters)
dd_start = utils.get_first_parameter_value_from_parameter_list(parameters,
constants.JULES_PARAM_DRIVE_DATA_START)
assert_that(dd_start, is_(datetime.datetime(1901, 1, 1)))
def convert_model_to_dictionary(self, run_model, parameters, land_cover_actions):
"""
Convert the run model from the database object to a dictionary that can be sent to the job runner service
:param run_model: the run model
:param parameters: Model run parameters
:param land_cover_actions:
:return: dictionary for the run model
"""
parameters = self.alter_driving_data_start(parameters)
parameters = self.alter_yearly_monthly_output_profiles(parameters)
# We need to identify if using fractional file, edited land cover or default .nc file
# If multicell AND land cover actions saved --> use edited fractional.nc file
# If multicell AND no land cover actions saved --> leave the .nc file as is
# If single cell AND user uploaded --> use a fractional.dat file.
# If single cell AND not user uploaded --> use an edited fractional.nc file
multicell = utils.get_first_parameter_value_from_parameter_list(
parameters, constants.JULES_PARAM_LATLON_REGION) or False
user_uploaded = utils.get_first_parameter_value_from_parameter_list(
parameters, constants.JULES_PARAM_DRIVE_FILE) == constants.USER_UPLOAD_FILE_NAME
json_land_cover = {}
if multicell:
if land_cover_actions:
json_land_cover = self._create_json_land_cover_with_land_cover_actions(land_cover_actions, parameters)
else:
if user_uploaded:
json_land_cover = self._create_fractional_file_and_set_parameters(parameters, run_model)
else:
json_land_cover = self._create_json_land_cover_with_single_point_edits(parameters, run_model)
namelist_files = []
for parameter in parameters:
namelist_file = self._find_or_create_namelist_file(namelist_files,
parameter.namelist.namelist_file.filename)
if len(parameter.parameter_values) == 0:
self._find_or_create_namelist(
namelist_file[constants.JSON_MODEL_NAMELISTS],
parameter.namelist.name,
parameter.namelist.index_in_file,
None)
else:
for parameter_value in parameter.parameter_values:
namelist = self._find_or_create_namelist(
namelist_file[constants.JSON_MODEL_NAMELISTS],
parameter.namelist.name,
parameter.namelist.index_in_file,
parameter_value.group_id)
namelist[constants.JSON_MODEL_PARAMETERS][parameter.name] = parameter_value.value
return \
{
constants.JSON_MODEL_RUN_ID: run_model.id,
constants.JSON_MODEL_CODE_VERSION: run_model.code_version.name,
constants.JSON_MODEL_NAMELIST_FILES: namelist_files,
constants.JSON_USER_ID: run_model.user.id,
constants.JSON_USER_NAME: run_model.user.username,
constants.JSON_USER_EMAIL: run_model.user.email,
constants.JSON_LAND_COVER: json_land_cover
}
