def test_GIVEN_model_with_parameters_WHEN_get_parameter_value_THEN_parameter_value_returned(self):
with session_scope(Session) as session:
user = User()
user.name = 'user1'
session.add(user)
session.commit()
param = session.query(Parameter).first()
param_id = param.id
param_name = param.name
param_namelist = param.namelist.name
model_run = ModelRun()
model_run.name = "MR1"
model_run.user_id = user.id
model_run.status = self._status(constants.MODEL_RUN_STATUS_CREATED)
session.add(model_run)
session.commit()
parameter1 = ParameterValue()
parameter1.parameter_id = param_id
parameter1.set_value_from_python("param1 value")
parameter1.model_run_id = model_run.id
session.add(parameter1)
model_run_returned = self.model_run_service.get_model_being_created_with_non_default_parameter_values(user)
param_value_returned = model_run_returned.get_python_parameter_value([param_namelist, param_name])
assert_that(param_value_returned, is_("param1 value"))
def test_GIVEN_model_run_has_parameters_WHEN_get_model_run_by_id_THEN_model_run_has_parameter_values_loaded(self):
# Add a user and give them a model
user = self.login()
model_run = self.create_run_model(0, "MR1", user)
with session_scope(Session) as session:
# Add parameter value
parameter_value = ParameterValue()
parameter_value.parameter_id = 1
parameter_value.set_value_from_python(123)
parameter = Parameter()
parameter.name = "Param"
parameter.parameter_values = [parameter_value]
# Give them a model
model_run = session.query(ModelRun).filter(ModelRun.id == model_run.id).one()
model_run.parameter_values = [parameter_value]
session.add(model_run)
with session_scope(Session) as session:
model_run_id = session.query(ModelRun).filter(ModelRun.name == model_run.name).one().id
# Get the users model runs
model_run_returned = self.model_run_service.get_model_by_id(user, model_run_id)
pv = model_run_returned.parameter_values[0]
assert_that(pv.value, is_('123'))
assert_that(pv.parameter.name, is_("Param"))
def duplicate_run_model(self, model_id, user):
"""
Duplicate the run model and all its parameters to a new run model
:param model_id: model run id to duplicate
:param user: the user duplicating the model
:return: nothing
"""
id_for_user_upload_driving_dataset = self._dataset_service.get_id_for_user_upload_driving_dataset()
self.delete_model_run_being_created(user)
with self.transaction_scope() as session:
model_run_to_duplicate = session\
.query(ModelRun)\
.join(ModelRunStatus)\
.outerjoin(ParameterValue)\
.outerjoin(LandCoverAction) \
.filter(ModelRun.id == model_id) \
.filter(or_(
ModelRun.user_id == user.id,
ModelRunStatus.name == constants.MODEL_RUN_STATUS_PUBLISHED,
ModelRunStatus.name == constants.MODEL_RUN_STATUS_PUBLIC,)) \
.options(contains_eager(ModelRun.parameter_values)) \
.options(contains_eager(ModelRun.land_cover_actions)) \
.one()
new_model_run_name = model_run_to_duplicate.name
if self._is_duplicate_name(model_run_to_duplicate.name, session, user):
new_model_run_name = "{} (Copy)".format(model_run_to_duplicate.name)
copy_id = 1
while self._is_duplicate_name(new_model_run_name, session, user):
copy_id += 1
new_model_run_name = "{} (Copy {})".format(model_run_to_duplicate.name, copy_id)
new_model_run = ModelRun()
new_model_run.duplicate_from(model_run_to_duplicate)
new_model_run.name = new_model_run_name
new_model_run.user = user
new_model_run.change_status(session, constants.MODEL_RUN_STATUS_CREATED)
for parameter_value in model_run_to_duplicate.parameter_values:
new_parameter = ParameterValue()
new_parameter.duplicate_from(parameter_value)
new_parameter.model_run = new_model_run
for land_cover_action in model_run_to_duplicate.land_cover_actions:
new_land_cover_action = LandCoverAction()
new_land_cover_action.duplicate_from(land_cover_action)
new_land_cover_action.model_run = new_model_run
session.add(new_model_run)
if model_run_to_duplicate.driving_dataset_id == id_for_user_upload_driving_dataset:
try:
self._job_runner_client.duplicate_uploaded_driving_data(model_run_to_duplicate.id, new_model_run.id)
except ServiceException:
self.delete_model_run_being_created(user)
raise ServiceException("Could not duplicate the model run because "
"the user uploaded data can not be duplicated")
def _copy_parameter_set_into_model(self, parameter_values, model_run, session):
"""
Copy the parameters into a model
:param parameter_values: the values to copy
:param model_run: the model run to copy them to
:param session: the session to use
:return: nothing
"""
for parameter_value in parameter_values:
val = ParameterValue()
val.value = parameter_value.value
val.parameter_id = parameter_value.parameter_id
val.model_run = model_run
session.add(val)
def test_GIVEN_valid_output_and_model_run_WHEN_get_output_variable_name_THEN_output_variable_returned_(self):
with session_scope() as session:
param_val_output = ParameterValue()
param_val_output.set_value_from_python('gpp_gb')
parameter_service = ParameterService()
param_output = parameter_service.get_parameter_by_constant(constants.JULES_PARAM_OUTPUT_VAR, session)
param_output.parameter_values = [param_val_output]
model_run = ModelRun()
model_run.parameter_values = [param_val_output]
session.add(model_run)
session.commit()
var = self.parameter_service.get_output_variable_name(model_run.id, param_val_output.id)
assert_that(var, is_('gpp_gb'))
def _create_parameter_value_for_parameter(self, parameters, parameter_dict_name, parameter_dict_value, group_id):
"""
Create parameter value for a parameter
:param parameters: the parameters list
:param parameter_dict_name: the name of the parameter in the dictionary
:param parameter_dict_value: the value in the dictionary
:param group_id: group id
:return: the parameter value
"""
for parameter in parameters:
if parameter_dict_name.lower() == parameter.name.lower():
pv = ParameterValue()
pv.parameter = parameter
pv.group_id = group_id
pv.value = parameter_dict_value
return pv
log.critical("Can not find matching parameter for %s" % parameter_dict_name)
exit()
class TestParameterValue(BaseTest):
def setUp(self):
self.param_val = ParameterValue()
def test_GIVEN_nothing_WHEN_set_python_datetime_THEN_value_set_to_fortran_datetime(self):
value_py = datetime.datetime(1969, 7, 21, 20, 17, 0)
value_ft = "'1969-07-21 20:17:00'"
self.param_val.set_value_from_python(value_py)
assert_that(self.param_val.value, is_(value_ft))
def test_GIVEN_value_is_fortran_datetime_WHEN_get_value_THEN_returns_python_datetime(self):
value_py = datetime.datetime(1969, 7, 21, 20, 17, 0)
value_ft = "'1969-07-21 20:17:00'"
self.param_val.value = value_ft
assert_that(self.param_val.get_value_as_python(), is_(value_py))
def test_GIVEN_nothing_WHEN_set_python_string_THEN_value_set_to_fortran_string(self):
value_py = "String text"
value_ft = "'String text'"
self.param_val.set_value_from_python(value_py)
assert_that(self.param_val.value, is_(value_ft))
def test_GIVEN_value_is_fortran_string_WHEN_get_value_THEN_returns_python_string(self):
value_py = "String text"
value_ft = "'String text'"
self.param_val.set_value_from_python(value_py)
assert_that(self.param_val.value, is_(value_ft))
def test_GIVEN_nothing_WHEN_set_python_list_THEN_value_set_to_fortran_list(self):
value_py = [123.4, True, 100, datetime.datetime(1969, 7, 21, 20, 17, 0)]
value_ft = "123.4 .true. 100 '1969-07-21 20:17:00'"
self.param_val.set_value_from_python(value_py)
assert_that(self.param_val.value, is_(value_ft))
def test_GIVEN_value_is_fortran_list_WHEN_get_value_THEN_return_python_list(self):
value_py = [123.4, 3e4, 100]
value_ft = "123.4 3e4 100"
self.param_val.value = value_ft
assert_that(self.param_val.get_value_as_python(True), is_(value_py))
def setUp(self):
super(TestModelRunOutput, self).setUp()
self.clean_database()
self.user = self.login()
self.valid_params = {
'submit': u'Next',
'ov_select_1': 1,
'ov_yearly_1': 1,
'ov_monthly_1': 1,
'ov_select_2': 1,
'ov_hourly_2': 1,
'ov_daily_2': 1,
'ov_select_3': 1,
'ov_monthly_3': 1
}
self.model_run_service = ModelRunService()
param_run_start = self.model_run_service.get_parameter_by_constant(JULES_PARAM_RUN_START)
param_run_end = self.model_run_service.get_parameter_by_constant(JULES_PARAM_RUN_END)
with session_scope(Session) as session:
self.model_run = self.model_run_service._create_new_model_run(session, self.user)
self.model_run.name = "MR1"
pv_run_start = ParameterValue()
pv_run_start.parameter_id = param_run_start.id
pv_run_start.value = "'1901-01-01 00:00:00'"
pv_run_end = ParameterValue()
pv_run_end.parameter_id = param_run_end.id
pv_run_end.value = "'1902-01-01 00:00:00'"
self.model_run.parameter_values = [pv_run_start, pv_run_end]
session.add(self.model_run)
def store_parameter_values(self, parameters_to_set, user):
"""
Store the parameter values in the database
:param parameters_to_set: dictionary of parameter ids and parameter values
:param user: the logged in user
:return: Nothing
"""
with self.transaction_scope() as session:
model_run = self._get_model_run_being_created(session, user)
session.query(ParameterValue) \
.filter(ParameterValue.model_run == model_run) \
.delete()
parameters = self._get_parameters_for_creating_model(session, user)
for parameter in parameters:
if parameter.id in parameters_to_set.keys():
parameter_value = parameters_to_set[parameter.id]
if parameter_value is not None:
val = ParameterValue()
val.set_value_from_python(parameter_value)
val.parameter = parameter
val.model_run = model_run
session.add(val)
def setUp(self):
self.clean_database()
self.user = self.login()
self.create_two_driving_datasets()
dds = DatasetService().get_driving_datasets(self.user)[0]
with session_scope() as session:
model_run = ModelRun()
model_run.name = "model run"
model_run.change_status(session, MODEL_RUN_STATUS_CREATED)
model_run.user = self.user
model_run.driving_dataset_id = dds.id
session.add(model_run)
session.commit()
self.model_run_service = ModelRunService()
model_run = self.model_run_service._get_model_run_being_created(session, self.user)
parameter_val = ParameterValue()
parameter_val.parameter = self.model_run_service.get_parameter_by_constant(JULES_PARAM_LATLON_REGION)
parameter_val.set_value_from_python(True)
parameter_val.model_run_id = model_run.id
session.add(parameter_val)
self.add_land_cover_region(model_run)
def alter_yearly_monthly_output_profiles(self, parameters):
"""
Adjusts the output profiles to avoid JULES errors like:
"Jules error:file_ts_open: When using data_period=-1, data must start at 00:00:00 on 1st of month"
caused by asking for output profiles that aren't valid for the selected run starts and ends
:param parameters: List of parameters
:return:
"""
run_start = utils.get_first_parameter_value_from_parameter_list(parameters,
constants.JULES_PARAM_RUN_START)
run_end = utils.get_first_parameter_value_from_parameter_list(parameters,
constants.JULES_PARAM_RUN_END)
output_starts_to_add = []
for parameter in parameters:
if parameter.namelist.name == constants.JULES_PARAM_OUTPUT_PERIOD[0]:
if parameter.name == constants.JULES_PARAM_OUTPUT_PERIOD[1]:
for pv_output_period in parameter.parameter_values:
period = pv_output_period.get_value_as_python()
group_id = pv_output_period.group_id
if period == constants.JULES_YEARLY_PERIOD:
if not utils.is_first_of_year(run_start):
next_year = utils.next_first_of_year(run_start)
if next_year <= run_end:
output_start = ParameterValue()
output_start.set_value_from_python(next_year)
output_start.group_id = group_id
output_starts_to_add.append(output_start)
elif period == constants.JULES_MONTHLY_PERIOD:
if not utils.is_first_of_month(run_start):
next_month = utils.next_first_of_month(run_start)
if next_month <= run_end:
output_start = ParameterValue()
output_start.set_value_from_python(next_month)
output_start.group_id = group_id
output_starts_to_add.append(output_start)
# Add parameter values (output_start)
for parameter in parameters:
if parameter.namelist.name == constants.JULES_PARAM_OUTPUT_START[0]:
if parameter.name == constants.JULES_PARAM_OUTPUT_START[1]:
parameter.parameter_values += output_starts_to_add
return parameters
def _create_uploaded_driving_dataset(self, start_date, end_date, lat, lon, model_run_service):
class UploadedDrivingDataset(object):
"""
An uploaded driving dataset
"""
def __init__(self):
self.id = DatasetService().get_id_for_user_upload_driving_dataset()
self.driving_data_lat = None
self.driving_data_lon = None
self.driving_data_rows = None
self.parameter_values = []
driving_dataset = UploadedDrivingDataset()
driving_dataset.driving_data_lat = lat
driving_dataset.driving_data_lon = lon
driving_dataset.driving_data_rows = self.n_lines
driving_dataset.parameter_values = []
params_to_save = [[constants.JULES_PARAM_DRIVE_DATA_START, start_date],
[constants.JULES_PARAM_DRIVE_DATA_END, end_date],
[constants.JULES_PARAM_DRIVE_DATA_PERIOD, self.period],
[constants.JULES_PARAM_DRIVE_FILE, constants.USER_UPLOAD_FILE_NAME],
[constants.JULES_PARAM_DRIVE_NVARS, len(self.var_list or [])],
[constants.JULES_PARAM_DRIVE_VAR, self.var_list],
[constants.JULES_PARAM_DRIVE_INTERP, self.interp_list],
[constants.JULES_PARAM_LATLON_LATITUDE, lat],
[constants.JULES_PARAM_LATLON_LONGITUDE, lon],
[constants.JULES_PARAM_FRAC_FILE, constants.FRACTIONAL_FILENAME]]
for param in params_to_save:
parameter = model_run_service.get_parameter_by_constant(param[0])
param_val = ParameterValue()
param_val.set_value_from_python(param[1])
param_val.parameter_id = parameter.id
driving_dataset.parameter_values.append(param_val)
user_upload_id = self.dataset_service.get_id_for_user_upload_driving_dataset()
ds = self.dataset_service.get_driving_dataset_by_id(user_upload_id)
for pv in ds.parameter_values:
parameter_value = ParameterValue()
parameter_value.parameter_id = pv.parameter_id
parameter_value.value = pv.value
driving_dataset.parameter_values.append(parameter_value)
return driving_dataset
def setUp(self):
super(TestModelRunExtents, self).setUp()
self.clean_database()
self.user = self.login()
self.model_run_service = ModelRunService()
with session_scope(Session) as session:
self.driving_data = DrivingDataset()
self.driving_data.name = "d1"
self.driving_data.boundary_lat_north = 47.5
self.driving_data.boundary_lat_south = 13.8
self.driving_data.boundary_lon_east = 123.1
self.driving_data.boundary_lon_west = -15.0
self.driving_data.time_start = datetime.datetime(1901, 1, 1, 0, 0, 0)
self.driving_data.time_end = datetime.datetime(2001, 1, 1, 0, 0, 0)
session.add(self.driving_data)
session.commit()
self.model_run = ModelRun()
self.model_run.name = "MR1"
self.model_run.status = self._status(MODEL_RUN_STATUS_CREATED)
self.model_run.driving_dataset_id = self.driving_data.id
self.model_run.user = self.user
self.model_run.science_configuration_id = 2
param1 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_INTERP)
pv1 = ParameterValue()
pv1.parameter_id = param1.id
pv1.set_value_from_python(8 * ['nf'])
param2 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_DATA_PERIOD)
pv2 = ParameterValue()
pv2.parameter_id = param2.id
pv2.set_value_from_python(60 * 60)
self.model_run.parameter_values = [pv1, pv2]
session.add(self.model_run)
def _create_new_model_run(self, session, user):
"""
Create a brand new model run for user
:param session: session to use
:param user: Currently logged in user
:return: New model run
"""
model_run = ModelRun()
model_run.change_status(session, constants.MODEL_RUN_STATUS_CREATED)
model_run.user = user
parameters = [
[constants.JULES_PARAM_TIMESTEP_LEN, constants.TIMESTEP_LEN, None],
[constants.JULES_PARAM_OUTPUT_RUN_ID, constants.RUN_ID, None],
[constants.JULES_PARAM_OUTPUT_OUTPUT_DIR, "./" + constants.OUTPUT_DIR, None],
]
# Add CHESS defaults:
chess_defaults = ['surf_roff', 'sub_surf_roff', 'fqw_gb', 'rad_net', 'ftl_gb', 'gpp_gb', 'resp_p_gb',
'tstar_gb', 'snow_mass_gb', 't_soil', 'smc_tot', 'smcl', 'swet_liq_tot']
chess_periods = [(constants.JULES_MONTHLY_PERIOD, "_monthly")]
group_id = 0
for output_variable in chess_defaults:
for period, period_profile_name in chess_periods:
parameters.append([constants.JULES_PARAM_OUTPUT_VAR, output_variable, group_id])
parameters.append([constants.JULES_PARAM_OUTPUT_PERIOD, period, group_id])
parameters.append([constants.JULES_PARAM_OUTPUT_PROFILE_NAME,
output_variable + period_profile_name, group_id])
parameters.append([constants.JULES_PARAM_OUTPUT_NVARS, 1, group_id])
parameters.append([constants.JULES_PARAM_OUTPUT_MAIN_RUN, True, group_id])
parameters.append([constants.JULES_PARAM_OUTPUT_TYPE, 'M', group_id])
group_id += 1
parameters.append([constants.JULES_PARAM_OUTPUT_NPROFILES, group_id, None])
for constant, value, group_id in parameters:
param = self.parameter_service.get_parameter_by_constant(constant, session)
param_value = ParameterValue()
param_value.parameter = param
param_value.model_run = model_run
param_value.group_id = group_id
param_value.set_value_from_python(value)
return model_run
def save_parameter(self, model_run, param_namelist_name, value, session, group_id=None):
"""
Save parameter using a supplied session
:param model_run: Model run to save against
:param param_namelist_name: List containing the parameter namelist, name
:param session: Session to use
:param value: Value to set
:param group_id: Specify an optional group_id to group parameters
:return:
"""
parameter = self.get_parameter_by_constant(param_namelist_name, session)
try:
parameter_value = session.query(ParameterValue) \
.filter(ParameterValue.model_run_id == model_run.id) \
.filter(ParameterValue.group_id == group_id) \
.filter(ParameterValue.parameter_id == parameter.id).one()
except NoResultFound:
parameter_value = ParameterValue()
parameter_value.parameter = parameter
parameter_value.model_run = model_run
parameter_value.group_id = group_id
parameter_value.set_value_from_python(value)
session.add(parameter_value)
def setUp(self):
self.param_val = ParameterValue()
def set_up_single_cell_model_run(self):
self.clean_database()
self.user = self.login()
self.create_two_driving_datasets()
user_upload_id = DatasetService().get_id_for_user_upload_driving_dataset()
with session_scope(Session) as session:
self.model_run = ModelRun()
self.model_run.name = "MR1"
self.model_run.status = self._status(MODEL_RUN_STATUS_CREATED)
self.model_run.driving_dataset_id = user_upload_id
self.model_run.user = self.user
self.model_run.driving_data_lat = 51.75
self.model_run.driving_data_lon = -0.25
self.model_run.driving_data_rows = 248
param1 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_INTERP)
pv1 = ParameterValue()
pv1.parameter_id = param1.id
pv1.set_value_from_python(8 * ['nf'])
param2 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_DATA_PERIOD)
pv2 = ParameterValue()
pv2.parameter_id = param2.id
pv2.set_value_from_python(60 * 60)
param3 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_DATA_START)
pv3 = ParameterValue()
pv3.parameter_id = param3.id
pv3.value = "'1901-01-01 00:00:00'"
param4 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_DATA_END)
pv4 = ParameterValue()
pv4.parameter_id = param4.id
pv4.value = "'1901-01-31 21:00:00'"
param5 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_LATLON_REGION)
pv5 = ParameterValue()
pv5.parameter_id = param5.id
pv5.value = ".false."
param6 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_POINTS_FILE)
pv6 = ParameterValue()
pv6.parameter_id = param6.id
pv6.value = "51.75 -0.25"
self.model_run.parameter_values = [pv1, pv2, pv3, pv4, pv5, pv6]
session.add(self.model_run)
def set_up_single_cell_user_driving_data(self):
self.clean_database()
del self.driving_data
self.user = self.login()
user_upload_id = DatasetService().get_id_for_user_upload_driving_dataset()
with session_scope(Session) as session:
self.model_run = ModelRun()
self.model_run.name = "MR1"
self.model_run.status = self._status(MODEL_RUN_STATUS_CREATED)
self.model_run.driving_dataset_id = user_upload_id
self.model_run.user = self.user
self.model_run.driving_data_lat = 25
self.model_run.driving_data_lon = 40
self.model_run.driving_data_rows = 248
self.model_run.science_configuration_id = 2
param1 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_INTERP)
pv1 = ParameterValue()
pv1.parameter_id = param1.id
pv1.set_value_from_python(8 * ['nf'])
param2 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_DATA_PERIOD)
pv2 = ParameterValue()
pv2.parameter_id = param2.id
pv2.set_value_from_python(60 * 60)
param3 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_DATA_START)
pv3 = ParameterValue()
pv3.parameter_id = param3.id
pv3.value = "'1901-01-01 00:00:00'"
param4 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_DATA_END)
pv4 = ParameterValue()
pv4.parameter_id = param4.id
pv4.value = "'1901-01-31 21:00:00'"
self.model_run.parameter_values = [pv1, pv2, pv3, pv4]
session.add(self.model_run)
