def test_GIVEN_model_with_parameters_WHEN_get_parameter_value_THEN_parameter_value_returned(self): with session_scope(Session) as session: user = User() user.name = 'user1' session.add(user) session.commit() param = session.query(Parameter).first() param_id = param.id param_name = param.name param_namelist = param.namelist.name model_run = ModelRun() model_run.name = "MR1" model_run.user_id = user.id model_run.status = self._status(constants.MODEL_RUN_STATUS_CREATED) session.add(model_run) session.commit() parameter1 = ParameterValue() parameter1.parameter_id = param_id parameter1.set_value_from_python("param1 value") parameter1.model_run_id = model_run.id session.add(parameter1) model_run_returned = self.model_run_service.get_model_being_created_with_non_default_parameter_values(user) param_value_returned = model_run_returned.get_python_parameter_value([param_namelist, param_name]) assert_that(param_value_returned, is_("param1 value"))
def test_GIVEN_model_run_has_parameters_WHEN_get_model_run_by_id_THEN_model_run_has_parameter_values_loaded(self): # Add a user and give them a model user = self.login() model_run = self.create_run_model(0, "MR1", user) with session_scope(Session) as session: # Add parameter value parameter_value = ParameterValue() parameter_value.parameter_id = 1 parameter_value.set_value_from_python(123) parameter = Parameter() parameter.name = "Param" parameter.parameter_values = [parameter_value] # Give them a model model_run = session.query(ModelRun).filter(ModelRun.id == model_run.id).one() model_run.parameter_values = [parameter_value] session.add(model_run) with session_scope(Session) as session: model_run_id = session.query(ModelRun).filter(ModelRun.name == model_run.name).one().id # Get the users model runs model_run_returned = self.model_run_service.get_model_by_id(user, model_run_id) pv = model_run_returned.parameter_values[0] assert_that(pv.value, is_('123')) assert_that(pv.parameter.name, is_("Param"))
def duplicate_run_model(self, model_id, user): """ Duplicate the run model and all its parameters to a new run model :param model_id: model run id to duplicate :param user: the user duplicating the model :return: nothing """ id_for_user_upload_driving_dataset = self._dataset_service.get_id_for_user_upload_driving_dataset() self.delete_model_run_being_created(user) with self.transaction_scope() as session: model_run_to_duplicate = session\ .query(ModelRun)\ .join(ModelRunStatus)\ .outerjoin(ParameterValue)\ .outerjoin(LandCoverAction) \ .filter(ModelRun.id == model_id) \ .filter(or_( ModelRun.user_id == user.id, ModelRunStatus.name == constants.MODEL_RUN_STATUS_PUBLISHED, ModelRunStatus.name == constants.MODEL_RUN_STATUS_PUBLIC,)) \ .options(contains_eager(ModelRun.parameter_values)) \ .options(contains_eager(ModelRun.land_cover_actions)) \ .one() new_model_run_name = model_run_to_duplicate.name if self._is_duplicate_name(model_run_to_duplicate.name, session, user): new_model_run_name = "{} (Copy)".format(model_run_to_duplicate.name) copy_id = 1 while self._is_duplicate_name(new_model_run_name, session, user): copy_id += 1 new_model_run_name = "{} (Copy {})".format(model_run_to_duplicate.name, copy_id) new_model_run = ModelRun() new_model_run.duplicate_from(model_run_to_duplicate) new_model_run.name = new_model_run_name new_model_run.user = user new_model_run.change_status(session, constants.MODEL_RUN_STATUS_CREATED) for parameter_value in model_run_to_duplicate.parameter_values: new_parameter = ParameterValue() new_parameter.duplicate_from(parameter_value) new_parameter.model_run = new_model_run for land_cover_action in model_run_to_duplicate.land_cover_actions: new_land_cover_action = LandCoverAction() new_land_cover_action.duplicate_from(land_cover_action) new_land_cover_action.model_run = new_model_run session.add(new_model_run) if model_run_to_duplicate.driving_dataset_id == id_for_user_upload_driving_dataset: try: self._job_runner_client.duplicate_uploaded_driving_data(model_run_to_duplicate.id, new_model_run.id) except ServiceException: self.delete_model_run_being_created(user) raise ServiceException("Could not duplicate the model run because " "the user uploaded data can not be duplicated")
def _copy_parameter_set_into_model(self, parameter_values, model_run, session): """ Copy the parameters into a model :param parameter_values: the values to copy :param model_run: the model run to copy them to :param session: the session to use :return: nothing """ for parameter_value in parameter_values: val = ParameterValue() val.value = parameter_value.value val.parameter_id = parameter_value.parameter_id val.model_run = model_run session.add(val)
def test_GIVEN_valid_output_and_model_run_WHEN_get_output_variable_name_THEN_output_variable_returned_(self): with session_scope() as session: param_val_output = ParameterValue() param_val_output.set_value_from_python('gpp_gb') parameter_service = ParameterService() param_output = parameter_service.get_parameter_by_constant(constants.JULES_PARAM_OUTPUT_VAR, session) param_output.parameter_values = [param_val_output] model_run = ModelRun() model_run.parameter_values = [param_val_output] session.add(model_run) session.commit() var = self.parameter_service.get_output_variable_name(model_run.id, param_val_output.id) assert_that(var, is_('gpp_gb'))
def _create_parameter_value_for_parameter(self, parameters, parameter_dict_name, parameter_dict_value, group_id): """ Create parameter value for a parameter :param parameters: the parameters list :param parameter_dict_name: the name of the parameter in the dictionary :param parameter_dict_value: the value in the dictionary :param group_id: group id :return: the parameter value """ for parameter in parameters: if parameter_dict_name.lower() == parameter.name.lower(): pv = ParameterValue() pv.parameter = parameter pv.group_id = group_id pv.value = parameter_dict_value return pv log.critical("Can not find matching parameter for %s" % parameter_dict_name) exit()
class TestParameterValue(BaseTest): def setUp(self): self.param_val = ParameterValue() def test_GIVEN_nothing_WHEN_set_python_datetime_THEN_value_set_to_fortran_datetime(self): value_py = datetime.datetime(1969, 7, 21, 20, 17, 0) value_ft = "'1969-07-21 20:17:00'" self.param_val.set_value_from_python(value_py) assert_that(self.param_val.value, is_(value_ft)) def test_GIVEN_value_is_fortran_datetime_WHEN_get_value_THEN_returns_python_datetime(self): value_py = datetime.datetime(1969, 7, 21, 20, 17, 0) value_ft = "'1969-07-21 20:17:00'" self.param_val.value = value_ft assert_that(self.param_val.get_value_as_python(), is_(value_py)) def test_GIVEN_nothing_WHEN_set_python_string_THEN_value_set_to_fortran_string(self): value_py = "String text" value_ft = "'String text'" self.param_val.set_value_from_python(value_py) assert_that(self.param_val.value, is_(value_ft)) def test_GIVEN_value_is_fortran_string_WHEN_get_value_THEN_returns_python_string(self): value_py = "String text" value_ft = "'String text'" self.param_val.set_value_from_python(value_py) assert_that(self.param_val.value, is_(value_ft)) def test_GIVEN_nothing_WHEN_set_python_list_THEN_value_set_to_fortran_list(self): value_py = [123.4, True, 100, datetime.datetime(1969, 7, 21, 20, 17, 0)] value_ft = "123.4 .true. 100 '1969-07-21 20:17:00'" self.param_val.set_value_from_python(value_py) assert_that(self.param_val.value, is_(value_ft)) def test_GIVEN_value_is_fortran_list_WHEN_get_value_THEN_return_python_list(self): value_py = [123.4, 3e4, 100] value_ft = "123.4 3e4 100" self.param_val.value = value_ft assert_that(self.param_val.get_value_as_python(True), is_(value_py))
def setUp(self): super(TestModelRunOutput, self).setUp() self.clean_database() self.user = self.login() self.valid_params = { 'submit': u'Next', 'ov_select_1': 1, 'ov_yearly_1': 1, 'ov_monthly_1': 1, 'ov_select_2': 1, 'ov_hourly_2': 1, 'ov_daily_2': 1, 'ov_select_3': 1, 'ov_monthly_3': 1 } self.model_run_service = ModelRunService() param_run_start = self.model_run_service.get_parameter_by_constant(JULES_PARAM_RUN_START) param_run_end = self.model_run_service.get_parameter_by_constant(JULES_PARAM_RUN_END) with session_scope(Session) as session: self.model_run = self.model_run_service._create_new_model_run(session, self.user) self.model_run.name = "MR1" pv_run_start = ParameterValue() pv_run_start.parameter_id = param_run_start.id pv_run_start.value = "'1901-01-01 00:00:00'" pv_run_end = ParameterValue() pv_run_end.parameter_id = param_run_end.id pv_run_end.value = "'1902-01-01 00:00:00'" self.model_run.parameter_values = [pv_run_start, pv_run_end] session.add(self.model_run)
def store_parameter_values(self, parameters_to_set, user): """ Store the parameter values in the database :param parameters_to_set: dictionary of parameter ids and parameter values :param user: the logged in user :return: Nothing """ with self.transaction_scope() as session: model_run = self._get_model_run_being_created(session, user) session.query(ParameterValue) \ .filter(ParameterValue.model_run == model_run) \ .delete() parameters = self._get_parameters_for_creating_model(session, user) for parameter in parameters: if parameter.id in parameters_to_set.keys(): parameter_value = parameters_to_set[parameter.id] if parameter_value is not None: val = ParameterValue() val.set_value_from_python(parameter_value) val.parameter = parameter val.model_run = model_run session.add(val)
def setUp(self): self.clean_database() self.user = self.login() self.create_two_driving_datasets() dds = DatasetService().get_driving_datasets(self.user)[0] with session_scope() as session: model_run = ModelRun() model_run.name = "model run" model_run.change_status(session, MODEL_RUN_STATUS_CREATED) model_run.user = self.user model_run.driving_dataset_id = dds.id session.add(model_run) session.commit() self.model_run_service = ModelRunService() model_run = self.model_run_service._get_model_run_being_created(session, self.user) parameter_val = ParameterValue() parameter_val.parameter = self.model_run_service.get_parameter_by_constant(JULES_PARAM_LATLON_REGION) parameter_val.set_value_from_python(True) parameter_val.model_run_id = model_run.id session.add(parameter_val) self.add_land_cover_region(model_run)
def alter_yearly_monthly_output_profiles(self, parameters): """ Adjusts the output profiles to avoid JULES errors like: "Jules error:file_ts_open: When using data_period=-1, data must start at 00:00:00 on 1st of month" caused by asking for output profiles that aren't valid for the selected run starts and ends :param parameters: List of parameters :return: """ run_start = utils.get_first_parameter_value_from_parameter_list(parameters, constants.JULES_PARAM_RUN_START) run_end = utils.get_first_parameter_value_from_parameter_list(parameters, constants.JULES_PARAM_RUN_END) output_starts_to_add = [] for parameter in parameters: if parameter.namelist.name == constants.JULES_PARAM_OUTPUT_PERIOD[0]: if parameter.name == constants.JULES_PARAM_OUTPUT_PERIOD[1]: for pv_output_period in parameter.parameter_values: period = pv_output_period.get_value_as_python() group_id = pv_output_period.group_id if period == constants.JULES_YEARLY_PERIOD: if not utils.is_first_of_year(run_start): next_year = utils.next_first_of_year(run_start) if next_year <= run_end: output_start = ParameterValue() output_start.set_value_from_python(next_year) output_start.group_id = group_id output_starts_to_add.append(output_start) elif period == constants.JULES_MONTHLY_PERIOD: if not utils.is_first_of_month(run_start): next_month = utils.next_first_of_month(run_start) if next_month <= run_end: output_start = ParameterValue() output_start.set_value_from_python(next_month) output_start.group_id = group_id output_starts_to_add.append(output_start) # Add parameter values (output_start) for parameter in parameters: if parameter.namelist.name == constants.JULES_PARAM_OUTPUT_START[0]: if parameter.name == constants.JULES_PARAM_OUTPUT_START[1]: parameter.parameter_values += output_starts_to_add return parameters
def _create_uploaded_driving_dataset(self, start_date, end_date, lat, lon, model_run_service): class UploadedDrivingDataset(object): """ An uploaded driving dataset """ def __init__(self): self.id = DatasetService().get_id_for_user_upload_driving_dataset() self.driving_data_lat = None self.driving_data_lon = None self.driving_data_rows = None self.parameter_values = [] driving_dataset = UploadedDrivingDataset() driving_dataset.driving_data_lat = lat driving_dataset.driving_data_lon = lon driving_dataset.driving_data_rows = self.n_lines driving_dataset.parameter_values = [] params_to_save = [[constants.JULES_PARAM_DRIVE_DATA_START, start_date], [constants.JULES_PARAM_DRIVE_DATA_END, end_date], [constants.JULES_PARAM_DRIVE_DATA_PERIOD, self.period], [constants.JULES_PARAM_DRIVE_FILE, constants.USER_UPLOAD_FILE_NAME], [constants.JULES_PARAM_DRIVE_NVARS, len(self.var_list or [])], [constants.JULES_PARAM_DRIVE_VAR, self.var_list], [constants.JULES_PARAM_DRIVE_INTERP, self.interp_list], [constants.JULES_PARAM_LATLON_LATITUDE, lat], [constants.JULES_PARAM_LATLON_LONGITUDE, lon], [constants.JULES_PARAM_FRAC_FILE, constants.FRACTIONAL_FILENAME]] for param in params_to_save: parameter = model_run_service.get_parameter_by_constant(param[0]) param_val = ParameterValue() param_val.set_value_from_python(param[1]) param_val.parameter_id = parameter.id driving_dataset.parameter_values.append(param_val) user_upload_id = self.dataset_service.get_id_for_user_upload_driving_dataset() ds = self.dataset_service.get_driving_dataset_by_id(user_upload_id) for pv in ds.parameter_values: parameter_value = ParameterValue() parameter_value.parameter_id = pv.parameter_id parameter_value.value = pv.value driving_dataset.parameter_values.append(parameter_value) return driving_dataset
def setUp(self): super(TestModelRunExtents, self).setUp() self.clean_database() self.user = self.login() self.model_run_service = ModelRunService() with session_scope(Session) as session: self.driving_data = DrivingDataset() self.driving_data.name = "d1" self.driving_data.boundary_lat_north = 47.5 self.driving_data.boundary_lat_south = 13.8 self.driving_data.boundary_lon_east = 123.1 self.driving_data.boundary_lon_west = -15.0 self.driving_data.time_start = datetime.datetime(1901, 1, 1, 0, 0, 0) self.driving_data.time_end = datetime.datetime(2001, 1, 1, 0, 0, 0) session.add(self.driving_data) session.commit() self.model_run = ModelRun() self.model_run.name = "MR1" self.model_run.status = self._status(MODEL_RUN_STATUS_CREATED) self.model_run.driving_dataset_id = self.driving_data.id self.model_run.user = self.user self.model_run.science_configuration_id = 2 param1 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_INTERP) pv1 = ParameterValue() pv1.parameter_id = param1.id pv1.set_value_from_python(8 * ['nf']) param2 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_DATA_PERIOD) pv2 = ParameterValue() pv2.parameter_id = param2.id pv2.set_value_from_python(60 * 60) self.model_run.parameter_values = [pv1, pv2] session.add(self.model_run)
def _create_new_model_run(self, session, user): """ Create a brand new model run for user :param session: session to use :param user: Currently logged in user :return: New model run """ model_run = ModelRun() model_run.change_status(session, constants.MODEL_RUN_STATUS_CREATED) model_run.user = user parameters = [ [constants.JULES_PARAM_TIMESTEP_LEN, constants.TIMESTEP_LEN, None], [constants.JULES_PARAM_OUTPUT_RUN_ID, constants.RUN_ID, None], [constants.JULES_PARAM_OUTPUT_OUTPUT_DIR, "./" + constants.OUTPUT_DIR, None], ] # Add CHESS defaults: chess_defaults = ['surf_roff', 'sub_surf_roff', 'fqw_gb', 'rad_net', 'ftl_gb', 'gpp_gb', 'resp_p_gb', 'tstar_gb', 'snow_mass_gb', 't_soil', 'smc_tot', 'smcl', 'swet_liq_tot'] chess_periods = [(constants.JULES_MONTHLY_PERIOD, "_monthly")] group_id = 0 for output_variable in chess_defaults: for period, period_profile_name in chess_periods: parameters.append([constants.JULES_PARAM_OUTPUT_VAR, output_variable, group_id]) parameters.append([constants.JULES_PARAM_OUTPUT_PERIOD, period, group_id]) parameters.append([constants.JULES_PARAM_OUTPUT_PROFILE_NAME, output_variable + period_profile_name, group_id]) parameters.append([constants.JULES_PARAM_OUTPUT_NVARS, 1, group_id]) parameters.append([constants.JULES_PARAM_OUTPUT_MAIN_RUN, True, group_id]) parameters.append([constants.JULES_PARAM_OUTPUT_TYPE, 'M', group_id]) group_id += 1 parameters.append([constants.JULES_PARAM_OUTPUT_NPROFILES, group_id, None]) for constant, value, group_id in parameters: param = self.parameter_service.get_parameter_by_constant(constant, session) param_value = ParameterValue() param_value.parameter = param param_value.model_run = model_run param_value.group_id = group_id param_value.set_value_from_python(value) return model_run
def save_parameter(self, model_run, param_namelist_name, value, session, group_id=None): """ Save parameter using a supplied session :param model_run: Model run to save against :param param_namelist_name: List containing the parameter namelist, name :param session: Session to use :param value: Value to set :param group_id: Specify an optional group_id to group parameters :return: """ parameter = self.get_parameter_by_constant(param_namelist_name, session) try: parameter_value = session.query(ParameterValue) \ .filter(ParameterValue.model_run_id == model_run.id) \ .filter(ParameterValue.group_id == group_id) \ .filter(ParameterValue.parameter_id == parameter.id).one() except NoResultFound: parameter_value = ParameterValue() parameter_value.parameter = parameter parameter_value.model_run = model_run parameter_value.group_id = group_id parameter_value.set_value_from_python(value) session.add(parameter_value)
def setUp(self): self.param_val = ParameterValue()
def set_up_single_cell_model_run(self): self.clean_database() self.user = self.login() self.create_two_driving_datasets() user_upload_id = DatasetService().get_id_for_user_upload_driving_dataset() with session_scope(Session) as session: self.model_run = ModelRun() self.model_run.name = "MR1" self.model_run.status = self._status(MODEL_RUN_STATUS_CREATED) self.model_run.driving_dataset_id = user_upload_id self.model_run.user = self.user self.model_run.driving_data_lat = 51.75 self.model_run.driving_data_lon = -0.25 self.model_run.driving_data_rows = 248 param1 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_INTERP) pv1 = ParameterValue() pv1.parameter_id = param1.id pv1.set_value_from_python(8 * ['nf']) param2 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_DATA_PERIOD) pv2 = ParameterValue() pv2.parameter_id = param2.id pv2.set_value_from_python(60 * 60) param3 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_DATA_START) pv3 = ParameterValue() pv3.parameter_id = param3.id pv3.value = "'1901-01-01 00:00:00'" param4 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_DATA_END) pv4 = ParameterValue() pv4.parameter_id = param4.id pv4.value = "'1901-01-31 21:00:00'" param5 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_LATLON_REGION) pv5 = ParameterValue() pv5.parameter_id = param5.id pv5.value = ".false." param6 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_POINTS_FILE) pv6 = ParameterValue() pv6.parameter_id = param6.id pv6.value = "51.75 -0.25" self.model_run.parameter_values = [pv1, pv2, pv3, pv4, pv5, pv6] session.add(self.model_run)
def set_up_single_cell_user_driving_data(self): self.clean_database() del self.driving_data self.user = self.login() user_upload_id = DatasetService().get_id_for_user_upload_driving_dataset() with session_scope(Session) as session: self.model_run = ModelRun() self.model_run.name = "MR1" self.model_run.status = self._status(MODEL_RUN_STATUS_CREATED) self.model_run.driving_dataset_id = user_upload_id self.model_run.user = self.user self.model_run.driving_data_lat = 25 self.model_run.driving_data_lon = 40 self.model_run.driving_data_rows = 248 self.model_run.science_configuration_id = 2 param1 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_INTERP) pv1 = ParameterValue() pv1.parameter_id = param1.id pv1.set_value_from_python(8 * ['nf']) param2 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_DATA_PERIOD) pv2 = ParameterValue() pv2.parameter_id = param2.id pv2.set_value_from_python(60 * 60) param3 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_DATA_START) pv3 = ParameterValue() pv3.parameter_id = param3.id pv3.value = "'1901-01-01 00:00:00'" param4 = self.model_run_service.get_parameter_by_constant(JULES_PARAM_DRIVE_DATA_END) pv4 = ParameterValue() pv4.parameter_id = param4.id pv4.value = "'1901-01-31 21:00:00'" self.model_run.parameter_values = [pv1, pv2, pv3, pv4] session.add(self.model_run)