def parse_namelist_files_to_create_a_model_run(self, code_version, description, config_dir, name, namelist_files, status, user, spinup_in_years): """ Ceate a model run :param code_version: the code version object :param description: the description fo the model run :param config_dir: the configuration directory :param name: name of the model run :param namelist_files: namelist_files object :param status: status the model is to have :param user: the user creating the model run :param spinup_in_years: the spin up in years, for a science configuration :return: a model run """ model_run = ModelRun() model_run.name = name model_run.user = user model_run.code_version = code_version model_run.description = description model_run.status = status model_run.science_configuration_spinup_in_years = spinup_in_years model_run.parameter_values = [] log.info("Creating Model Run For: {0} ({1})".format(model_run.name, config_dir)) for namelist_file in namelist_files: filename = os.path.join(config_dir, namelist_file.filename) if os.path.exists(filename): log.info("Parsing: %s" % filename) nml_dict = self._read_namelist(filename) model_run.parameter_values.extend(self._create_parameter_values(namelist_file, nml_dict)) return model_run
def duplicate_run_model(self, model_id, user): """ Duplicate the run model and all its parameters to a new run model :param model_id: model run id to duplicate :param user: the user duplicating the model :return: nothing """ id_for_user_upload_driving_dataset = self._dataset_service.get_id_for_user_upload_driving_dataset() self.delete_model_run_being_created(user) with self.transaction_scope() as session: model_run_to_duplicate = session\ .query(ModelRun)\ .join(ModelRunStatus)\ .outerjoin(ParameterValue)\ .outerjoin(LandCoverAction) \ .filter(ModelRun.id == model_id) \ .filter(or_( ModelRun.user_id == user.id, ModelRunStatus.name == constants.MODEL_RUN_STATUS_PUBLISHED, ModelRunStatus.name == constants.MODEL_RUN_STATUS_PUBLIC,)) \ .options(contains_eager(ModelRun.parameter_values)) \ .options(contains_eager(ModelRun.land_cover_actions)) \ .one() new_model_run_name = model_run_to_duplicate.name if self._is_duplicate_name(model_run_to_duplicate.name, session, user): new_model_run_name = "{} (Copy)".format(model_run_to_duplicate.name) copy_id = 1 while self._is_duplicate_name(new_model_run_name, session, user): copy_id += 1 new_model_run_name = "{} (Copy {})".format(model_run_to_duplicate.name, copy_id) new_model_run = ModelRun() new_model_run.duplicate_from(model_run_to_duplicate) new_model_run.name = new_model_run_name new_model_run.user = user new_model_run.change_status(session, constants.MODEL_RUN_STATUS_CREATED) for parameter_value in model_run_to_duplicate.parameter_values: new_parameter = ParameterValue() new_parameter.duplicate_from(parameter_value) new_parameter.model_run = new_model_run for land_cover_action in model_run_to_duplicate.land_cover_actions: new_land_cover_action = LandCoverAction() new_land_cover_action.duplicate_from(land_cover_action) new_land_cover_action.model_run = new_model_run session.add(new_model_run) if model_run_to_duplicate.driving_dataset_id == id_for_user_upload_driving_dataset: try: self._job_runner_client.duplicate_uploaded_driving_data(model_run_to_duplicate.id, new_model_run.id) except ServiceException: self.delete_model_run_being_created(user) raise ServiceException("Could not duplicate the model run because " "the user uploaded data can not be duplicated")
def test_GIVEN_valid_model_WHEN_convert_to_dictionary_THEN_namelist_with_no_representation_are_present(self): job_runner_client = JobRunnerClient(config) def _alter_yearly_monthly_output_profiles(params): return params def _alter_driving_data_start(params): return params job_runner_client.alter_yearly_monthly_output_profiles = _alter_yearly_monthly_output_profiles job_runner_client.alter_driving_data_start = _alter_driving_data_start parameter = Parameter(name='param1') expected_parameter_value = '12' #There is no parameter value: parameter.parameter_values = [] namelist = Namelist(name='NAME_LIST') namelist_file = NamelistFile(filename='filename') # Need this to run parameter2 = Parameter(name=constants.JULES_PARAM_LATLON_REGION[1]) parameter2.parameter_values = [ParameterValue(value='.true.')] namelist2 = Namelist(name=constants.JULES_PARAM_LATLON_REGION[0]) namelist_file = NamelistFile(filename='filename') namelist2.parameters = [parameter2] namelist2.namelist_file = namelist_file namelist.parameters = [parameter] namelist.namelist_file = namelist_file model_run = ModelRun() model_run.id = 101 code_version = CodeVersion(name='Jules v3.4.1') user = User() user.id = 1 user.email = "email" user.username = "******" model_run.user = user model_run.code_version = code_version code_version.parameters = [parameter, parameter2] result = job_runner_client.convert_model_to_dictionary(model_run, code_version.parameters, []) namelist_file_result = result[constants.JSON_MODEL_NAMELIST_FILES][0] assert_that(namelist_file_result[constants.JSON_MODEL_NAMELIST_FILE_FILENAME], is_(namelist_file.filename), "value is correct") assert_that(namelist_file_result[constants.JSON_MODEL_NAMELISTS][0][constants.JSON_MODEL_NAMELIST_NAME], is_(namelist.name), "value is correct") parameters_result = namelist_file_result[constants.JSON_MODEL_NAMELISTS][0][constants.JSON_MODEL_PARAMETERS] assert_that(parameters_result, is_({}), "there are no values") assert_that(len(result[constants.JSON_LAND_COVER]), is_(0))
def test_GIVEN_valid_model_WHEN_convert_to_dictionary_THEN_dictionary_is_correct(self): job_runner_client = JobRunnerClient(config) def _alter_yearly_monthly_output_profiles(params): return params def _alter_driving_data_start(params): return params job_runner_client.alter_yearly_monthly_output_profiles = _alter_yearly_monthly_output_profiles job_runner_client.alter_driving_data_start = _alter_driving_data_start parameter = Parameter(name=constants.JULES_PARAM_POINTS_FILE[1]) expected_parameter_value = '51 0.5' expected_index = 3 #There is only one parameter value per run: parameter.parameter_values = [ParameterValue(value=expected_parameter_value)] namelist = Namelist(name=constants.JULES_PARAM_POINTS_FILE[0]) namelist_file = NamelistFile(filename='filename') namelist.parameters = [parameter] namelist.namelist_file = namelist_file namelist.index_in_file = expected_index model_run = ModelRun() model_run.id = 101 model_run.land_cover_frac = "1 2 3 4 5 6 7 8 9" code_version = CodeVersion(name='Jules v3.4.1') model_run.code_version = code_version code_version.parameters = [parameter] user = User() user.id = 1 user.email = "email" user.username = "******" model_run.user = user result = job_runner_client.convert_model_to_dictionary(model_run, code_version.parameters, []) assert_that(result[constants.JSON_MODEL_RUN_ID], is_(model_run.id), "value is correct") assert_that(result[constants.JSON_MODEL_CODE_VERSION], is_(model_run.code_version.name), "value is correct") assert_that(result[constants.JSON_USER_ID], is_(user.id), "user id") assert_that(result[constants.JSON_USER_NAME], is_(user.username), "user name") assert_that(result[constants.JSON_USER_EMAIL], is_(user.email), "user email") namelist_file_result = result[constants.JSON_MODEL_NAMELIST_FILES][0] assert_that(namelist_file_result[constants.JSON_MODEL_NAMELIST_FILE_FILENAME], is_(namelist_file.filename), "value is correct") assert_that(namelist_file_result[constants.JSON_MODEL_NAMELISTS][0][constants.JSON_MODEL_NAMELIST_NAME], is_(namelist.name), "namelist name") assert_that(namelist_file_result[constants.JSON_MODEL_NAMELISTS][0][constants.JSON_MODEL_NAMELIST_INDEX], is_(namelist.index_in_file), "namelist index") parameters_result = namelist_file_result[constants.JSON_MODEL_NAMELISTS][0][constants.JSON_MODEL_PARAMETERS] assert_that(parameters_result, is_({parameter.name: expected_parameter_value}), "value is correct")
def _create_new_model_run(self, session, user): """ Create a brand new model run for user :param session: session to use :param user: Currently logged in user :return: New model run """ model_run = ModelRun() model_run.change_status(session, constants.MODEL_RUN_STATUS_CREATED) model_run.user = user parameters = [ [constants.JULES_PARAM_TIMESTEP_LEN, constants.TIMESTEP_LEN, None], [constants.JULES_PARAM_OUTPUT_RUN_ID, constants.RUN_ID, None], [constants.JULES_PARAM_OUTPUT_OUTPUT_DIR, "./" + constants.OUTPUT_DIR, None], ] # Add CHESS defaults: chess_defaults = ['surf_roff', 'sub_surf_roff', 'fqw_gb', 'rad_net', 'ftl_gb', 'gpp_gb', 'resp_p_gb', 'tstar_gb', 'snow_mass_gb', 't_soil', 'smc_tot', 'smcl', 'swet_liq_tot'] chess_periods = [(constants.JULES_MONTHLY_PERIOD, "_monthly")] group_id = 0 for output_variable in chess_defaults: for period, period_profile_name in chess_periods: parameters.append([constants.JULES_PARAM_OUTPUT_VAR, output_variable, group_id]) parameters.append([constants.JULES_PARAM_OUTPUT_PERIOD, period, group_id]) parameters.append([constants.JULES_PARAM_OUTPUT_PROFILE_NAME, output_variable + period_profile_name, group_id]) parameters.append([constants.JULES_PARAM_OUTPUT_NVARS, 1, group_id]) parameters.append([constants.JULES_PARAM_OUTPUT_MAIN_RUN, True, group_id]) parameters.append([constants.JULES_PARAM_OUTPUT_TYPE, 'M', group_id]) group_id += 1 parameters.append([constants.JULES_PARAM_OUTPUT_NPROFILES, group_id, None]) for constant, value, group_id in parameters: param = self.parameter_service.get_parameter_by_constant(constant, session) param_value = ParameterValue() param_value.parameter = param param_value.model_run = model_run param_value.group_id = group_id param_value.set_value_from_python(value) return model_run
def setUp(self): self.clean_database() self.user = self.login() self.create_two_driving_datasets() dds = DatasetService().get_driving_datasets(self.user)[0] with session_scope() as session: model_run = ModelRun() model_run.name = "model run" model_run.change_status(session, MODEL_RUN_STATUS_CREATED) model_run.user = self.user model_run.driving_dataset_id = dds.id session.add(model_run) session.commit() self.model_run_service = ModelRunService() model_run = self.model_run_service._get_model_run_being_created(session, self.user) parameter_val = ParameterValue() parameter_val.parameter = self.model_run_service.get_parameter_by_constant(JULES_PARAM_LATLON_REGION) parameter_val.set_value_from_python(True) parameter_val.model_run_id = model_run.id session.add(parameter_val) self.add_land_cover_region(model_run)
def test_GIVEN_model_contains_repeated_namelists_WHEN_convert_to_dictionary_THEN_namelists_grouped_correctly(self): job_runner_client = JobRunnerClient(config) def _alter_yearly_monthly_output_profiles(params): return params def _alter_driving_data_start(params): return params job_runner_client.alter_yearly_monthly_output_profiles = _alter_yearly_monthly_output_profiles job_runner_client.alter_driving_data_start = _alter_driving_data_start # Set the namelists here: nml_output = Namelist(name=constants.JULES_NML_OUTPUT) nml_output_profile = Namelist(name=constants.JULES_NML_OUTPUT_PROFILE) nml_output.namelist_file = nml_output_profile.namelist_file = NamelistFile(filename="output.nml") nml_model_grid = Namelist(name=constants.JULES_PARAM_LATLON_REGION[0]) nml_model_grid.namelist_file = NamelistFile(filename="model_grid.nml") # Set the parameters here: param_nprofiles = Parameter(name=constants.JULES_PARAM_OUTPUT_NPROFILES[1]) param_profile_name = Parameter(name=constants.JULES_PARAM_OUTPUT_PROFILE_NAME[1]) param_var = Parameter(name=constants.JULES_PARAM_OUTPUT_VAR[1]) param_latlon_region = Parameter(name=constants.JULES_PARAM_LATLON_REGION[1]) # Set the parameter values param_nprofiles.parameter_values = [ParameterValue(value=2)] param_profile_name.parameter_values = [ParameterValue(value="monthly", group_id=0), ParameterValue(value="yearly", group_id=1)] param_var.parameter_values = [ParameterValue(value='"emis_gb", "ftl_gb"', group_id=0), ParameterValue(value='"snow_mass_gb", "tstar_gb"', group_id=1)] param_latlon_region.parameter_values = [ParameterValue(value=".true.")] # Assign the parameters to namelists nml_output.parameters = [param_nprofiles] nml_output_profile.parameters = [param_profile_name, param_var] nml_model_grid.parameters = [param_latlon_region] model_run = ModelRun() model_run.id = 101 code_version = CodeVersion(name='Jules v3.4.1') user = User() user.id = 1 user.email = "email" user.username = "******" model_run.user = user model_run.code_version = code_version code_version.parameters = [param_profile_name, param_var, param_nprofiles, param_latlon_region] result = job_runner_client.convert_model_to_dictionary(model_run, code_version.parameters, []) # Check the result dictionary has the correct run ID and code version assert_that(result[constants.JSON_MODEL_RUN_ID], is_(model_run.id), "value is correct") assert_that(result[constants.JSON_MODEL_CODE_VERSION], is_(model_run.code_version.name), "value is correct") # Check that the namelist_files has the correct namelist file namelist_file_result = result[constants.JSON_MODEL_NAMELIST_FILES][0] assert_that(namelist_file_result[constants.JSON_MODEL_NAMELIST_FILE_FILENAME], is_("output.nml"), "value is correct") # Check that the namelist file contains the three namelists namelists = namelist_file_result[constants.JSON_MODEL_NAMELISTS] assert_that(len(namelists), is_(3)) # Check the dictionaries are as expected profiles_dict1 = {constants.JSON_MODEL_NAMELIST_GROUP_ID: 0, constants.JSON_MODEL_NAMELIST_INDEX: None, constants.JSON_MODEL_NAMELIST_NAME: constants.JULES_NML_OUTPUT_PROFILE, constants.JSON_MODEL_PARAMETERS: { constants.JULES_PARAM_OUTPUT_VAR[1]: '"emis_gb", "ftl_gb"', constants.JULES_PARAM_OUTPUT_PROFILE_NAME[1]: "monthly" }} profiles_dict2 = {constants.JSON_MODEL_NAMELIST_GROUP_ID: 1, constants.JSON_MODEL_NAMELIST_INDEX: None, constants.JSON_MODEL_NAMELIST_NAME: constants.JULES_NML_OUTPUT_PROFILE, constants.JSON_MODEL_PARAMETERS: { constants.JULES_PARAM_OUTPUT_VAR[1]: '"snow_mass_gb", "tstar_gb"', constants.JULES_PARAM_OUTPUT_PROFILE_NAME[1]: "yearly" }} output_dict = {constants.JSON_MODEL_NAMELIST_GROUP_ID: None, constants.JSON_MODEL_NAMELIST_INDEX: None, constants.JSON_MODEL_NAMELIST_NAME: constants.JULES_NML_OUTPUT, constants.JSON_MODEL_PARAMETERS: { constants.JULES_PARAM_OUTPUT_NPROFILES[1]: 2 }} assert_that(namelists, contains_inanyorder(profiles_dict1, profiles_dict2, output_dict))
def test_GIVEN_model_with_land_cover_actions_WHEN_convert_to_dictionary_THEN_land_cover_actions_present(self): job_runner_client = JobRunnerClient(config) def _alter_yearly_monthly_output_profiles(params): return params def _alter_driving_data_start(params): return params job_runner_client.alter_yearly_monthly_output_profiles = _alter_yearly_monthly_output_profiles job_runner_client.alter_driving_data_start = _alter_driving_data_start parameter = Parameter(name='file') expected_parameter_value = "'base_frac_file.nc'" parameter.parameter_values = [ParameterValue(value=expected_parameter_value)] parameter2 = Parameter(name='frac_name') expected_parameter_value2 = "'frac'" parameter2.parameter_values = [ParameterValue(value=expected_parameter_value2)] parameter3 = Parameter(name='latlon_region') expected_parameter_value3 = ".true." parameter3.parameter_values = [ParameterValue(value=expected_parameter_value3)] namelist = Namelist(name='JULES_FRAC') namelist_file = NamelistFile(filename='filename') expected_index = 3 namelist.parameters = [parameter, parameter2] namelist.namelist_file = namelist_file namelist.index_in_file = expected_index namelist2 = Namelist(name='JULES_MODEL_GRID') namelist_file = NamelistFile(filename='filename') expected_index = 3 namelist2.parameters = [parameter3] namelist2.namelist_file = namelist_file namelist2.index_in_file = expected_index model_run = ModelRun() model_run.id = 101 code_version = CodeVersion(name='Jules v3.4.1') model_run.code_version = code_version code_version.parameters = [parameter, parameter2, parameter3] user = User() user.id = 1 user.email = "email" user.username = "******" model_run.user = user lcr1 = LandCoverRegion() lcr1.mask_file = "region1.nc" lca1 = LandCoverAction() lca1.value_id = 9 lca1.order = 1 lca1.region = lcr1 lcr2 = LandCoverRegion() lcr2.mask_file = "region2.nc" lca2 = LandCoverAction() lca2.value_id = 5 lca2.order = 2 lca2.region = lcr2 land_cover_actions = [lca1, lca2] result = job_runner_client.convert_model_to_dictionary(model_run, code_version.parameters, land_cover_actions) result_lc = result[constants.JSON_LAND_COVER] assert_that(result_lc[constants.JSON_LAND_COVER_BASE_FILE], is_("base_frac_file.nc")) assert_that(result_lc[constants.JSON_LAND_COVER_BASE_KEY], is_("frac")) assert_that(result_lc[constants.JSON_LAND_COVER_ICE_INDEX], is_(9)) result_lc_actions = result_lc[constants.JSON_LAND_COVER_ACTIONS] assert_that(len(result_lc_actions), is_(2)) action1 = result_lc_actions[0] assert_that(action1[constants.JSON_LAND_COVER_MASK_FILE], is_("region1.nc")) assert_that(action1[constants.JSON_LAND_COVER_ORDER], is_(1)) assert_that(action1[constants.JSON_LAND_COVER_VALUE], is_(9)) action2 = result_lc_actions[1] assert_that(action2[constants.JSON_LAND_COVER_MASK_FILE], is_("region2.nc")) assert_that(action2[constants.JSON_LAND_COVER_ORDER], is_(2)) assert_that(action2[constants.JSON_LAND_COVER_VALUE], is_(5)) namelist_file_result = result[constants.JSON_MODEL_NAMELIST_FILES][0] parameters_result = namelist_file_result[constants.JSON_MODEL_NAMELISTS][0][constants.JSON_MODEL_PARAMETERS] assert_that(parameters_result['file'], is_("'" + constants.USER_EDITED_FRACTIONAL_FILENAME + "'")) assert_that(parameters_result['frac_name'], is_("'frac'"))