def test_GIVEN_valid_model_WHEN_convert_to_dictionary_THEN_dictionary_is_correct(self): job_runner_client = JobRunnerClient(config) def _alter_yearly_monthly_output_profiles(params): return params def _alter_driving_data_start(params): return params job_runner_client.alter_yearly_monthly_output_profiles = _alter_yearly_monthly_output_profiles job_runner_client.alter_driving_data_start = _alter_driving_data_start parameter = Parameter(name=constants.JULES_PARAM_POINTS_FILE[1]) expected_parameter_value = '51 0.5' expected_index = 3 #There is only one parameter value per run: parameter.parameter_values = [ParameterValue(value=expected_parameter_value)] namelist = Namelist(name=constants.JULES_PARAM_POINTS_FILE[0]) namelist_file = NamelistFile(filename='filename') namelist.parameters = [parameter] namelist.namelist_file = namelist_file namelist.index_in_file = expected_index model_run = ModelRun() model_run.id = 101 model_run.land_cover_frac = "1 2 3 4 5 6 7 8 9" code_version = CodeVersion(name='Jules v3.4.1') model_run.code_version = code_version code_version.parameters = [parameter] user = User() user.id = 1 user.email = "email" user.username = "******" model_run.user = user result = job_runner_client.convert_model_to_dictionary(model_run, code_version.parameters, []) assert_that(result[constants.JSON_MODEL_RUN_ID], is_(model_run.id), "value is correct") assert_that(result[constants.JSON_MODEL_CODE_VERSION], is_(model_run.code_version.name), "value is correct") assert_that(result[constants.JSON_USER_ID], is_(user.id), "user id") assert_that(result[constants.JSON_USER_NAME], is_(user.username), "user name") assert_that(result[constants.JSON_USER_EMAIL], is_(user.email), "user email") namelist_file_result = result[constants.JSON_MODEL_NAMELIST_FILES][0] assert_that(namelist_file_result[constants.JSON_MODEL_NAMELIST_FILE_FILENAME], is_(namelist_file.filename), "value is correct") assert_that(namelist_file_result[constants.JSON_MODEL_NAMELISTS][0][constants.JSON_MODEL_NAMELIST_NAME], is_(namelist.name), "namelist name") assert_that(namelist_file_result[constants.JSON_MODEL_NAMELISTS][0][constants.JSON_MODEL_NAMELIST_INDEX], is_(namelist.index_in_file), "namelist index") parameters_result = namelist_file_result[constants.JSON_MODEL_NAMELISTS][0][constants.JSON_MODEL_PARAMETERS] assert_that(parameters_result, is_({parameter.name: expected_parameter_value}), "value is correct")
def test_GIVEN_model_with_land_cover_actions_WHEN_convert_to_dictionary_THEN_land_cover_actions_present(self): job_runner_client = JobRunnerClient(config) def _alter_yearly_monthly_output_profiles(params): return params def _alter_driving_data_start(params): return params job_runner_client.alter_yearly_monthly_output_profiles = _alter_yearly_monthly_output_profiles job_runner_client.alter_driving_data_start = _alter_driving_data_start parameter = Parameter(name='file') expected_parameter_value = "'base_frac_file.nc'" parameter.parameter_values = [ParameterValue(value=expected_parameter_value)] parameter2 = Parameter(name='frac_name') expected_parameter_value2 = "'frac'" parameter2.parameter_values = [ParameterValue(value=expected_parameter_value2)] parameter3 = Parameter(name='latlon_region') expected_parameter_value3 = ".true." parameter3.parameter_values = [ParameterValue(value=expected_parameter_value3)] namelist = Namelist(name='JULES_FRAC') namelist_file = NamelistFile(filename='filename') expected_index = 3 namelist.parameters = [parameter, parameter2] namelist.namelist_file = namelist_file namelist.index_in_file = expected_index namelist2 = Namelist(name='JULES_MODEL_GRID') namelist_file = NamelistFile(filename='filename') expected_index = 3 namelist2.parameters = [parameter3] namelist2.namelist_file = namelist_file namelist2.index_in_file = expected_index model_run = ModelRun() model_run.id = 101 code_version = CodeVersion(name='Jules v3.4.1') model_run.code_version = code_version code_version.parameters = [parameter, parameter2, parameter3] user = User() user.id = 1 user.email = "email" user.username = "******" model_run.user = user lcr1 = LandCoverRegion() lcr1.mask_file = "region1.nc" lca1 = LandCoverAction() lca1.value_id = 9 lca1.order = 1 lca1.region = lcr1 lcr2 = LandCoverRegion() lcr2.mask_file = "region2.nc" lca2 = LandCoverAction() lca2.value_id = 5 lca2.order = 2 lca2.region = lcr2 land_cover_actions = [lca1, lca2] result = job_runner_client.convert_model_to_dictionary(model_run, code_version.parameters, land_cover_actions) result_lc = result[constants.JSON_LAND_COVER] assert_that(result_lc[constants.JSON_LAND_COVER_BASE_FILE], is_("base_frac_file.nc")) assert_that(result_lc[constants.JSON_LAND_COVER_BASE_KEY], is_("frac")) assert_that(result_lc[constants.JSON_LAND_COVER_ICE_INDEX], is_(9)) result_lc_actions = result_lc[constants.JSON_LAND_COVER_ACTIONS] assert_that(len(result_lc_actions), is_(2)) action1 = result_lc_actions[0] assert_that(action1[constants.JSON_LAND_COVER_MASK_FILE], is_("region1.nc")) assert_that(action1[constants.JSON_LAND_COVER_ORDER], is_(1)) assert_that(action1[constants.JSON_LAND_COVER_VALUE], is_(9)) action2 = result_lc_actions[1] assert_that(action2[constants.JSON_LAND_COVER_MASK_FILE], is_("region2.nc")) assert_that(action2[constants.JSON_LAND_COVER_ORDER], is_(2)) assert_that(action2[constants.JSON_LAND_COVER_VALUE], is_(5)) namelist_file_result = result[constants.JSON_MODEL_NAMELIST_FILES][0] parameters_result = namelist_file_result[constants.JSON_MODEL_NAMELISTS][0][constants.JSON_MODEL_PARAMETERS] assert_that(parameters_result['file'], is_("'" + constants.USER_EDITED_FRACTIONAL_FILENAME + "'")) assert_that(parameters_result['frac_name'], is_("'frac'"))
def run(self, namelist_documentation_file, page_url, code_version): """ Run the parser on the file and return the name list file object :param namelist_documentation_file: filename for the name list file documentation :param page_url: page url :param code_version: the CodeVersion to use for the code version :return: the NamelistFile object """ log.info("Parsing %s" % namelist_documentation_file) tree = html.parse(namelist_documentation_file) namelist_file = NamelistFile() namelist_file.filename = str(tree.xpath('//h1/tt/span/text()')[0]) log.info("Filename: %s" % namelist_file.filename) namelist_file.namelists = [] namelist_index = 0 for namelist_name in tree.xpath('//h2/tt/span/text()'): namelist = Namelist() namelist.name = str(namelist_name) log.info(" Namelist: %s" % namelist.name) namelist_file.namelists.append(namelist) namelist.index_in_file = namelist_index namelist_index += 1 #parameters at the bottom level parameters = tree.xpath('//div[@id="namelist-{namelist_name}"]/dl'.format(namelist_name=namelist_name)) if namelist.name.upper() in self._EXTRA_PARAMETERS: log.info("Adding extra parameters for %s" % namelist.name) namelist.parameters.extend(self._EXTRA_PARAMETERS[namelist.name]) for parameter in namelist.parameters: parameter.code_versions = [code_version] #parameters in optional sections parameters.extend( tree.xpath('//div[@id="namelist-{namelist_name}"]/div/dl'.format(namelist_name=namelist_name))) parameters.extend( tree.xpath('//div[@id="namelist-{namelist_name}"]/div/div/dl[@class="member"]' .format(namelist_name=namelist_name))) parameters.extend( tree.xpath('//div[@id="namelist-{namelist_name}"]/div/div/dl[@class="last member"]' .format(namelist_name=namelist_name))) for parameter_elements in parameters: parameter_name = str(parameter_elements.xpath("dt")[0].attrib["id"].replace(namelist_name + '::', '')) log.info(" Parameter: %s" % parameter_name) parameter = Parameter() parameter.name = parameter_name parameter.default_value = None parameter.code_versions = [code_version] parameter.url_suffix = page_url + parameter_elements.xpath("dt/a")[0].attrib["href"] log.info(" Web page suffix: %s" % parameter.url_suffix) possible_description = parameter_elements.xpath("dd/p") parameter.description = etree.tostring(possible_description[0]) if len(possible_description) > 0 else "" log.info(" Description: %s" % parameter.description.strip()) for parameter_property in parameter_elements.xpath("dd/table/tbody/tr"): names = parameter_property.xpath("th/text()") if len(names) == 1: td = parameter_property.xpath("td")[0] value = "".join([x for x in td.itertext()]) if names[0] == 'Type :': parameter.type = value log.info(" Type: %s" % parameter.type) elif names[0] == 'Default :': self._set_parameter_default(parameter, value) log.info(" Default: %s" % parameter.default_value) elif names[0] == 'Permitted :': self._get_parameter_limits(parameter, value) log.info(" min: %s" % parameter.min) log.info(" max: %s" % parameter.max) parameter.required = (parameter.default_value is None) namelist.parameters.append(parameter) return namelist_file