def parse_all(self, root_path, code_version): """ Parse all know name list files and return the namelist file objects :param root_path: the root path for the namelist files :param code_version: the code version object :return: namelist file objects """ namelist_files = [] post_processing_parameter = Parameter(name="id", type="integer", code_versions=[code_version]) post_processing_namelist = Namelist(name='POST_PROCESSING', index_in_file=1) post_processing_namelist.parameters = [post_processing_parameter] post_processing_namelist_file = NamelistFile(filename="post_processing.nml") post_processing_namelist_file.namelists = [post_processing_namelist] namelist_files.append(post_processing_namelist_file) namelist_file = "" try: for namelist_file in self._NAMELIST_FILES: namelist_files.append(self.run(root_path + namelist_file, namelist_file, code_version)) return namelist_files except IOError, ex: log.critical("Can not read namelist file %s " % namelist_file) log.critical(ex)
def run(self, namelist_documentation_file, page_url, code_version): """ Run the parser on the file and return the name list file object :param namelist_documentation_file: filename for the name list file documentation :param page_url: page url :param code_version: the CodeVersion to use for the code version :return: the NamelistFile object """ log.info("Parsing %s" % namelist_documentation_file) tree = html.parse(namelist_documentation_file) namelist_file = NamelistFile() namelist_file.filename = str(tree.xpath('//h1/tt/span/text()')[0]) log.info("Filename: %s" % namelist_file.filename) namelist_file.namelists = [] namelist_index = 0 for namelist_name in tree.xpath('//h2/tt/span/text()'): namelist = Namelist() namelist.name = str(namelist_name) log.info(" Namelist: %s" % namelist.name) namelist_file.namelists.append(namelist) namelist.index_in_file = namelist_index namelist_index += 1 #parameters at the bottom level parameters = tree.xpath('//div[@id="namelist-{namelist_name}"]/dl'.format(namelist_name=namelist_name)) if namelist.name.upper() in self._EXTRA_PARAMETERS: log.info("Adding extra parameters for %s" % namelist.name) namelist.parameters.extend(self._EXTRA_PARAMETERS[namelist.name]) for parameter in namelist.parameters: parameter.code_versions = [code_version] #parameters in optional sections parameters.extend( tree.xpath('//div[@id="namelist-{namelist_name}"]/div/dl'.format(namelist_name=namelist_name))) parameters.extend( tree.xpath('//div[@id="namelist-{namelist_name}"]/div/div/dl[@class="member"]' .format(namelist_name=namelist_name))) parameters.extend( tree.xpath('//div[@id="namelist-{namelist_name}"]/div/div/dl[@class="last member"]' .format(namelist_name=namelist_name))) for parameter_elements in parameters: parameter_name = str(parameter_elements.xpath("dt")[0].attrib["id"].replace(namelist_name + '::', '')) log.info(" Parameter: %s" % parameter_name) parameter = Parameter() parameter.name = parameter_name parameter.default_value = None parameter.code_versions = [code_version] parameter.url_suffix = page_url + parameter_elements.xpath("dt/a")[0].attrib["href"] log.info(" Web page suffix: %s" % parameter.url_suffix) possible_description = parameter_elements.xpath("dd/p") parameter.description = etree.tostring(possible_description[0]) if len(possible_description) > 0 else "" log.info(" Description: %s" % parameter.description.strip()) for parameter_property in parameter_elements.xpath("dd/table/tbody/tr"): names = parameter_property.xpath("th/text()") if len(names) == 1: td = parameter_property.xpath("td")[0] value = "".join([x for x in td.itertext()]) if names[0] == 'Type :': parameter.type = value log.info(" Type: %s" % parameter.type) elif names[0] == 'Default :': self._set_parameter_default(parameter, value) log.info(" Default: %s" % parameter.default_value) elif names[0] == 'Permitted :': self._get_parameter_limits(parameter, value) log.info(" min: %s" % parameter.min) log.info(" max: %s" % parameter.max) parameter.required = (parameter.default_value is None) namelist.parameters.append(parameter) return namelist_file