def test_remove_tags_unicode(): snippet = u'<p>😋</p>' result = remove_tags(snippet) expected = u'😋' assert result == expected
def abstract(self): abstract_nodes = self.root.xpath('./front//abstract[1]') if not abstract_nodes: return abstract = remove_tags(abstract_nodes[0], **self.remove_tags_config_abstract).strip() return abstract
def test_remove_tags_strip_keeps_tails(): strip = 'self::foo' snippet = '<foo>This goes</foo> but this remains.' result = remove_tags(snippet, strip=strip) expected = u' but this remains.' assert result == expected
def abstract(self): abstract_nodes = self.root.xpath('./front//abstract[1]') if not abstract_nodes: return abstract = remove_tags(abstract_nodes[0], **self.remove_tags_config_abstract).strip() return abstract
def test_remove_tags_allowed_tags_strip(): allowed_tags = ('b', ) strip = '@class="hidden"' snippet = '<p><b><i>Only</i></b> this text remains.<span class="hidden">Not this one.</span></p>' result = remove_tags(snippet, allowed_tags=allowed_tags, strip=strip) expected = u'<b>Only</b> this text remains.' assert result == expected
def abstract(self): abstract_nodes = self.root.xpath( ".//head/abstract[not(@graphical)]/abstract-sec/simple-para") if not abstract_nodes: return abstract_paragraphs = [ remove_tags(abstract_node, **self.remove_tags_config_abstract).strip("/ \n") for abstract_node in abstract_nodes ] abstract = ' '.join(abstract_paragraphs) return abstract
def get_affiliation(self, id_): """Get the affiliation with the specified id. Args: id_(str): the value of the ``id`` attribute of the affiliation. Returns: Optional[str]: the affiliation with that id or ``None`` if there is no match. """ affiliation_node = self.root.xpath("//aff[@id=$id_]", id_=id_) if affiliation_node: affiliation = remove_tags( affiliation_node[0], strip="self::label | self::email" ).strip() return affiliation
def get_affiliation(self, id_): """Get the affiliation with the specified id. Args: id_(str): the value of the ``id`` attribute of the affiliation. Returns: Optional[str]: the affiliation with that id or ``None`` if there is no match. """ affiliation_node = self.root.xpath('//aff[@id=$id_]', id_=id_)[0] affiliation = remove_tags( affiliation_node, strip='self::label | self::email' ).strip() return affiliation
def get_reference(self, ref_node): """Extract one reference. Args: ref_node(scrapy.selector.Selector): a selector on a single reference, i.e. ``<ref>``. Returns: dict: the parsed reference, as generated by :class:`inspire_schemas.api.ReferenceBuilder` """ for citation_node in ref_node.xpath('./mixed-citation'): builder = ReferenceBuilder() builder.add_raw_reference( ref_node.extract().strip(), source=self.builder.source, ref_format='JATS' ) fields = [ ( ( 'self::node()[@publication-type="journal" ' 'or @publication-type="eprint"]/source/text()' ), builder.set_journal_title, ), ( 'self::node()[@publication-type="book"]/source/text()', builder.add_parent_title, ), ('./publisher-name/text()', builder.set_publisher), ('./volume/text()', builder.set_journal_volume), ('./issue/text()', builder.set_journal_issue), ('./year/text()', builder.set_year), ('./pub-id[@pub-id-type="arxiv"]/text()', builder.add_uid), ('./pub-id[@pub-id-type="doi"]/text()', builder.add_uid), ( 'pub-id[@pub-id-type="other"]' '[contains(preceding-sibling::text(),"Report No")]/text()', builder.add_report_number ), ('./article-title/text()', builder.add_title), ('../label/text()', lambda x: builder.set_label(x.strip('[].'))) ] for xpath, field_handler in fields: value = citation_node.xpath(xpath).extract_first() citation_node.xpath(xpath) if value: field_handler(value) remainder = remove_tags( citation_node, strip='self::person-group' '|self::pub-id' '|self::article-title' '|self::volume' '|self::issue' '|self::year' '|self::label' '|self::publisher-name' '|self::source[../@publication-type!="proc"]' '|self::object-id' '|self::page-range' '|self::issn' ).strip('"\';,. \t\n\r').replace('()', '') if remainder: builder.add_misc(remainder) for editor in self.get_reference_authors(citation_node, 'editor'): builder.add_author(editor, 'editor') for author in self.get_reference_authors(citation_node, 'author'): builder.add_author(author, 'author') page_range = citation_node.xpath('./page-range/text()').extract_first() if page_range: page_artid = split_page_artid(page_range) builder.set_page_artid(*page_artid) yield builder.obj
def get_reference_iter(self, ref_node): """Extract one reference. Args: ref_node(scrapy.selector.Selector): a selector on a single reference, i.e. ``<ref>``. Yields: dict: the parsed reference, as generated by :class:`inspire_schemas.api.ReferenceBuilder` """ # handle also unstructured refs for citation_node in ref_node.xpath("./reference|./other-ref"): builder = ReferenceBuilder() builder.add_raw_reference( ref_node.extract().strip(), source=self.builder.source, ref_format="Elsevier", ) fields = [ ( ("string(.//series/title/maintitle[1])"), builder.set_journal_title, ), ( "string(.//title[parent::edited-book|parent::book]/maintitle[1])", builder.add_parent_title, ), ("string(./publisher/name[1])", builder.set_publisher), ("string(.//volume-nr[1])", builder.set_journal_volume), ("string(.//issue-nr[1])", builder.set_journal_issue), ("string(.//date[1])", builder.set_year), ("string(.//inter-ref[1])", builder.add_url), ("string(.//doi[1])", builder.add_uid), ( 'string(pub-id[@pub-id-type="other"]' '[contains(preceding-sibling::text(),"Report No")][1])', builder.add_report_number, ), ("string(./title/maintitle[1])", builder.add_title), ] for xpath, field_handler in fields: value = citation_node.xpath(xpath).extract_first() citation_node.xpath(xpath) if value: field_handler(value) label_value = ref_node.xpath("string(./label[1])").extract_first() builder.set_label(label_value.strip("[]")) pages = self.get_reference_pages(citation_node) artid = self.get_reference_artid(citation_node) if artid: builder.set_page_artid(artid=artid) if any(pages): builder.set_page_artid(*pages) remainder = (remove_tags( citation_node, strip="self::authors" "|self::article-number" "|self::volume-nr" "|self::issue-nr" "|self::inter-ref" "|self::maintitle" "|self::date" "|self::label" "|self::publisher" "|self::doi" "|self::pages").strip("\"';,. \t\n\r").replace("()", "")) if remainder: builder.add_misc(remainder) for editor in self.get_reference_editors(citation_node): builder.add_author(editor, "editor") for author in self.get_reference_authors(citation_node): builder.add_author(author, "author") yield builder.obj
def get_reference(self, ref_node): """Extract one reference. Args: ref_node(scrapy.selector.Selector): a selector on a single reference, i.e. ``<ref>``. Returns: dict: the parsed reference, as generated by :class:`inspire_schemas.api.ReferenceBuilder` """ for citation_node in ref_node.xpath('./mixed-citation'): builder = ReferenceBuilder() builder.add_raw_reference( ref_node.extract().strip(), source=self.builder.source, ref_format='JATS' ) fields = [ ( ( 'self::node()[@publication-type="journal" ' 'or @publication-type="eprint"]/source/text()' ), builder.set_journal_title, ), ( 'self::node()[@publication-type="book"]/source/text()', builder.add_parent_title, ), ('./publisher-name/text()', builder.set_publisher), ('./volume/text()', builder.set_journal_volume), ('./issue/text()', builder.set_journal_issue), ('./year/text()', builder.set_year), ('./pub-id[@pub-id-type="arxiv"]/text()', builder.add_uid), ('./pub-id[@pub-id-type="doi"]/text()', builder.add_uid), ( 'pub-id[@pub-id-type="other"]' '[contains(preceding-sibling::text(),"Report No")]/text()', builder.add_report_number ), ('./article-title/text()', builder.add_title), ('../label/text()', lambda x: builder.set_label(x.strip('[].'))) ] for xpath, field_handler in fields: value = citation_node.xpath(xpath).extract_first() citation_node.xpath(xpath) if value: field_handler(value) remainder = remove_tags( citation_node, strip='self::person-group' '|self::pub-id' '|self::article-title' '|self::volume' '|self::issue' '|self::year' '|self::label' '|self::publisher-name' '|self::source[../@publication-type!="proc"]' '|self::object-id' '|self::page-range' '|self::issn' ).strip('"\';,. \t\n\r').replace('()', '') if remainder: builder.add_misc(remainder) for editor in self.get_reference_authors(citation_node, 'editor'): builder.add_author(editor, 'editor') for author in self.get_reference_authors(citation_node, 'author'): builder.add_author(author, 'author') page_range = citation_node.xpath('./page-range/text()').extract_first() if page_range: page_artid = split_page_artid(page_range) builder.set_page_artid(*page_artid) yield builder.obj