def test_attribute_default_depending_element():
present = DefaultAttrTestDoc(attr=5, default_attr=(1, 2))
assert present.default_attr == (1, 2)
lack = DefaultAttrTestDoc(attr=5)
assert lack.default_attr == (5, 10)
present = read(DefaultAttrTestDoc,
[b'<default-attr-test attr="5" default-attr="1,2" />'])
assert present.default_attr == (1, 2)
lack = read(DefaultAttrTestDoc, [b'<default-attr-test attr="5" />'])
assert lack.default_attr == (5, 10)
def test_attribute_default():
present = DefaultAttrTestDoc(default_attr=(1, 2))
assert present.default_attr == (1, 2)
lack = DefaultAttrTestDoc()
assert lack.default_attr == (0, 0)
present = read(DefaultAttrTestDoc,
[b'<default-attr-test default-attr="1,2" />'])
assert present.default_attr == (1, 2)
lack = read(DefaultAttrTestDoc, [b'<default-attr-test />'])
assert lack.default_attr == (0, 0)
def test_attribute_default_depending_element():
present = DefaultAttrTestDoc(attr=5, default_attr=(1, 2))
assert present.default_attr == (1, 2)
lack = DefaultAttrTestDoc(attr=5)
assert lack.default_attr == (5, 10)
present = read(
DefaultAttrTestDoc,
[b'<default-attr-test attr="5" default-attr="1,2" />']
)
assert present.default_attr == (1, 2)
lack = read(DefaultAttrTestDoc, [b'<default-attr-test attr="5" />'])
assert lack.default_attr == (5, 10)
def test_attribute_default():
present = DefaultAttrTestDoc(default_attr=(1, 2))
assert present.default_attr == (1, 2)
lack = DefaultAttrTestDoc()
assert lack.default_attr == (0, 0)
present = read(
DefaultAttrTestDoc,
[b'<default-attr-test default-attr="1,2" />']
)
assert present.default_attr == (1, 2)
lack = read(DefaultAttrTestDoc, [b'<default-attr-test />'])
assert lack.default_attr == (0, 0)
def test_element_list_consume_buffer_regression_root_stack_top_should_be_1():
xml = [b'<a><b><!-- 1 --><c></c><c><d>', b'content</d></c><c></c></b><b>',
b'<!-- 2 --><c><d>abc</d></c></b><b><!-- 3 --></b></a>']
doc = read(ELConsumeBufferRegressionTestDoc, xml)
assert len(doc.b) == 3
b = doc.b[0]
assert len(b.c) == 3
def test_source():
entry = read(Entry, [b'''
<entry xmlns="http://www.w3.org/2005/Atom">
<source>
<title>Source of all knowledge</title>
<id>urn:uuid:28213c50-f84c-11d9-8cd6-0800200c9a66</id>
<updated>2003-12-13T17:46:27Z</updated>
<category term="technology"/>
<category term="business"/>
</source>
<title>Atom-Powered Robots Run Amok</title>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
</entry>
'''])
source = entry.source
assert isinstance(source, Source)
assert source.title == Text(value='Source of all knowledge')
assert source.id == 'urn:uuid:28213c50-f84c-11d9-8cd6-0800200c9a66'
assert source.updated_at == datetime.datetime(2003, 12, 13, 17, 46, 27,
tzinfo=utc)
categories = source.categories
assert isinstance(categories[0], Category)
assert categories[0].term == 'technology'
assert isinstance(categories[1], Category)
assert categories[1].term == 'business'
assert len(categories) == 2
def fx_test_doc():
consume_log = []
xml = string_chunks(
consume_log, '<test attr="속성 값" attr-decoder="Decoder Test">',
['TEST_START'], '\n', '\t', '<title>', '제목 ', 'test', '</title>',
['TITLE_CLOSE'], '\n', '\t', '<multi>', 'a', '</multi>',
['MULTI_1_CLOSE'], '\n', '\t', '<content>', 'Content', ' test',
'</content>', ['CONTENT_CLOSE'], '\n', '\t', '<multi>', 'b',
'</multi>', ['MULTI_2_CLOSE'], '\n', '\t', '<text-content>', '텍스트 ',
'내용', '</text-content>', ['TEXT_CONTENT_CLOSE'], '\n', '\t',
'<text-multi>', 'a', '</text-multi>', ['TEXT_MULTI_1_CLOSE'], '\n',
'\t', '<multi>', 'c', '</multi>', ['MULTI_3_CLOSE'], '\n', '\t',
'<s-multi>c</s-multi><s-multi>a</s-multi><s-multi>b</s-multi>\n', '\t',
'<s-text-multi>c</s-text-multi>', '\n', '\t',
'<s-text-multi>a</s-text-multi>', '\n', '\t',
'<s-text-multi>b</s-text-multi>', '\n', '\t', '<text-multi>', 'b',
'</text-multi>', ['TEXT_MULTI_2_CLOSE'], '\n', '\t', '<text-decoder>',
'123.456', '</text-decoder>', '\n', '\t', '<text-decoder-decorator>',
'123', '</text-decoder-decorator>', '\n', '\t',
'<text-combined-decoder>', '1234', '</text-combined-decoder>', '\n',
'\t', '<nst:ns-element xmlns:nst="http://earthreader.github.io/" ',
'nst:ns-attr="namespace attribute value">', ['NS_ELEMENT_START'],
'Namespace test', '</nst:ns-element>', '\n', '\t',
'<nst2:ns-text xmlns:nst2="http://earthreader.github.io/">',
'Namespace test', '</nst2:ns-text>', '\n', '\t', '<content-decoder>',
'content decoder', '</content-decoder>', '\n'
'</test>', ['TEST_CLOSE'], '\n')
return read(TestDoc, xml), consume_log
def test_element_list_consume_buffer_regression_root_stack_top_should_be_1():
xml = [b'<a><b><!-- 1 --><c></c><c><d>', b'content</d></c><c></c></b><b>',
b'<!-- 2 --><c><d>abc</d></c></b><b><!-- 3 --></b></a>']
doc = read(ELConsumeBufferRegressionTestDoc, xml)
assert len(doc.b) == 3
b = doc.b[0]
assert len(b.c) == 3
def test_source():
entry = read(Entry, [b'''
<entry xmlns="http://www.w3.org/2005/Atom">
<source>
<title>Source of all knowledge</title>
<id>urn:uuid:28213c50-f84c-11d9-8cd6-0800200c9a66</id>
<updated>2003-12-13T17:46:27Z</updated>
<category term="technology"/>
<category term="business"/>
</source>
<title>Atom-Powered Robots Run Amok</title>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
</entry>
'''])
source = entry.source
assert isinstance(source, Source)
assert source.title == Text(value='Source of all knowledge')
assert source.id == 'urn:uuid:28213c50-f84c-11d9-8cd6-0800200c9a66'
assert source.updated_at == datetime.datetime(2003, 12, 13, 17, 46, 27,
tzinfo=utc)
categories = source.categories
assert isinstance(categories[0], Category)
assert categories[0].term == 'technology'
assert isinstance(categories[1], Category)
assert categories[1].term == 'business'
assert len(categories) == 2
def fx_feed():
return read(Feed, [b'''
<feed xmlns="http://www.w3.org/2005/Atom"
xmlns:mark="http://earthreader.org/mark/">
<title>Example Feed</title>
<link href="http://example.org/"/>
<updated>2003-12-13T18:30:02Z</updated>
<author><name>John Doe</name></author>
<author><name>Jane Doe</name></author>
<id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
<category term="technology"/>
<category term="business"/>
<rights>Public Domain</rights>
<entry>
<title>Atom-Powered Robots Run Amok</title>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
<author><name>Jane Doe</name></author>
<mark:read updated="2013-11-06T14:36:00Z">true</mark:read>
</entry>
<entry>
<title>Danger, Will Robinson!</title>
<link href="http://example.org/2003/12/13/lost"/>
<id>urn:uuid:b12f2c10-ffc1-11d9-8cd6-0800200c9a66</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Don't Panic!</summary>
</entry>
</feed>
'''])
def fx_test_doc():
consume_log = []
xml = string_chunks(
consume_log,
'<test attr="속성 값" attr-decoder="Decoder Test">',
['TEST_START'], '\n',
'\t', '<title>', '제목 ', 'test', '</title>', ['TITLE_CLOSE'], '\n',
'\t', '<multi>', 'a', '</multi>', ['MULTI_1_CLOSE'], '\n',
'\t', '<content>', 'Content', ' test',
'</content>', ['CONTENT_CLOSE'], '\n',
'\t', '<multi>', 'b', '</multi>', ['MULTI_2_CLOSE'], '\n',
'\t', '<text-content>', '텍스트 ', '내용',
'</text-content>', ['TEXT_CONTENT_CLOSE'], '\n',
'\t', '<text-multi>', 'a', '</text-multi>',
['TEXT_MULTI_1_CLOSE'], '\n',
'\t', '<multi>', 'c', '</multi>', ['MULTI_3_CLOSE'], '\n',
'\t', '<text-multi>', 'b', '</text-multi>',
['TEXT_MULTI_2_CLOSE'], '\n',
'\t', '<text-decoder>', '123.456', '</text-decoder>', '\n',
'\t', '<text-decoder-decorator>', '123',
'</text-decoder-decorator>', '\n',
'\t', '<text-combined-decoder>', '1234',
'</text-combined-decoder>', '\n',
'\t', '<nst:ns-element xmlns:nst="http://earthreader.github.io/" ',
'nst:ns-attr="namespace attribute value">', ['NS_ELEMENT_START'],
'Namespace test', '</nst:ns-element>', '\n',
'\t', '<nst2:ns-text xmlns:nst2="http://earthreader.github.io/">',
'Namespace test', '</nst2:ns-text>', '\n',
'\t', '<content-decoder>', 'content decoder', '</content-decoder>', '\n'
'</test>', ['TEST_CLOSE'], '\n'
)
return read(TestDoc, xml), consume_log
def fx_feed():
return read(Feed, [
b'''
<feed xmlns="http://www.w3.org/2005/Atom"
xmlns:mark="http://earthreader.org/mark/">
<title>Example Feed</title>
<link href="http://example.org/"/>
<updated>2003-12-13T18:30:02Z</updated>
<author><name>John Doe</name></author>
<author><name>Jane Doe</name></author>
<id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
<category term="technology"/>
<category term="business"/>
<rights>Public Domain</rights>
<entry>
<title>Atom-Powered Robots Run Amok</title>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
<author><name>Jane Doe</name></author>
<mark:read updated="2013-11-06T14:36:00Z">true</mark:read>
</entry>
<entry>
<title>Danger, Will Robinson!</title>
<link href="http://example.org/2003/12/13/lost"/>
<id>urn:uuid:b12f2c10-ffc1-11d9-8cd6-0800200c9a66</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Don't Panic!</summary>
</entry>
</feed>
'''
])
def apply_timestamp(stage, feed_id, timestamp):
with stage:
feed = stage.feeds[feed_id]
feed.entries[0].read = Mark(marked=True, updated_at=timestamp)
assert feed.entries[0].read.updated_at == timestamp
written = read(Feed, write(feed, as_bytes=True))
assert written.entries[0].read.updated_at == timestamp, repr((written.entries[0].read.updated_at, timestamp))
stage.feeds[feed_id] = feed
def fx_xmlns_doc():
return read(XmlnsDoc, [b'''
<nstest xmlns="http://earthreader.github.io/"
xmlns:nst="https://github.com/earthreader/libearth">
<samens>Same namespace</samens>
<nst:otherns>Other namespace</nst:otherns>
</nstest>
'''])
def fx_xmlns_doc():
return read(XmlnsDoc, [b'''
<nstest xmlns="http://earthreader.github.io/"
xmlns:nst="https://github.com/earthreader/libearth">
<samens>Same namespace</samens>
<nst:otherns>Other namespace</nst:otherns>
</nstest>
'''])
def apply_timestamp(stage, feed_id, timestamp):
with stage:
feed = stage.feeds[feed_id]
feed.entries[0].read = Mark(marked=True, updated_at=timestamp)
assert feed.entries[0].read.updated_at == timestamp
written = read(Feed, write(feed, as_bytes=True))
assert written.entries[0].read.updated_at == timestamp, repr(
(written.entries[0].read.updated_at, timestamp))
stage.feeds[feed_id] = feed
def test_rss_parser():
my_opener = urllib2.build_opener(TestHTTPHandler)
urllib2.install_opener(my_opener)
crawled_feed, data_for_crawl = rss2.parse_rss(
rss_xml,
'http://sourcetest.com/rss.xml'
)
feed = read(Feed, write(crawled_feed, as_bytes=True))
assert crawled_feed.id == feed.id
title = crawled_feed.title
assert title.type == feed.title.type
assert title.value == feed.title.value
links = crawled_feed.links
assert links[1].mimetype == feed.links[1].mimetype
assert links[1].relation == feed.links[1].relation
assert links[1].uri == feed.links[1].uri
rights = crawled_feed.rights
assert rights.type == feed.rights.type
assert rights.value == feed.rights.value
contributors = crawled_feed.contributors
assert contributors[0].name == feed.contributors[0].name
assert contributors[0].email == feed.contributors[0].email
assert contributors[1].name == feed.contributors[1].name
assert contributors[1].email == feed.contributors[1].email
updated_at = crawled_feed.updated_at
assert updated_at == feed.updated_at
categories = crawled_feed.categories
assert categories[0].term == feed.categories[0].term
entries = crawled_feed.entries
assert entries[0].title.type == feed.entries[0].title.type
assert entries[0].title.value == feed.entries[0].title.value
assert entries[0].links[0].mimetype == feed.entries[0].links[0].mimetype
assert entries[0].links[0].relation == feed.entries[0].links[0].relation
assert entries[0].links[0].uri == feed.entries[0].links[0].uri
assert entries[0].content.value == feed.entries[0].content.value
assert entries[0].authors[0].name == feed.entries[0].authors[0].name
assert entries[0].authors[0].email == feed.entries[0].authors[0].email
assert entries[0].links[1].mimetype == feed.entries[0].links[1].mimetype
assert entries[0].links[1].uri == feed.entries[0].links[1].uri
assert entries[0].id == feed.entries[0].id
assert (entries[0].published_at ==
entries[0].updated_at ==
feed.entries[0].published_at ==
feed.entries[0].updated_at)
assert data_for_crawl == {
'lastBuildDate': datetime.datetime(2002, 9, 7, 0, 0, 1, tzinfo=utc),
'ttl': '10',
}
source = entries[0].source
assert source.title.type == feed.entries[0].source.title.type
assert source.title.value == feed.entries[0].source.title.value
assert source.links[1].mimetype == feed.entries[0].source.links[1].mimetype
assert source.links[1].uri == feed.entries[0].source.links[1].uri
assert source.links[1].relation == feed.entries[0].source.links[1].relation
assert source.subtitle.type == feed.entries[0].source.subtitle.type
assert source.subtitle.value == feed.entries[0].source.subtitle.value
assert not source.entries
def test_complete_single_chunk():
chunk = b'''
<nstest xmlns="http://earthreader.github.io/"
xmlns:nst="https://github.com/earthreader/libearth">
</nstest>
'''
doc = read(XmlnsDoc, [chunk])
complete(doc)
assert doc.samens_attr is None
assert doc.otherns_attr is None
def test_complete_single_chunk():
chunk = b'''
<nstest xmlns="http://earthreader.github.io/"
xmlns:nst="https://github.com/earthreader/libearth">
</nstest>
'''
doc = read(XmlnsDoc, [chunk])
complete(doc)
assert doc.samens_attr is None
assert doc.otherns_attr is None
def fx_non_exist_opml(fx_test_stage):
feed_urls = ['http://feedone.com/feed/atom/']
generator = crawl(feed_urls, 1)
for result in generator:
feed_data = result[1]
feed_url = result[0]
feed_id = get_hash(feed_url)
with fx_test_stage as stage:
stage.feeds[feed_id] = feed_data
with fx_test_stage as stage:
stage.subscriptions = read(SubscriptionList, opml_with_non_exist_feed)
def test_complete_terminates_streaming():
f = io.BytesIO(b'''
<nstest xmlns="http://earthreader.github.io/"
xmlns:nst="https://github.com/earthreader/libearth">
</nstest>
''')
doc = read(XmlnsDoc, f)
complete(doc)
f.close()
assert doc.samens_attr is None
assert doc.otherns_attr is None
def fx_non_exist_opml(fx_test_stage):
feed_urls = ['http://feedone.com/feed/atom/']
generator = crawl(feed_urls, 1)
for result in generator:
feed_data = result[1]
feed_url = result[0]
feed_id = get_hash(feed_url)
with fx_test_stage as stage:
stage.feeds[feed_id] = feed_data
with fx_test_stage as stage:
stage.subscriptions = read(SubscriptionList, opml_with_non_exist_feed)
def test_complete_terminates_streaming():
f = io.BytesIO(b'''
<nstest xmlns="http://earthreader.github.io/"
xmlns:nst="https://github.com/earthreader/libearth">
</nstest>
''')
doc = read(XmlnsDoc, f)
complete(doc)
f.close()
assert doc.samens_attr is None
assert doc.otherns_attr is None
def test_rss_parser():
my_opener = urllib2.build_opener(TestHTTPHandler)
urllib2.install_opener(my_opener)
crawled_feed, data_for_crawl = parse_rss(rss_xml,
'http://sourcetest.com/rss.xml')
feed = read(Feed, write(crawled_feed, as_bytes=True))
assert crawled_feed.id == feed.id
title = crawled_feed.title
assert title.type == feed.title.type
assert title.value == feed.title.value
links = crawled_feed.links
assert links[1].mimetype == feed.links[1].mimetype
assert links[1].relation == feed.links[1].relation
assert links[1].uri == feed.links[1].uri
rights = crawled_feed.rights
assert rights.type == feed.rights.type
assert rights.value == feed.rights.value
contributors = crawled_feed.contributors
assert contributors[0].name == feed.contributors[0].name
assert contributors[0].email == feed.contributors[0].email
assert contributors[1].name == feed.contributors[1].name
assert contributors[1].email == feed.contributors[1].email
updated_at = crawled_feed.updated_at
assert updated_at == feed.updated_at
categories = crawled_feed.categories
assert categories[0].term == feed.categories[0].term
entries = crawled_feed.entries
assert entries[0].title.type == feed.entries[0].title.type
assert entries[0].title.value == feed.entries[0].title.value
assert entries[0].links[0].mimetype == feed.entries[0].links[0].mimetype
assert entries[0].links[0].relation == feed.entries[0].links[0].relation
assert entries[0].links[0].uri == feed.entries[0].links[0].uri
assert entries[0].content.value == feed.entries[0].content.value
assert entries[0].authors[0].name == feed.entries[0].authors[0].name
assert entries[0].authors[0].email == feed.entries[0].authors[0].email
assert entries[0].links[1].mimetype == feed.entries[0].links[1].mimetype
assert entries[0].links[1].uri == feed.entries[0].links[1].uri
assert entries[0].id == feed.entries[0].id
assert (entries[0].published_at == entries[0].updated_at ==
feed.entries[0].published_at == feed.entries[0].updated_at)
assert data_for_crawl == {
'lastBuildDate': datetime.datetime(2002, 9, 7, 0, 0, 1, tzinfo=utc),
'ttl': '10',
}
source = entries[0].source
assert source.title.type == feed.entries[0].source.title.type
assert source.title.value == feed.entries[0].source.title.value
assert source.links[1].mimetype == feed.entries[0].source.links[1].mimetype
assert source.links[1].uri == feed.entries[0].source.links[1].uri
assert source.links[1].relation == feed.entries[0].source.links[1].relation
assert source.subtitle.type == feed.entries[0].source.subtitle.type
assert source.subtitle.value == feed.entries[0].source.subtitle.value
assert not source.entries
def fx_hinted_doc():
consume_log = []
xml = string_chunks(
consume_log, '<test xmlns:l="http://earthreader.org/schema/">', '\n',
'\t', '<l:hint tag="multi" id="length" value="3" />', '\n', '\t',
'<l:hint tag="s-multi" id="length" value="0" />', ['HINT'], '\n', '\t',
'<title>Title</title>', '\n', '\t', '<multi>a</multi>',
['MULTI_STARTED'], '\n', '\t', '<content>Content</content>', '\n',
'\t', '<multi>b</multi>', '\n', '\t', '<multi>c</multi>', '\n',
'</test>')
doc = read(TestDoc, xml)
return doc, consume_log
def test_stage_write(fx_repo, fx_session, fx_stage):
doc = TestDoc()
min_ts = now()
with fx_stage:
wdoc = fx_stage.write(['doc.{0}.xml'.format(fx_session.identifier)],
doc)
assert wdoc.__revision__.session is fx_session
assert min_ts <= wdoc.__revision__.updated_at <= now()
xml = fx_repo.data['doc.{0}.xml'.format(fx_session.identifier)]
read_doc = read(TestDoc, [xml])
assert isinstance(read_doc, TestDoc)
assert read_doc.__revision__ == wdoc.__revision__
def test_stage_write(fx_repo, fx_session, fx_stage):
doc = TestDoc()
min_ts = now()
with fx_stage:
wdoc = fx_stage.write(['doc.{0}.xml'.format(fx_session.identifier)],
doc)
assert wdoc.__revision__.session is fx_session
assert min_ts <= wdoc.__revision__.updated_at <= now()
xml = fx_repo.data['doc.{0}.xml'.format(fx_session.identifier)]
read_doc = read(TestDoc, [xml])
assert isinstance(read_doc, TestDoc)
assert read_doc.__revision__ == wdoc.__revision__
def test_partial_load_test():
doc = read(PartialLoadTestDoc, b'''
<x:partial-load-test xmlns:x="http://example.com/">
<x:entry>
<x:value>as<!--
-->df</x:value>
</x:entry>
<x:entry>
<x:value>as<!--
-->df</x:value>
</x:entry>
</x:partial-load-test>
'''.splitlines())
assert doc.entry.value == 'asdf'
def test_partial_load_test():
doc = read(PartialLoadTestDoc, b'''
<x:partial-load-test xmlns:x="http://example.com/">
<x:entry>
<x:value>as<!--
-->df</x:value>
</x:entry>
<x:entry>
<x:value>as<!--
-->df</x:value>
</x:entry>
</x:partial-load-test>
'''.splitlines())
assert doc.entry.value == 'asdf'
def test_merge_outlines_without_created_at(fx_stages):
"""https://github.com/earthreader/libearth/issues/65"""
subscriptions = read(SubscriptionList, [b'''
<opml version="2.0">
<body>
<outline text="CNET News.com" type="rss" version="RSS2"
xmlUrl="http://news.com/2547-1_3-0-5.xml"/>
</body>
</opml>
'''])
stage, _ = fx_stages
with stage:
stage.subscriptions = subscriptions
read_subscriptions = stage.subscriptions
stage.subscriptions = read_subscriptions
assert len(stage.subscriptions) == 1
def fx_hinted_doc():
consume_log = []
xml = string_chunks(
consume_log,
'<test xmlns:l="http://earthreader.org/schema/">', '\n',
'\t', '<l:hint tag="multi" id="length" value="3" />', '\n',
'\t', '<l:hint tag="s-multi" id="length" value="0" />', ['HINT'], '\n',
'\t', '<title>Title</title>', '\n',
'\t', '<multi>a</multi>', ['MULTI_STARTED'], '\n',
'\t', '<content>Content</content>', '\n',
'\t', '<multi>b</multi>', '\n',
'\t', '<multi>c</multi>', '\n',
'</test>'
)
doc = read(TestDoc, xml)
return doc, consume_log
def test_entry_read():
# http://www.intertwingly.net/wiki/pie/FormatTests
entry = read(Entry, [
b'''
<entry xmlns="http://www.w3.org/2005/Atom">
<title>Atom-Powered Robots Run Amok</title>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
<category term="technology"/>
<category term="business"/>
<contributor>
<name>John Smith</name>
</contributor>
<contributor>
<name>Jane Doe</name>
</contributor>
</entry>
'''
])
assert entry.id == 'urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a'
assert entry.title == Text(value='Atom-Powered Robots Run Amok')
assert entry.updated_at == datetime.datetime(2003,
12,
13,
18,
30,
2,
tzinfo=utc)
assert isinstance(entry.links[0], Link)
assert entry.links[0].uri == 'http://example.org/2003/12/13/atom03'
assert entry.links[0].relation == 'alternate'
assert len(entry.links) == 1
assert isinstance(entry.summary, Text)
assert entry.summary.type == 'text'
assert entry.summary.value == 'Some text.'
assert isinstance(entry.categories[0], Category)
assert entry.categories[0].term == 'technology'
assert entry.categories[1].term == 'business'
assert len(entry.categories) == 2
assert isinstance(entry.contributors[0], Person)
assert entry.contributors[0].name == 'John Smith'
assert entry.contributors[1].name == 'Jane Doe'
assert len(entry.contributors) == 2
def test_entry_read():
# http://www.intertwingly.net/wiki/pie/FormatTests
entry = read(
Entry,
[
b"""
<entry xmlns="http://www.w3.org/2005/Atom">
<title>Atom-Powered Robots Run Amok</title>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
<category term="technology"/>
<category term="business"/>
<contributor>
<name>John Smith</name>
</contributor>
<contributor>
<name>Jane Doe</name>
</contributor>
</entry>
"""
],
)
assert entry.id == "urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a"
assert entry.title == Text(value="Atom-Powered Robots Run Amok")
assert entry.updated_at == datetime.datetime(2003, 12, 13, 18, 30, 2, tzinfo=utc)
assert isinstance(entry.links[0], Link)
assert entry.links[0].uri == "http://example.org/2003/12/13/atom03"
assert entry.links[0].relation == "alternate"
assert len(entry.links) == 1
assert isinstance(entry.summary, Text)
assert entry.summary.type == "text"
assert entry.summary.value == "Some text."
assert isinstance(entry.categories[0], Category)
assert entry.categories[0].term == "technology"
assert entry.categories[1].term == "business"
assert len(entry.categories) == 2
assert isinstance(entry.contributors[0], Person)
assert entry.contributors[0].name == "John Smith"
assert entry.contributors[1].name == "Jane Doe"
assert len(entry.contributors) == 2
def fx_filtering_entries(fx_test_stage):
authors = [Person(name='vio')]
now = datetime.datetime(2013, 10, 30, 20, 55, 30, tzinfo=utc)
feed = Feed(id='http://feedone.com/feed/atom/', authors=authors,
title=Text(value='Feed One'),
updated_at=now)
for i in range(10):
feed.entries.append(
Entry(id='http://feedone.com/feed/atom/' + str(i) + '/',
authors=authors,
title=Text(value=str(i + 1)),
updated_at=now + datetime.timedelta(days=1) * i)
)
for i in range(5):
feed.entries[i].read = Mark(marked=True, updated_at=now)
for i in range(3, 7):
feed.entries[i].starred = Mark(marked=True, updated_at=now)
with fx_test_stage as stage:
stage.feeds[get_hash('http://feedone.com/feed/atom/')] = feed
stage.subscriptions = read(SubscriptionList, opml_for_filtering)
def fx_filtering_entries(fx_test_stage):
authors = [Person(name='vio')]
now = datetime.datetime(2013, 10, 30, 20, 55, 30, tzinfo=utc)
feed = Feed(id='http://feedone.com/feed/atom/', authors=authors,
title=Text(value='Feed One'),
updated_at=now)
for i in range(10):
feed.entries.append(
Entry(id='http://feedone.com/feed/atom/' + str(i) + '/',
authors=authors,
title=Text(value=str(i + 1)),
updated_at=now + datetime.timedelta(days=1) * i)
)
for i in range(5):
feed.entries[i].read = Mark(marked=True, updated_at=now)
for i in range(3, 7):
feed.entries[i].starred = Mark(marked=True, updated_at=now)
with fx_test_stage as stage:
stage.feeds[get_hash('http://feedone.com/feed/atom/')] = feed
stage.subscriptions = read(SubscriptionList, opml_for_filtering)
def fx_subscription_list():
return read(SubscriptionList, [XML])
def fx_categorized_subscription_list():
return read(SubscriptionList, [XML_CATEGORY])
def test_atom_parser():
url = 'http://vio.atomtest.com/feed/atom'
crawled_feed, _ = parse_atom(atom_xml, url)
feed = read(Feed, write(crawled_feed, as_bytes=True))
title = crawled_feed.title
assert title.type == feed.title.type
assert title.value == feed.title.value
subtitle = crawled_feed.subtitle
assert subtitle.type == feed.subtitle.type
assert subtitle.value == feed.subtitle.value
links = crawled_feed.links
assert links[0].relation == feed.links[0].relation
assert links[0].mimetype == feed.links[0].mimetype
assert links[0].uri == feed.links[0].uri
assert links[1].relation == feed.links[1].relation
assert links[1].mimetype == feed.links[1].mimetype
assert links[1].uri == feed.links[1].uri
authors = crawled_feed.authors
assert authors[0].name == feed.authors[0].name
assert authors[0].email == feed.authors[0].email
categories = crawled_feed.categories
assert categories[0].term == feed.categories[0].term
contributors = crawled_feed.contributors
assert contributors[0].name == feed.contributors[0].name
generator = crawled_feed.generator
assert generator.uri == feed.generator.uri
assert generator.value == feed.generator.value
icon = crawled_feed.icon
assert icon == feed.icon
logo = crawled_feed.logo
assert logo == feed.logo
rights = crawled_feed.rights
assert rights.type == feed.rights.type
assert rights.value == feed.rights.value
updated_at = crawled_feed.updated_at
assert updated_at == feed.updated_at
entries = crawled_feed.entries
assert entries[0].id == feed.entries[0].id
assert entries[0].authors[0].name == feed.entries[0].authors[0].name
assert entries[0].title.type == feed.entries[0].title.type
assert entries[0].title.value == feed.entries[0].title.value
assert entries[0].links[0].relation == feed.entries[0].links[0].relation
assert entries[0].links[0].uri == feed.entries[0].links[0].uri
assert entries[0].updated_at == feed.entries[0].updated_at
assert entries[0].published_at == feed.entries[0].published_at
assert entries[0].categories[0].scheme_uri == \
feed.entries[0].categories[0].scheme_uri
assert entries[0].categories[0].term == feed.entries[0].categories[0].term
assert entries[0].categories[1].scheme_uri == \
feed.entries[0].categories[1].scheme_uri
assert entries[0].categories[1].term == feed.entries[0].categories[1].term
assert entries[0].content.type == feed.entries[0].content.type
assert entries[0].content.value == feed.entries[0].content.value
assert entries[0].content.source_uri == feed.entries[0].content.source_uri
assert entries[0].summary.type == feed.entries[0].summary.type
assert entries[0].summary.value == feed.entries[0].summary.value
assert entries[1].id == feed.entries[1].id
assert entries[1].authors[0].name == feed.entries[1].authors[0].name
assert entries[1].title.type == feed.entries[1].title.type
assert entries[1].title.value == feed.entries[1].title.value
assert entries[1].updated_at == feed.entries[1].updated_at
assert entries[2].id == feed.entries[2].id
assert entries[2].authors[0].name == feed.entries[2].authors[0].name
assert entries[2].title.type == feed.entries[2].title.type
assert entries[2].title.value == feed.entries[2].title.value
assert entries[2].updated_at == feed.entries[2].updated_at
source = entries[2].source
feed_source = feed.entries[2].source
assert source.authors[0].name == feed_source.authors[0].name
assert source.categories[0].term == feed_source.categories[0].term
assert source.contributors[0].name == feed_source.contributors[0].name
assert source.links[0] == feed_source.links[0]
assert source.id == feed_source.id
assert source.generator == feed_source.generator
assert source.icon == feed_source.icon
assert source.logo == feed_source.logo
assert source.rights == feed_source.rights
assert source.subtitle == feed_source.subtitle
def test_attribute_codec():
doc = CodecTestDoc(attr=(1, 2))
tree = fromstringlist(write(doc))
assert tree.attrib['attr'] == '1,2'
doc2 = read(CodecTestDoc, etree_tobyteslist(tree))
assert doc2.attr == (1, 2)
def test_element_list_consume_buffer_regression():
xml = [b'<a><b><c></c><c><d>content', b'</d></c><c></c></b><b></b></a>']
doc = read(ELConsumeBufferRegressionTestDoc, xml)
assert len(doc.b) == 2
b = doc.b[0]
assert len(b.c) == 3
def test_text_codec():
doc = CodecTestDoc(text=(3, 4))
tree = fromstringlist(write(doc))
assert tree.find('text').text == '3,4'
doc2 = read(CodecTestDoc, etree_tobyteslist(tree))
assert doc2.text == (3, 4)
def fx_headless_subscription_list():
return read(SubscriptionList, [XML_NO_HEAD])
def test_read_none_text():
doc = read(CodecTestDoc, [b'<codec-test attr="1,2"></codec-test>'])
assert doc.attr == (1, 2)
assert doc.text is None
def test_read_none_attribute():
doc = read(CodecTestDoc, [b'<codec-test><text>1,2</text></codec-test>'])
assert doc.attr is None
assert doc.text == (1, 2)
def test_content_codec():
doc = ContentCodecTestDoc(c=(5, 6))
tree = fromstringlist(write(doc, as_bytes=True))
assert tree.text == '5,6'
doc2 = read(ContentCodecTestDoc, etree_tobyteslist(tree))
assert doc2.c == (5, 6)
def test_text_codec():
doc = CodecTestDoc(text=(3, 4))
tree = fromstringlist(write(doc))
assert tree.find('text').text == '3,4'
doc2 = read(CodecTestDoc, etree_tobyteslist(tree))
assert doc2.text == (3, 4)
def test_read_none_attribute():
doc = read(CodecTestDoc, [b'<codec-test><text>1,2</text></codec-test>'])
assert doc.attr is None
assert doc.text == (1, 2)
def test_read_none_text():
doc = read(CodecTestDoc, [b'<codec-test attr="1,2"></codec-test>'])
assert doc.attr == (1, 2)
assert doc.text is None
def fx_subscription_list():
return read(SubscriptionList, [XML])
def fx_headless_subscription_list():
return read(SubscriptionList, [XML_NO_HEAD])
def fx_recursive_subscription_list():
return read(SubscriptionList, [XML_RECURSIVE])
def test_element_list_consume_buffer_regression():
xml = [b'<a><b><c></c><c><d>content', b'</d></c><c></c></b><b></b></a>']
doc = read(ELConsumeBufferRegressionTestDoc, xml)
assert len(doc.b) == 2
b = doc.b[0]
assert len(b.c) == 3
def fx_duplicated_subscription_list():
return read(SubscriptionList, [XML_DUPLICATION])
def fx_categorized_subscription_list():
return read(SubscriptionList, [XML_CATEGORY])
def test_atom_parser():
url = 'http://vio.atomtest.com/feed/atom'
crawled_feed, _ = parse_atom(atom_xml, url)
feed = read(Feed, write(crawled_feed, as_bytes=True))
title = crawled_feed.title
assert title.type == feed.title.type
assert title.value == feed.title.value
subtitle = crawled_feed.subtitle
assert subtitle.type == feed.subtitle.type
assert subtitle.value == feed.subtitle.value
links = crawled_feed.links
assert links[0].relation == feed.links[0].relation
assert links[0].mimetype == feed.links[0].mimetype
assert links[0].uri == feed.links[0].uri
assert links[1].relation == feed.links[1].relation
assert links[1].mimetype == feed.links[1].mimetype
assert links[1].uri == feed.links[1].uri
authors = crawled_feed.authors
assert authors[0].name == feed.authors[0].name
assert authors[0].email == feed.authors[0].email
categories = crawled_feed.categories
assert categories[0].term == feed.categories[0].term
contributors = crawled_feed.contributors
assert contributors[0].name == feed.contributors[0].name
generator = crawled_feed.generator
assert generator.uri == feed.generator.uri
assert generator.value == feed.generator.value
icon = crawled_feed.icon
assert icon == feed.icon
logo = crawled_feed.logo
assert logo == feed.logo
rights = crawled_feed.rights
assert rights.type == feed.rights.type
assert rights.value == feed.rights.value
updated_at = crawled_feed.updated_at
assert updated_at == feed.updated_at
entries = crawled_feed.entries
assert entries[0].id == feed.entries[0].id
assert entries[0].authors[0].name == feed.entries[0].authors[0].name
assert entries[0].title.type == feed.entries[0].title.type
assert entries[0].title.value == feed.entries[0].title.value
assert entries[0].links[0].relation == feed.entries[0].links[0].relation
assert entries[0].links[0].uri == feed.entries[0].links[0].uri
assert entries[0].updated_at == feed.entries[0].updated_at
assert entries[0].published_at == feed.entries[0].published_at
assert entries[0].categories[0].scheme_uri == \
feed.entries[0].categories[0].scheme_uri
assert entries[0].categories[0].term == feed.entries[0].categories[0].term
assert entries[0].categories[1].scheme_uri == \
feed.entries[0].categories[1].scheme_uri
assert entries[0].categories[1].term == feed.entries[0].categories[1].term
assert entries[0].content.type == feed.entries[0].content.type
assert entries[0].content.value == feed.entries[0].content.value
assert entries[0].content.source_uri == feed.entries[0].content.source_uri
assert entries[0].summary.type == feed.entries[0].summary.type
assert entries[0].summary.value == feed.entries[0].summary.value
assert entries[1].id == feed.entries[1].id
assert entries[1].authors[0].name == feed.entries[1].authors[0].name
assert entries[1].title.type == feed.entries[1].title.type
assert entries[1].title.value == feed.entries[1].title.value
assert entries[1].updated_at == feed.entries[1].updated_at
assert entries[2].id == feed.entries[2].id
assert entries[2].authors[0].name == feed.entries[2].authors[0].name
assert entries[2].title.type == feed.entries[2].title.type
assert entries[2].title.value == feed.entries[2].title.value
assert entries[2].updated_at == feed.entries[2].updated_at
source = entries[2].source
feed_source = feed.entries[2].source
assert source.authors[0].name == feed_source.authors[0].name
assert source.categories[0].term == feed_source.categories[0].term
assert source.contributors[0].name == feed_source.contributors[0].name
assert source.links[0] == feed_source.links[0]
assert source.id == feed_source.id
assert source.generator == feed_source.generator
assert source.icon == feed_source.icon
assert source.logo == feed_source.logo
assert source.rights == feed_source.rights
assert source.subtitle == feed_source.subtitle
def test_attribute_codec():
doc = CodecTestDoc(attr=(1, 2))
tree = fromstringlist(write(doc))
assert tree.attrib['attr'] == '1,2'
doc2 = read(CodecTestDoc, etree_tobyteslist(tree))
assert doc2.attr == (1, 2)
def fx_duplicated_subscription_list():
return read(SubscriptionList, [XML_DUPLICATION])
def test_content_codec():
doc = ContentCodecTestDoc(c=(5, 6))
tree = fromstringlist(write(doc, as_bytes=True))
assert tree.text == '5,6'
doc2 = read(ContentCodecTestDoc, etree_tobyteslist(tree))
assert doc2.c == (5, 6)
def fx_recursive_subscription_list():
return read(SubscriptionList, [XML_RECURSIVE])