async def test_from_table_to_attr(self):
xml_engine = PyQueryAdapter()
collector = any()
html = xml_engine(
'<table><tr><td>1</td><td>2</td></tr><tr><td>3</td><td>4</td></tr></table>'
)
_, result = await collector(None, html)
self.assertEqual(result, html)
async def test_obj_collector(self):
xml_node = PyQueryAdapter()(page)
collector = obj_opengraph()
_, result = await collector(None, xml_node)
expected = OpenGraph(
title='OpenGraph Title',
description='OpenGraph Description',
)
self.assertEqual(result, expected)
async def test_auto_pipe(self):
xml_engine = PyQueryAdapter()
collector = pipe(
css('tr td:nth-child(2)'),
foreach(as_text(), fn(lambda it: str(it))),
)
html = xml_engine(raw_html)
expected = ['2', '4']
_, result = await collector(None, html)
self.assertEqual(expected, list(result))
async def test_tr_with_css(self):
xml_engine = PyQueryAdapter()
collector = pipe(
xpath('//tr/td[position()=2]'),
foreach(as_text()),
)
html = xml_engine(raw_html)
expected = ['2', '4']
_, result = await collector(None, html)
self.assertEqual(expected, list(result))
async def test_get_link_text(self):
xml_engine = PyQueryAdapter()
collector = pipe(
css('ul > li > a'),
foreach(text()),
)
html = xml_engine(raw_html)
expected = [['Fuu 1'], ['Fuu 2']]
_, result = await collector(None, html)
self.assertEqual(expected, list(result))
async def test_filter(self):
xml_engine = PyQueryAdapter()
collector = pipe(
css('tr td:nth-child(2)'),
text(),
filter(lambda it: int(it) > 2),
)
html = xml_engine(raw_html)
expected = ['4']
_, result = await collector(None, html)
self.assertEqual(expected, list(result))
async def test_get_link_or_text(self):
xml_engine = PyQueryAdapter()
collector = pipe(css('ul > li > a'),
foreach(default(
attr('href'),
text(),
)))
html = xml_engine(raw_html)
expected = [['//fuu'], ['unknown']]
_, result = await collector(None, html)
self.assertEqual(expected, list(result))
async def test_tr_with_xpath(self):
xml_engine = PyQueryAdapter()
collector = pipe(
css('tr td:nth-child(2)'),
fn(AsyncIterableAdapter),
aforeach(as_text()),
)
html = xml_engine(raw_html)
expected = ['2', '4']
_, result = await collector(None, html)
self.assertEqual(expected, list(result))
async def test_limit_with_empty_items(self):
xml_engine = PyQueryAdapter()
collector = pipe(
const([]),
foreach(
as_text(),
limit=None,
),
)
html = xml_engine(raw_html)
expected = []
_, result = await collector(None, html)
self.assertEqual(expected, list(result))
async def test_limit(self):
xml_engine = PyQueryAdapter()
collector = pipe(
css('tr td'),
foreach(
as_text(),
limit=1,
),
)
html = xml_engine(raw_html)
expected = ['1', '2', '3', '4']
_, result = await collector(None, html)
self.assertEqual(expected, list(result))
async def test_dict_collector(self):
xml_node = PyQueryAdapter()(page)
collector = dict_opengraph
_, result = await collector(None, xml_node)
expected = {
'title': 'OpenGraph Title',
'type': None,
'locale': None,
'description': "OpenGraph Description",
'url': None,
'site_name': None,
'image': None,
'audio': None,
'video': None,
'app_id': None,
}
self.assertDictEqual(result, expected)
def setUpClass(cls):
cls.xml_engine = PyQueryAdapter()