def test_extract_posts_link():
with patch.multiple('site_crawler.crawler',
extract_post_links=DEFAULT,
extract_sample_category_from_site=DEFAULT) as values:
extract_post_links_mock = values['extract_post_links']
extract_sample_category_from_site_mock = values[
'extract_sample_category_from_site']
extract_sample_category_from_site_mock.return_value = 'cat1'
parser = mock_category_parser()
def extract_post_return_values(url, post_links):
if url == 'cat1':
assert post_links == []
post_links += ['post2', 'post3', 'post5']
return parser
elif url == 'page1':
assert post_links == ['post2', 'post3', 'post5']
post_links += ['post7', 'post11', 'post13']
extract_post_links_mock.side_effect = extract_post_return_values
parser.get_pagination_pages.return_value = ['page1', 'page2', 'page3']
result = extract_posts_link(5)
assert 2 == extract_post_links_mock.call_count
assert result == ['post2', 'post3', 'post5', 'post7', 'post11']
def test_extract_posts_link():
with patch.multiple('site_crawler.crawler',
extract_post_links=DEFAULT,
extract_sample_category_from_site=DEFAULT) as values:
extract_post_links_mock = values['extract_post_links']
extract_sample_category_from_site_mock = values['extract_sample_category_from_site']
extract_sample_category_from_site_mock.return_value = 'cat1'
parser = mock_category_parser()
def extract_post_return_values(url, post_links):
if url == 'cat1':
assert post_links == []
post_links += ['post2', 'post3', 'post5']
return parser
elif url == 'page1':
assert post_links == ['post2', 'post3', 'post5']
post_links += ['post7', 'post11', 'post13']
extract_post_links_mock.side_effect = extract_post_return_values
parser.get_pagination_pages.return_value = ['page1', 'page2', 'page3']
result = extract_posts_link(5)
assert 2 == extract_post_links_mock.call_count
assert result == ['post2', 'post3', 'post5', 'post7', 'post11']
def test_extract_post_links():
posts_link = [2, 3]
with patch.multiple('site_crawler.crawler', category_parser_from_url=DEFAULT, CategoryPageParser=DEFAULT) as values:
category_parser_from_url_mock = values['category_parser_from_url']
category_parser_from_url_mock.return_value = parser = mock_category_parser()
parser.get_post_links.return_value = [5, 7]
extract_post_links('category url', posts_link)
category_parser_from_url_mock.assert_called_once_with('category url')
parser.get_post_links.assert_called_once_with()
assert posts_link == [2, 3, 5, 7]
def test_extract_post_links():
posts_link = [2, 3]
with patch.multiple('site_crawler.crawler',
category_parser_from_url=DEFAULT,
CategoryPageParser=DEFAULT) as values:
category_parser_from_url_mock = values['category_parser_from_url']
category_parser_from_url_mock.return_value = parser = mock_category_parser(
)
parser.get_post_links.return_value = [5, 7]
extract_post_links('category url', posts_link)
category_parser_from_url_mock.assert_called_once_with('category url')
parser.get_post_links.assert_called_once_with()
assert posts_link == [2, 3, 5, 7]
