def test_extract_sample_posts_info():
with patch.multiple('site_crawler.crawler',
extract_posts_link=DEFAULT,
PostPageParser=DEFAULT,
get_soup_from_url=DEFAULT,
extract_sample_category_from_site=DEFAULT) as values:
extract_posts_link_mock = values['extract_posts_link']
get_soup_from_url_mock = values['get_soup_from_url']
PostPageParser_mock = values['PostPageParser']
soup1, soup2 = mock_BeautifulSoup(), mock_BeautifulSoup()
parser1, parser2 = mock_post_parser(), mock_post_parser()
extract_posts_link_mock.return_value = ['post1', 'post2']
get_soup_from_url_mock.side_effect = lambda x: dict(post1=soup1,
post2=soup2)[x]
PostPageParser_mock.parser_factory.side_effect = lambda x: parser1 if soup1 == x else parser2
parser1.jsonify.return_value = 'json1'
parser2.jsonify.return_value = 'json2'
result = list(extract_sample_posts_info(3))
extract_posts_link_mock.assert_called_once_with(3)
get_soup_from_url_mock.assert_any_call('post1')
get_soup_from_url_mock.assert_any_call('post2')
PostPageParser_mock.parser_factory.assert_any_call(soup1)
PostPageParser_mock.parser_factory.assert_any_call(soup2)
assert get_soup_from_url_mock.call_count == 2
assert PostPageParser_mock.parser_factory.call_count == 2
assert result == ['json1', 'json2']
def test_extract_sample_posts_info():
with patch.multiple('site_crawler.crawler',
extract_posts_link=DEFAULT,
PostPageParser=DEFAULT,
get_soup_from_url=DEFAULT,
extract_sample_category_from_site=DEFAULT) as values:
extract_posts_link_mock = values['extract_posts_link']
get_soup_from_url_mock = values['get_soup_from_url']
PostPageParser_mock = values['PostPageParser']
soup1, soup2 = mock_BeautifulSoup(), mock_BeautifulSoup()
parser1, parser2 = mock_post_parser(), mock_post_parser()
extract_posts_link_mock.return_value = ['post1', 'post2']
get_soup_from_url_mock.side_effect = lambda x: dict(post1=soup1, post2=soup2)[x]
PostPageParser_mock.parser_factory.side_effect = lambda x: parser1 if soup1 == x else parser2
parser1.jsonify.return_value = 'json1'
parser2.jsonify.return_value = 'json2'
result = list(extract_sample_posts_info(3))
extract_posts_link_mock.assert_called_once_with(3)
get_soup_from_url_mock.assert_any_call('post1')
get_soup_from_url_mock.assert_any_call('post2')
PostPageParser_mock.parser_factory.assert_any_call(soup1)
PostPageParser_mock.parser_factory.assert_any_call(soup2)
assert get_soup_from_url_mock.call_count == 2
assert PostPageParser_mock.parser_factory.call_count == 2
assert result == ['json1', 'json2']
def test_category_parser_from_url():
with patch.multiple('site_crawler.crawler', get_soup_from_url=DEFAULT, CategoryPageParser=DEFAULT) as values:
get_soup_from_url_mock = values['get_soup_from_url']
CategoryPageParser_mock = values['CategoryPageParser']
get_soup_from_url_mock .return_value = soup = mock_BeautifulSoup()
CategoryPageParser_mock.return_value = parser = Mock(CategoryPageParser)
result = category_parser_from_url('category url')
get_soup_from_url_mock.assert_called_once_with('category url')
CategoryPageParser_mock.assert_called_once_with(soup)
assert result == parser
def test_get_soup_from_url():
with patch.multiple('site_crawler.crawler', get_url_content=DEFAULT, build_beautiful_soup=DEFAULT) as values:
get_url_content_mock = values['get_url_content']
build_beautiful_soup_mock = values['build_beautiful_soup']
get_url_content_mock.return_value = 'sample html'
build_beautiful_soup_mock.return_value = soup = mock_BeautifulSoup()
result = get_soup_from_url('sample url')
get_url_content_mock.assert_called_once_with('sample url')
build_beautiful_soup_mock.assert_called_once_with('sample html')
assert result == soup
def test_get_soup_from_url():
with patch.multiple('site_crawler.crawler',
get_url_content=DEFAULT,
build_beautiful_soup=DEFAULT) as values:
get_url_content_mock = values['get_url_content']
build_beautiful_soup_mock = values['build_beautiful_soup']
get_url_content_mock.return_value = 'sample html'
build_beautiful_soup_mock.return_value = soup = mock_BeautifulSoup()
result = get_soup_from_url('sample url')
get_url_content_mock.assert_called_once_with('sample url')
build_beautiful_soup_mock.assert_called_once_with('sample html')
assert result == soup
def test_extract_sample_category_from_site():
with patch.multiple('site_crawler.crawler',
get_soup_from_url=DEFAULT,
pick_random_category=DEFAULT) as values:
get_soup_from_url_mock = values['get_soup_from_url']
pick_random_category_mock = values['pick_random_category']
get_soup_from_url_mock.return_value = soup = mock_BeautifulSoup()
pick_random_category_mock.return_value = sample = 'http://.../cat/1'
result = extract_sample_category_from_site()
get_soup_from_url_mock.assert_called_once_with('http://p30download.com/')
pick_random_category_mock.assert_called_once_with(soup)
assert result == sample
def test_get_categories():
soup = mock_BeautifulSoup()
with patch.multiple('site_parser.index_parser',
get_base_categories_lists=DEFAULT,
extract_categories=DEFAULT) as values:
get_base_categories_lists_mock = values['get_base_categories_lists']
extract_categories_mock = values['extract_categories']
get_base_categories_lists_mock.return_value = cats = map(mock_link_tag, ['a', 'b'])
extract_categories_mock.return_value = ['a', 'b']
result = get_categories(soup)
get_base_categories_lists_mock.assert_called_once_with(soup)
extract_categories_mock.assert_called_once_with(cats)
assert result == ['a', 'b']
def test_extract_sample_category_from_site():
with patch.multiple('site_crawler.crawler',
get_soup_from_url=DEFAULT,
pick_random_category=DEFAULT) as values:
get_soup_from_url_mock = values['get_soup_from_url']
pick_random_category_mock = values['pick_random_category']
get_soup_from_url_mock.return_value = soup = mock_BeautifulSoup()
pick_random_category_mock.return_value = sample = 'http://.../cat/1'
result = extract_sample_category_from_site()
get_soup_from_url_mock.assert_called_once_with(
'http://p30download.com/')
pick_random_category_mock.assert_called_once_with(soup)
assert result == sample
def test_category_parser_from_url():
with patch.multiple('site_crawler.crawler',
get_soup_from_url=DEFAULT,
CategoryPageParser=DEFAULT) as values:
get_soup_from_url_mock = values['get_soup_from_url']
CategoryPageParser_mock = values['CategoryPageParser']
get_soup_from_url_mock.return_value = soup = mock_BeautifulSoup()
CategoryPageParser_mock.return_value = parser = Mock(
CategoryPageParser)
result = category_parser_from_url('category url')
get_soup_from_url_mock.assert_called_once_with('category url')
CategoryPageParser_mock.assert_called_once_with(soup)
assert result == parser
