def test_cached_detail_fields(self):
scrape(self.search_url,
self.output,
limit=10,
get_details=True,
memcached='localhost:11211')
_test_fields(self.output, paths.DETAIL_FIELDS)
def test_get_all_results():
fp = tempfile.NamedTemporaryFile()
output = fp.name
fp.close()
scrape('https://www.etsy.com/il-en/search?q=bunny%20wizard%20hat', output)
line_count = len(open(output).readlines())
assert line_count > 1
def test_cached_detail_results(self):
scrape(self.search_url,
self.output,
limit=10,
get_details=True,
memcached='localhost:11211')
line_count = len(open(self.output).readlines())
assert line_count == 11
def setup(self):
self.search_url = get_search_url()
fp = tempfile.NamedTemporaryFile()
self.output = fp.name
fp.close()
scrape(self.search_url,
self.output,
limit=10,
get_details=True,
memcached='localhost:11211')
def test_fail_log_callback():
fp = tempfile.NamedTemporaryFile()
fail_log = fp.name
fp.close()
fail_log = mock.Mock()
scrape('https://www.ddddetsy.com/search?q=test',
limit=10,
fail_log_callback=fail_log)
fail_log.assert_called_with('https://www.ddddetsy.com/search?q=test',
mock.ANY)
def test_fail_log():
fp = tempfile.NamedTemporaryFile()
output = fp.name
fp.close()
fail_log = fp.name
fp.close()
scrape('https://www.notactuallyetsy.com/search?q=test',
output,
limit=10,
fail_log=fail_log)
line_count = len(open(fail_log).readlines())
assert line_count > 0
def test_progress_callback(self, search_url):
progress = mock.Mock()
scrape(search_url, limit=10, progress_callback=progress)
progress.assert_called()
def test_message_callback(self, search_url):
message = mock.Mock()
scrape(search_url, limit=10, message_callback=message)
message.assert_called_with('Scraped 10 products, failed to scrape 0.')
def setup(self):
fp = tempfile.NamedTemporaryFile()
self.output = fp.name
fp.close()
scrape(get_search_url(), self.output, limit=10)
def test_write_to_std_out(search_url):
# TODO Need to capture and test stdout
scrape(search_url, limit=10)