def test_worker(): p = Patu(urls=['www.djangoproject.com'], depth=1) for url, referer in p.next_urls.iteritems(): p.task_queue.put(url) p.task_queue.put('STOP') p.worker() content = p.done_queue.get().content with open(TEST_HTML) as f: eq_(f.read(), content)
def test_worker_input_file(): p = Patu(urls=['www.djangoproject.com'], depth=1, input_file=TEST_INPUT) for url, referer in p.next_urls.iteritems(): p.task_queue.put(url) p.task_queue.put('STOP') p.worker() p.done_queue.put('STOP') for u in iter(p.done_queue.get, 'STOP'): try: url = u.url except AttributeError: url = False assert url in SEEN_URLS or not url
def test_worker_statuses(): """ This is kind of wanking - just trying to get test coverage in the worker processes """ url_statuses = [ ('www.djangoproject.com/offsite_redirect', 200), ('error.me', 500), ('io.me', -1), ('keyboard.me', -1) ] for address, error_code in url_statuses: p = Patu(urls=[address], depth=1) for url, referer in p.next_urls.iteritems(): p.task_queue.put(url) p.task_queue.put('STOP') p.worker() u = p.done_queue.get() eq_(u.status_code, error_code)