def test_on_interrupt_is_not_called_without_interrupt(self):
myCrawler = DomainCrawler()
callback = CallableMock()
myCrawler.on_interrupt(callback)
self.assertEqual(callback.call_count(), 0)
myCrawler.crawl("tiggers")
self.assertEqual(callback.call_count(), 0)
def test_calls_callback_on_finish(self):
crawler = DomainCrawler()
callback = CallableMock()
crawler.on_finish(callback)
self.assertEqual(callback.call_count(), 0)
crawler.crawl("tiggers")
self.assertEqual(callback.call_count(), 1)
def test_worker_thread_calls_argument(self):
try:
with ThreadJoiner(0.5):
service = WorkService()
requested_work = CallableMock()
service.request_work(requested_work)
except RuntimeError:
pass
self.assertEqual(requested_work.call_count(), 1)
def test_calls_callback_on_interrupt(self):
parser.parse.raises(KeyboardInterrupt())
my_crawler = DomainCrawler()
callback = CallableMock()
my_crawler.on_interrupt(callback)
self.assertEqual(callback.call_count(), 0)
with self.assertRaises(KeyboardInterrupt):
my_crawler.crawl("tiggers")
self.assertEqual(callback.call_count(), 1)
def test_does_not_call_unsubscribed_method_again(self):
callback1 = CallableMock()
callback2 = CallableMock()
self.subject(callback1)
observable2 = self.subject(callback2)
self.assertEqual(callback1.call_count(), 0)
self.assertEqual(callback2.call_count(), 0)
self.subject.next()
self.assertEqual(callback1.call_count(), 1)
self.assertEqual(callback2.call_count(), 1)
observable2.unsubscribe()
for i in range(2, 25):
self.subject.next()
self.assertEqual(callback1.call_count(), i)
self.assertEqual(callback2.call_count(), 1)
def setUp(self):
Inject.reset()
self.startMock = CallableMock()
threading_mock.Thread.reset(returns=Any(daemon=False, start=self.startMock))
Inject.register_feature("threading", threading_mock)
Inject.register_feature("config_service", config_mock)
Inject.register_feature("queue", lambda: queue_mock)
Inject.register_feature("log_service", log_service_mock)
def test_calls_all_callbacks_on_next(self):
callback1 = CallableMock()
callback2 = CallableMock()
self.subject(callback1)
self.subject(callback2)
self.assertEqual(callback1.call_count(), 0)
self.assertEqual(callback2.call_count(), 0)
self.subject.next()
self.assertEqual(callback1.call_count(), 1)
self.assertEqual(callback2.call_count(), 1)
def test_calls_all_callbacks_every_time(self):
callback1 = CallableMock()
callback2 = CallableMock()
self.subject(callback1)
self.subject(callback2)
self.assertEqual(callback1.call_count(), 0)
self.assertEqual(callback2.call_count(), 0)
for i in range(1, 25):
self.subject.next()
self.assertEqual(callback1.call_count(), i)
self.assertEqual(callback2.call_count(), i)
def setUp(self):
Inject.reset()
Inject.register_feature('site_parser_factory',
lambda: Any(parse=CallableMock()))
Inject.register_feature(
'log_service',
Any(info=CallableMock(),
error=CallableMock(),
debug=CallableMock()))
Inject.register_feature(
'work_service',
Any(request_work=CallableMock(callback=lambda cb: cb()),
terminate_all=CallableMock(),
active_count=CallableMock))
def setUp(self):
self.log_service = Any(debug=CallableMock(), error=CallableMock())
self.crawl_result = Any(link="",
status_code=200,
page_size=0,
links=[])
self.parser = Any(parse=CallableMock(returns=self.crawl_result))
self.crawler = Any(
status=Any(urls_in_progress=[],
urls_to_crawl=[],
visited_urls=set()),
site_parser_factory=CallableMock(returns=self.parser),
process_links=CallableMock(),
get_status_message=CallableMock())
Inject.reset()
Inject.register_feature('log_service', self.log_service)
import unittest
import threading
from tl.testing.thread import ThreadJoiner, ThreadAwareTestCase
from lollygag.services.work.work_service import WorkService, get_labels
from lollygag.dependency_injection.inject import Inject
from lollygag.utility.test_utils import Any, CallableMock
try:
import Queue
except ImportError:
import queue as Queue
threading_mock = Any(Thread=CallableMock())
config_mock = Any(threads=1)
queue_mock = Any(put=CallableMock(), get=CallableMock(), task_done=CallableMock(), join=CallableMock())
log_service_mock = Any(debug=CallableMock(), info=CallableMock())
class WorkServiceTests(unittest.TestCase):
def setUp(self):
Inject.reset()
self.startMock = CallableMock()
threading_mock.Thread.reset(returns=Any(daemon=False, start=self.startMock))
Inject.register_feature("threading", threading_mock)
Inject.register_feature("config_service", config_mock)
Inject.register_feature("queue", lambda: queue_mock)
Inject.register_feature("log_service", log_service_mock)
def tearDown(self):
Inject.reset()
def __reset_threadmocks(self):
self.callback = lambda *a, **kw: None
self.start_mock = CallableMock(
callback=lambda *a, **kw: self.callback())
work_service.request_work.reset(callback=lambda cb: cb())
work_service.active_count.reset(returns=0)
import unittest
from lollygag.core.crawlers.domain_crawler import DomainCrawler
from lollygag.core.crawlers.jobs.crawl_job import CrawlJob
from lollygag.dependency_injection.inject import Inject
from lollygag.utility.test_utils import Any, CallableMock
crawl_result = Any(link="", status_code=200, page_size=0, links=[])
parser = Any(parse=CallableMock(returns=crawl_result))
site_parser_factory = lambda: parser
config = Any(threads=1, url=None, skip=[])
log = Any(write=CallableMock(),
info=CallableMock(),
error=CallableMock(),
debug=CallableMock())
work_service = Any(request_work=CallableMock(),
terminate_all=CallableMock(),
active_count=CallableMock())
class DomainCrawlerTests(unittest.TestCase):
def setUp(self):
Inject.reset()
Inject.register_feature("site_parser_factory", site_parser_factory)
Inject.register_feature("config_service", config)
Inject.register_feature("log_service", log)
Inject.register_feature("work_service", work_service)
def tearDown(self):
Inject.reset()
parser.parse.reset()
log.write.reset()
def setUp(self):
self.crawler = Any(on_start=CallableMock(),
on_finish=CallableMock(),
on_interrupt=CallableMock())
def test_crawl_calls_on_start(self):
crawler = Crawler()
callback = CallableMock()
crawler.on_start(callback)
crawler.crawl("foo")
self.assertEqual(callback.call_count(), 1)