def test_basic_dispatch(self): rules = PatternRules() f = Mock() rules.add_rule(r'http://my\.test\.com/', f) rules.dispatch('http://my.test.com/') f.assert_any_calls()
def test_dispatch_with_kwargs(self): rules = PatternRules() f = Mock() rules.add_rule(r'^http://my\.test\.com/(?P<title>[a-zA-Z]+)/([0-9]{4})/$', f) rules.dispatch('http://my.test.com/foo/0042/') f.assert_called_with('0042', title='foo')
def test_adding_multiple_rules(self): rules = PatternRules() rules.add_rules(( rule(r'[a-z]', lambda: True), rule(r'[0-9]', lambda: True) )) self.assertEqual(2, len(rules))
def test_dispatch_return(self): rules = PatternRules() def factory(): return ('foo', 'bar') rules.add_rule(r'[a-z]+', factory) result = rules.dispatch('foo') self.assertEqual(('foo', 'bar'), result)
def test_inject_context(self): rules = PatternRules() f = Mock() rules.add_rule(r'[a-z]+/([0-9]+)', f, inject_context=True) rules.dispatch('foo/0042') f.assert_called_once_with({ 'input': 'foo/0042', 'pattern': r'[a-z]+/([0-9]+)', 'args': ('0042',), 'kwargs': {} }, '0042')
def test_dispatch_context_processor(self): rules = PatternRules() value = None def processor(ctx): ctx['test_processor'] = 'foobar' def handler(ctx): nonlocal value value = ctx['test_processor'] rules.add_rule(r'[a-z]+', handler, inject_context=True) rules.dispatch('foo', context_processor=processor) self.assertEqual(value, 'foobar')
def __init__(self, config): self.config = config self.rules = PatternRules() self.logger = self.create_logger('art-dl') self.rules.add_rules(rules)
def test_unhandled_exception(self): rules = PatternRules() rules.add_rule(r'[a-z]', lambda: True) with self.assertRaises(RuleException): rules.dispatch('123')
def test_adding_rules(self): rules = PatternRules() rules.add_rule(r'[a-z]', lambda: True) self.assertEqual(1, len(rules))
class Application(object): def __init__(self, config): self.config = config self.rules = PatternRules() self.logger = self.create_logger('art-dl') self.rules.add_rules(rules) @staticmethod def _create_context_processor(http_client, logger, output_directory, overwrite): def context_processor(ctx): ctx['http_client'] = http_client ctx['output_directory'] = output_directory ctx['logger'] = logger ctx['overwrite'] = overwrite return context_processor @staticmethod def create_logger(name): logger = logging.getLogger(name) logger.setLevel(logging.DEBUG) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) fmt = logging.Formatter( '%(asctime)s (%(name)s) [%(levelname)s] %(message)s') ch.setFormatter(fmt) logger.addHandler(ch) return logger @staticmethod def log_config(logger, config): s = ['Config:\n'] for k, v in vars(config).items(): s.append('\t') s.append(k) s.append(': ') s.append(str(v)) s.append('\n') logger.debug(''.join(s)) def run(self): # TODO: Better logging if self.config.debug: logging.basicConfig(level=logging.DEBUG) loop = get_event_loop() self.log_config(self.logger, self.config) loop.set_debug(self.config.debug) client = ThrottledClient(loop, self.config.concurrent) tasks = None try: # TODO: Needs readability scrapers = [self.rules.dispatch( gallery, context_processor=self._create_context_processor( client, self.create_logger(gallery), self.config.output_directory, self.config.overwrite) ) for gallery in self.config.galleries] tasks = asyncio.gather(*(s.run() for s in scrapers)) loop.run_until_complete(tasks) except KeyboardInterrupt: self.logger.info('Shutting down...') self.logger.info('Cancelling Tasks') all_tasks = asyncio.gather(*asyncio.Task.all_tasks()) all_tasks.cancel() self.logger.info('Restarting loop') loop.run_forever() if tasks: tasks.exception() # Avoid warning for not fetching Exceptions all_tasks.exception() self.logger.info('Done') finally: client.close() loop.close()