def test_basic_dispatch(self):
rules = PatternRules()
f = Mock()
rules.add_rule(r'http://my\.test\.com/', f)
rules.dispatch('http://my.test.com/')
f.assert_any_calls()
def test_dispatch_with_kwargs(self):
rules = PatternRules()
f = Mock()
rules.add_rule(r'^http://my\.test\.com/(?P<title>[a-zA-Z]+)/([0-9]{4})/$', f)
rules.dispatch('http://my.test.com/foo/0042/')
f.assert_called_with('0042', title='foo')
def test_adding_multiple_rules(self):
rules = PatternRules()
rules.add_rules((
rule(r'[a-z]', lambda: True),
rule(r'[0-9]', lambda: True)
))
self.assertEqual(2, len(rules))
def test_dispatch_return(self):
rules = PatternRules()
def factory():
return ('foo', 'bar')
rules.add_rule(r'[a-z]+', factory)
result = rules.dispatch('foo')
self.assertEqual(('foo', 'bar'), result)
def test_inject_context(self):
rules = PatternRules()
f = Mock()
rules.add_rule(r'[a-z]+/([0-9]+)', f, inject_context=True)
rules.dispatch('foo/0042')
f.assert_called_once_with({
'input': 'foo/0042',
'pattern': r'[a-z]+/([0-9]+)',
'args': ('0042',),
'kwargs': {}
}, '0042')
def test_dispatch_context_processor(self):
rules = PatternRules()
value = None
def processor(ctx):
ctx['test_processor'] = 'foobar'
def handler(ctx):
nonlocal value
value = ctx['test_processor']
rules.add_rule(r'[a-z]+', handler, inject_context=True)
rules.dispatch('foo', context_processor=processor)
self.assertEqual(value, 'foobar')
def __init__(self, config):
self.config = config
self.rules = PatternRules()
self.logger = self.create_logger('art-dl')
self.rules.add_rules(rules)
def test_unhandled_exception(self):
rules = PatternRules()
rules.add_rule(r'[a-z]', lambda: True)
with self.assertRaises(RuleException):
rules.dispatch('123')
def test_adding_rules(self):
rules = PatternRules()
rules.add_rule(r'[a-z]', lambda: True)
self.assertEqual(1, len(rules))
class Application(object):
def __init__(self, config):
self.config = config
self.rules = PatternRules()
self.logger = self.create_logger('art-dl')
self.rules.add_rules(rules)
@staticmethod
def _create_context_processor(http_client, logger,
output_directory, overwrite):
def context_processor(ctx):
ctx['http_client'] = http_client
ctx['output_directory'] = output_directory
ctx['logger'] = logger
ctx['overwrite'] = overwrite
return context_processor
@staticmethod
def create_logger(name):
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
fmt = logging.Formatter(
'%(asctime)s (%(name)s) [%(levelname)s] %(message)s')
ch.setFormatter(fmt)
logger.addHandler(ch)
return logger
@staticmethod
def log_config(logger, config):
s = ['Config:\n']
for k, v in vars(config).items():
s.append('\t')
s.append(k)
s.append(': ')
s.append(str(v))
s.append('\n')
logger.debug(''.join(s))
def run(self):
# TODO: Better logging
if self.config.debug:
logging.basicConfig(level=logging.DEBUG)
loop = get_event_loop()
self.log_config(self.logger, self.config)
loop.set_debug(self.config.debug)
client = ThrottledClient(loop, self.config.concurrent)
tasks = None
try:
# TODO: Needs readability
scrapers = [self.rules.dispatch(
gallery,
context_processor=self._create_context_processor(
client, self.create_logger(gallery),
self.config.output_directory,
self.config.overwrite)
)
for gallery in self.config.galleries]
tasks = asyncio.gather(*(s.run() for s in scrapers))
loop.run_until_complete(tasks)
except KeyboardInterrupt:
self.logger.info('Shutting down...')
self.logger.info('Cancelling Tasks')
all_tasks = asyncio.gather(*asyncio.Task.all_tasks())
all_tasks.cancel()
self.logger.info('Restarting loop')
loop.run_forever()
if tasks:
tasks.exception() # Avoid warning for not fetching Exceptions
all_tasks.exception()
self.logger.info('Done')
finally:
client.close()
loop.close()
