def __init__(self,
name,
url_patterns,
opener_cls,
user_conf,
starts,
unit_cls=None,
login_hook=None,
error_handler=None,
finish_callback=None,
**kw):
self.name = name
if not JOB_NAME_RE.match(name):
raise ConfigurationError(
'Job name can only contain alphabet, number and space.')
self.uniq_name = self._get_uniq_name(self.name)
self.url_patterns = url_patterns
self.opener_cls = opener_cls
self.user_conf = user_conf
self.starts = starts
self.login_hook = login_hook
self.error_handler = error_handler
self.finish_callback = finish_callback
self.settings = Settings(user_conf=user_conf, **kw)
self.unit_cls = unit_cls or \
(Bundle if self.settings.job.mode == 'bundle' else Url)
def __init__(self,
name,
url_patterns,
opener_cls,
starts,
is_bundle=False,
unit_cls=str,
instances=1,
debug=False,
user_conf=None,
login_hook=None):
self.name = name
self.real_name = self.name.replace(' ', '_')
if not JOB_NAME_RE.match(name):
raise ConfigurationError(
'Job name can only contain alphabet, number and space.')
self.url_patterns = url_patterns
self.opener_cls = opener_cls
self.starts = starts
self.is_bundle = is_bundle
self.unit_cls = unit_cls
self.instances = instances
self.debug = debug
self.user_conf = user_conf
self.login_hook = login_hook
self.context = Context(user_conf=user_conf)
def _login(self, opener):
if self.job.login_hook is not None:
if 'login' not in self.ctx.job or \
not isinstance(self.ctx.job.login, list):
raise ConfigurationError(
'If login_hook set, config files must contains `login`')
kw = random.choice(self.ctx.job.login)
login_success = self.job.login_hook(opener, **kw)
if not login_success:
self.logger.info('login fail')
return login_success
def _login(self, opener):
if self.job.login_hook is not None:
if 'login' not in self.ctx.job or \
not isinstance(self.ctx.job.login, list):
raise ConfigurationError(
'If login_hook set, config files must contains `login`')
kw = random.choice(self.ctx.job.login)
login_result = self.job.login_hook(opener, **kw)
if isinstance(login_result, tuple) and len(login_result) == 2:
self.logger.error('login fail, reason: %s' % login_result[1])
return login_result[0]
elif not login_result:
self.logger.error('login fail')
return login_result
return True