class JobDescription(object):
def __init__(self, name, url_patterns, opener_cls, user_conf, starts,
unit_cls=None, login_hook=None, error_handler=None,
finish_callback=None, **kw):
self.name = name
if not JOB_NAME_RE.match(name):
raise ConfigurationError('Job name can only contain alphabet, number and space.')
self.uniq_name = self._get_uniq_name(self.name)
self.url_patterns = url_patterns
self.opener_cls = opener_cls
self.user_conf = user_conf
self.starts = starts
self.login_hook = login_hook
self.error_handler = error_handler
self.finish_callback = finish_callback
self.settings = Settings(user_conf=user_conf, **kw)
self.unit_cls = unit_cls or \
(Bundle if self.settings.job.mode == 'bundle' else Url)
def _get_uniq_name(self, name):
hash_val = hashlib.md5(name).hexdigest()[8:-8]
return base58_encode(int(hash_val, 16))
def add_urlpattern(self, url_pattern):
self.url_patterns += url_pattern
def update_settings(self, settings):
self.settings.update(settings)
class JobDescription(object):
def __init__(self,
name,
url_patterns,
opener_cls,
user_conf,
starts,
unit_cls=None,
login_hook=None,
error_handler=None,
finish_callback=None,
**kw):
self.name = name
if not JOB_NAME_RE.match(name):
raise ConfigurationError(
'Job name can only contain alphabet, number and space.')
self.uniq_name = self._get_uniq_name(self.name)
self.url_patterns = url_patterns
self.opener_cls = opener_cls
self.user_conf = user_conf
self.starts = starts
self.login_hook = login_hook
self.error_handler = error_handler
self.finish_callback = finish_callback
self.settings = Settings(user_conf=user_conf, **kw)
self.unit_cls = unit_cls or \
(Bundle if self.settings.job.mode == 'bundle' else Url)
def _get_uniq_name(self, name):
hash_val = hashlib.md5(name).hexdigest()[8:-8]
return base58_encode(int(hash_val, 16))
def add_urlpattern(self, url_pattern):
self.url_patterns += url_pattern
def update_settings(self, settings):
self.settings.update(settings)
