def __init__(self, datadir, spider_cls=None, settings=None, **kwargs):
self.spider_cls = load_object(spider_cls) if spider_cls else IblSpider
self._specs = open_project_from_dir(datadir)
settings = settings.copy()
settings.frozen = False
settings.set('PLUGINS', load_plugins(settings))
self.settings = settings
def __init__(self, datadir, spider_cls=None, settings=None, **kwargs):
self.spider_cls = load_object(spider_cls) if spider_cls else IblSpider
self._specs = open_project_from_dir(datadir)
settings = settings.copy()
settings.frozen = False
settings.set('PLUGINS', load_plugins(settings))
self.settings = settings
def __init__(self, datadir, spider_cls=None, settings=None, **kwargs):
logging.info('Slybot %s Spider', slybot.__version__)
if settings is None:
settings = get_project_settings()
self.spider_cls = load_object(spider_cls) if spider_cls else IblSpider
self._specs = open_project_from_dir(datadir)
settings = settings.copy()
settings.frozen = False
settings.set('LOADED_PLUGINS', load_plugins(settings))
self.settings = settings
def __init__(self, datadir, spider_cls=None, settings=None, **kwargs):
logging.info('Slybot %s Spider', slybot.__version__)
if settings is None:
settings = get_project_settings()
self.spider_cls = load_object(spider_cls) if spider_cls else IblSpider
self._specs = open_project_from_dir(datadir)
settings = settings.copy()
settings.frozen = False
settings.set('LOADED_PLUGINS', load_plugins(settings))
self.settings = settings
def __init__(self, datadir, spider_cls=None, settings=None, **kwargs):
logging.info('Slybot %s Spider', slybot.__version__)
if is_zipfile(datadir):
tempdir = tempfile.mkdtemp(prefix='slybot-')
ZipFile(datadir).extractall(tempdir)
atexit.register(shutil.rmtree, tempdir)
datadir = tempdir
if settings is None:
settings = get_project_settings()
self.spider_cls = load_object(spider_cls) if spider_cls else IblSpider
self._specs = open_project_from_dir(datadir)
settings = settings.copy()
settings.frozen = False
settings.set('LOADED_PLUGINS', load_plugins(settings))
self.settings = settings
def __init__(self, datadir, spider_cls=None, settings=None, **kwargs):
logging.info('Slybot %s Spider', slybot.__version__)
if is_zipfile(datadir):
tempdir = tempfile.mkdtemp(prefix='slybot-')
ZipFile(datadir).extractall(tempdir)
atexit.register(shutil.rmtree, tempdir)
datadir = tempdir
if settings is None:
settings = get_project_settings()
self.spider_cls = load_object(spider_cls) if spider_cls else IblSpider
self._specs = open_project_from_dir(datadir)
settings = settings.copy()
settings.frozen = False
settings.set('LOADED_PLUGINS', load_plugins(settings))
self.settings = settings
def load_slybot_spec(self, project):
"""load the spec for a given project"""
return open_project_from_dir(self.projectdir)
def test_test_project(self):
specs = open_project_from_dir(_TEST_PROJECT_DIR)
self.assertTrue(validate_project_schema(specs))
def __init__(self, datadir, spider_cls=None):
self.spider_cls = load_object(spider_cls) if spider_cls else IblSpider
self._specs = open_project_from_dir(datadir)
def load_slybot_spec(self, project):
"""load the spec for a given project"""
return open_project_from_dir(self.projectdir)
def __init__(self, datadir, spider_cls=None):
self.spider_cls = load_object(spider_cls) if spider_cls else IblSpider
self._specs = open_project_from_dir(datadir)
