def __init__(self, datadir, spider_cls=None, settings=None, **kwargs): self.spider_cls = load_object(spider_cls) if spider_cls else IblSpider self._specs = open_project_from_dir(datadir) settings = settings.copy() settings.frozen = False settings.set('PLUGINS', load_plugins(settings)) self.settings = settings
def __init__(self, datadir, spider_cls=None, settings=None, **kwargs): logging.info('Slybot %s Spider', slybot.__version__) if settings is None: settings = get_project_settings() self.spider_cls = load_object(spider_cls) if spider_cls else IblSpider self._specs = open_project_from_dir(datadir) settings = settings.copy() settings.frozen = False settings.set('LOADED_PLUGINS', load_plugins(settings)) self.settings = settings
def __init__(self, datadir, spider_cls=None, settings=None, **kwargs): logging.info('Slybot %s Spider', slybot.__version__) if is_zipfile(datadir): tempdir = tempfile.mkdtemp(prefix='slybot-') ZipFile(datadir).extractall(tempdir) atexit.register(shutil.rmtree, tempdir) datadir = tempdir if settings is None: settings = get_project_settings() self.spider_cls = load_object(spider_cls) if spider_cls else IblSpider self._specs = open_project_from_dir(datadir) settings = settings.copy() settings.frozen = False settings.set('LOADED_PLUGINS', load_plugins(settings)) self.settings = settings
def load_slybot_spec(self, project): """load the spec for a given project""" return open_project_from_dir(self.projectdir)
def test_test_project(self): specs = open_project_from_dir(_TEST_PROJECT_DIR) self.assertTrue(validate_project_schema(specs))
def __init__(self, datadir, spider_cls=None): self.spider_cls = load_object(spider_cls) if spider_cls else IblSpider self._specs = open_project_from_dir(datadir)