def create_root(config):
from scrapy import log
from scrapy.settings import CrawlerSettings
from slyd.crawlerspec import (CrawlerSpecManager,
create_crawler_spec_resource)
from slyd.bot import create_bot_resource
import slyd.settings
from slyd.projects import ProjectsResource
root = Resource()
root.putChild("static", File(config['docroot']))
crawler_settings = CrawlerSettings(settings_module=slyd.settings)
spec_manager = CrawlerSpecManager(crawler_settings)
# add project management at /projects
projects = ProjectsResource(crawler_settings)
root.putChild('projects', projects)
# add crawler at /projects/PROJECT_ID/bot
log.msg("Slybot specs loading from %s/[PROJECT]" % spec_manager.basedir,
level=log.DEBUG)
projects.putChild("bot", create_bot_resource(spec_manager))
# add spec at /projects/PROJECT_ID/spec
spec = create_crawler_spec_resource(spec_manager)
projects.putChild("spec", spec)
return root
def create_root(config):
from scrapy import log
from scrapy.settings import Settings
from slyd.crawlerspec import (CrawlerSpecManager,
create_crawler_spec_resource)
from slyd.bot import create_bot_resource
import slyd.settings
from slyd.projects import ProjectsResource
root = Resource()
root.putChild("static", File(config['docroot']))
crawler_settings = Settings()
crawler_settings.setmodule(slyd.settings)
spec_manager = CrawlerSpecManager(crawler_settings)
# add project management at /projects
projects = ProjectsResource(crawler_settings)
root.putChild('projects', projects)
# add crawler at /projects/PROJECT_ID/bot
log.msg("Slybot specs loading from %s/[PROJECT]" % spec_manager.basedir,
level=log.DEBUG)
projects.putChild("bot", create_bot_resource(spec_manager))
# add spec at /projects/PROJECT_ID/spec
spec = create_crawler_spec_resource(spec_manager)
projects.putChild("spec", spec)
return root
def setUp(self):
sm = test_spec_manager()
spec_resource = create_crawler_spec_resource(sm)
self.temp_project_dir = mkdtemp(dir=SPEC_DATA_DIR, prefix='test-run-')
self.project = basename(self.temp_project_dir)
self.specsite = TestSite(spec_resource, project=self.project)
test_project_dir = join(SPEC_DATA_DIR, 'test')
copy_tree(test_project_dir, self.temp_project_dir)
def setUp(self):
sm = test_spec_manager()
spec_resource = create_crawler_spec_resource(sm)
self.temp_project_dir = mkdtemp(dir=SPEC_DATA_DIR,
prefix='test-run-')
self.project = basename(self.temp_project_dir)
self.specsite = TestSite(spec_resource, project=self.project)
test_project_dir = join(SPEC_DATA_DIR, 'test')
copy_tree(test_project_dir, self.temp_project_dir)
