Python create_crawler_spec_resourceの例

コード例 #1

ファイル: tap.py プロジェクト: 4nakin/portia

def create_root(config):
    from scrapy import log
    from scrapy.settings import CrawlerSettings
    from slyd.crawlerspec import (CrawlerSpecManager,
        create_crawler_spec_resource)
    from slyd.bot import create_bot_resource
    import slyd.settings
    from slyd.projects import ProjectsResource

    root = Resource()
    root.putChild("static", File(config['docroot']))

    crawler_settings = CrawlerSettings(settings_module=slyd.settings)
    spec_manager = CrawlerSpecManager(crawler_settings)

    # add project management at /projects
    projects = ProjectsResource(crawler_settings)
    root.putChild('projects', projects)

    # add crawler at /projects/PROJECT_ID/bot
    log.msg("Slybot specs loading from %s/[PROJECT]" % spec_manager.basedir,
        level=log.DEBUG)
    projects.putChild("bot", create_bot_resource(spec_manager))

    # add spec at /projects/PROJECT_ID/spec
    spec = create_crawler_spec_resource(spec_manager)
    projects.putChild("spec", spec)
    return root

コード例 #2

ファイル: tap.py プロジェクト: strategist922/portia-proxy

def create_root(config):
    from scrapy import log
    from scrapy.settings import Settings
    from slyd.crawlerspec import (CrawlerSpecManager,
                                  create_crawler_spec_resource)
    from slyd.bot import create_bot_resource
    import slyd.settings
    from slyd.projects import ProjectsResource

    root = Resource()
    root.putChild("static", File(config['docroot']))

    crawler_settings = Settings()
    crawler_settings.setmodule(slyd.settings)
    spec_manager = CrawlerSpecManager(crawler_settings)

    # add project management at /projects
    projects = ProjectsResource(crawler_settings)
    root.putChild('projects', projects)

    # add crawler at /projects/PROJECT_ID/bot
    log.msg("Slybot specs loading from %s/[PROJECT]" % spec_manager.basedir,
            level=log.DEBUG)
    projects.putChild("bot", create_bot_resource(spec_manager))

    # add spec at /projects/PROJECT_ID/spec
    spec = create_crawler_spec_resource(spec_manager)
    projects.putChild("spec", spec)
    return root

コード例 #3

ファイル: test_spec.py プロジェクト: strategist922/portia-proxy

 def setUp(self):
     sm = test_spec_manager()
     spec_resource = create_crawler_spec_resource(sm)
     self.temp_project_dir = mkdtemp(dir=SPEC_DATA_DIR, prefix='test-run-')
     self.project = basename(self.temp_project_dir)
     self.specsite = TestSite(spec_resource, project=self.project)
     test_project_dir = join(SPEC_DATA_DIR, 'test')
     copy_tree(test_project_dir, self.temp_project_dir)

コード例 #4

ファイル: test_spec.py プロジェクト: 1060460048/portia

 def setUp(self):
     sm = test_spec_manager()
     spec_resource = create_crawler_spec_resource(sm)
     self.temp_project_dir = mkdtemp(dir=SPEC_DATA_DIR,
         prefix='test-run-')
     self.project = basename(self.temp_project_dir)
     self.specsite = TestSite(spec_resource, project=self.project)
     test_project_dir = join(SPEC_DATA_DIR, 'test')
     copy_tree(test_project_dir, self.temp_project_dir)