示例#1
0
 def __init__(self, spider):
     Process.__init__(self)
     settings = get_project_settings()
     self.crawler = Crawler(settings)
     self.crawler.configure()
     self.crawler.signals.connect(reactor.stop, signal=signals.spider_closed)
     self.spider = spider
 def __init__(self, spider):
     Process.__init__(self)
     self.crawler = Crawler(spider.__class__, get_project_settings())
     # self.crawler.configure()
     self.crawler.signals.connect(reactor.stop,
                                  signal=signals.spider_closed)
     self.spider = spider
示例#3
0
 def __init__(self, spider):
     Process.__init__(self)
     setting = Settings()
     setting.setmodule(s)
     self.crawler = Crawler(setting)
     self.crawler.configure()
     self.crawler.signals.connect(reactor.stop, signal=signals.spider_closed)
     self.spider = spider
示例#4
0
文件: tasks.py 项目: 2bethere/SPTest
        def __init__(self, spider):
            Process.__init__(self)
            settings = get_project_settings()
            self.crawler = Crawler(settings)

            if not hasattr(project, 'crawler'):
                self.crawler.install()
                self.crawler.configure()
                self.crawler.signals.connect(reactor.stop, signal=signals.spider_closed)
            self.spider = spider
示例#5
0
    def __init__(self, spider):
        Process.__init__(self)
        setting = Settings()
        setting.setmodule(settings,1)
        self.crawler = Crawler(setting)

        if not hasattr(project, 'crawler'):
            self.crawler.configure()
            self.crawler.signals.connect(reactor.stop, signal = signals.spider_closed)
        self.spider = spider
示例#6
0
 def __init__(self, spider,key_word,crawl_num,n_crawls):
     Process.__init__(self)
     settings = get_project_settings()
     self.spider = spider
     self.crawler = Crawler(spider.__class__, settings)
     # self.crawler.configure()
     self.crawler.signals.connect(reactor.stop, signal=signals.spider_closed)
     self.n_crawls = n_crawls
     self.crawl_num = crawl_num
     self.key_word = key_word
示例#7
0
    def __init__(self, spider):
        Process.__init__(self)
        settings = get_project_settings()
        self.crawler = Crawler(settings)
        if not hasattr(project, 'crawler'):
            self.crawler.install()
            self.crawler.configure()

        self.crawler.signals.connect(reactor.stop, signal=signals.spider_closed)
        self.spider = spider
示例#8
0
    def __init__(self, current_dt, server, topic):

        Process.__init__(self)

        self.current_dt = current_dt
        self.server = server
        self.topic = topic

        self.crawler = Crawler(VIXSpiderSpider,
                               settings={'USER_AGENT': user_agent})

        self.crawler.signals.connect(reactor.stop,
                                     signal=scrapy_signals.spider_closed)
示例#9
0
    def __init__(self, countries, importance, event_list, current_dt, server,
                 topic):

        Process.__init__(self)

        self.countries = countries
        self.importance = importance
        self.event_list = event_list
        self.current_dt = current_dt
        self.server = server
        self.topic = topic

        self.crawler = Crawler(EconomicIndicatorsSpiderSpider,
                               settings={'USER_AGENT': user_agent})

        self.crawler.signals.connect(reactor.stop,
                                     signal=scrapy_signals.spider_closed)
示例#10
0
文件: boot.py 项目: Varato/qfbot
 def __init__(self, spider):
     Process.__init__(self)
     self.spider = spider
 def __init__(self, spider):
     Process.__init__(self)
     self.spider = spider
示例#12
0
 def __init__(self):
     """
     Initializer.
     """
     Process.__init__(self)
     self._isRunnable = True
示例#13
0
 def __init__(self, spider, urls):
     Process.__init__(self)
     settings = Settings()
     self.crawler = CrawlerProcess(settings)
     self.urls = urls
     self.spider = spider
示例#14
0
 def __init__(self, proceedings: List[str]):
     Process.__init__(self)
     self.proceedings = proceedings
     self.runner = CrawlerRunner(get_project_settings())