from test.framework.spider import Test_Spider_2
from test.framework.spider.test_spider.test_Spider_03 import Test_Spider_3
from test.framework.setting import Setting
from test.framework.core.crawler import Crawler
from test.framework.spider.test_spider.test_Spider_01 import Test_Spider_1
from twisted.internet import reactor, defer
import logging
LOG_FORMAT = '%(asctime)s-%(filename)s[line:%(lineno)d]-%(levelname)s: %(message)s'
DATE_FORMAT = "%m/%d/%Y %H:%M:%S %p"
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT, datefmt=DATE_FORMAT)
def finish_crawl(content):
logging.info("finish")
return content
settings = Setting()
crawler_01 = Crawler(Test_Spider_1, settings)
crawler_02 = Crawler(Test_Spider_2, settings)
crawler_03 = Crawler(Test_Spider_3, settings)
spider_01 = crawler_01._create_spider()
spider_02 = crawler_02._create_spider()
spider_03 = crawler_03._create_spider()
c1 = crawler_01.crawl()
c2 = crawler_02.crawl()
c3 = crawler_03.crawl()
dd = defer.DeferredList([c1, c2, c3])
dd.addBoth(lambda _: reactor.stop())
reactor.run()
# # print(tradingday)
# if data_Text == 'IF1810':
# print(data_Text+":"+volume)
# self.total_sale_1810 += int(volume)
#
# if data_Text == 'IF1811':
# # print(data_Text+":"+volume)
# self.total_sale_1811 += int(volume)
#
# if data_Text == 'IF1812':
# # print(data_Text+":"+volume)
# self.total_sale_1812 += int(volume)
#
# if data_Text == 'IF1903':
# # print(data_Text+":"+volume)
# self.total_sale_1903 += int(volume)
#
# if data_Text == 'IF1906':
# # print(data_Text+":"+volume)
# self.total_sale_1906 += int(volume)
return None
if __name__ == '__main__':
settings = Setting()
crawler_01 = Crawler(Cffex_Rank, settings)
c1 = crawler_01.crawl()
c1.addBoth(lambda _: reactor.stop())
reactor.run()
from test.framework.core.crawler import Crawler, _get_spider_loader
from twisted.internet.defer import DeferredList
from twisted.internet import reactor
from test.framework.setting import Setting
s = Setting()
cls = _get_spider_loader(s)
_active = set()
for name, module in cls._spiders.items():
crawler = Crawler(module, s)
#spider = crawler._create_spider()
d = crawler.crawl()
_active.add(d)
dd = DeferredList(_active)
if dd.called:
print("have called")
#dd.addCallback(lambda _ :reactor.callLater(5,crawler.stop))
#dd.addCallback(crawler.stop)
dd.addBoth(lambda _: reactor.stop())
reactor.run()