Ejemplos de AsyncioSpiderFactory en Python

Ejemplo n.º 1

Archivo: zb_trade.py Proyecto: goddessofpom/POM-crawler

from core.factory import AsyncioSpiderFactory
from component.cleaners import ZBTradeCleaner
from component.limiters import StandardLimiter
from component.ip_controllers import StandardController
from component.exception_handler import ZBTradeExceptionHandler

if __name__ == "__main__":
    limiter = StandardLimiter("ZB")
    ip_controller = StandardController()
    cleaner = ZBTradeCleaner()
    exception_handler = ZBTradeExceptionHandler()

    spider = AsyncioSpiderFactory.make_spider(
        "ZBTradeSpider",
        "ZB",
        limiter=limiter,
        cleaner=cleaner,
        exception_handler=exception_handler,
        ip_controller=ip_controller)

    coinpairs = spider.get_coinpairs()
    depth_url = "http://api.zb.cn/data/v1/trades?size=50&market="
    url_list = [[depth_url + coinpair.replace("/", "_").lower(), coinpair]
                for coinpair in coinpairs]
    spider.add_task(url_list)

    spider.run()

Ejemplo n.º 2

from core.factory import AsyncioSpiderFactory
from component.cleaners import OKCoinDepthCleaner
from component.limiters import StandardLimiter
from component.ip_controllers import StandardController
from component.exception_handler import OKCoinDepthExceptionHandler

if __name__ == "__main__":
    limiter = StandardLimiter("OKCOIN")
    ip_controller = StandardController()
    cleaner = OKCoinDepthCleaner()
    exception_handler = OKCoinDepthExceptionHandler()

    spider = AsyncioSpiderFactory.make_spider(
        "OKCoinDepthSpider",
        "OKCOIN",
        limiter=limiter,
        cleaner=cleaner,
        exception_handler=exception_handler,
        ip_controller=ip_controller)

    coinpairs = spider.get_coinpairs()
    depth_url = "https://www.okcoin.com/api/v1/depth.do?symbol="
    url_list = [[depth_url + coinpair.replace("/", "_").lower(), coinpair]
                for coinpair in coinpairs]
    spider.add_task(url_list)

    spider.run()

Ejemplo n.º 3

Archivo: binance_depth.py Proyecto: goddessofpom/POM-crawler

from core.factory import AsyncioSpiderFactory
from component.cleaners import BinanceDepthCleaner
from component.limiters import BinanceLimiter
from component.ip_controllers import StandardController
from component.exception_handler import BinanceDepthExceptionHandler


if __name__ == "__main__":
    limiter = BinanceLimiter("Binance")
    cleaner = BinanceDepthCleaner()
    exception_handler = BinanceDepthExceptionHandler()
    ip_controller = StandardController()
    spider = AsyncioSpiderFactory.make_spider(
        "BinanceDepthSpider", "Binance", limiter=limiter, cleaner=cleaner,
        exception_handler=exception_handler, ip_controller=ip_controller
    )
    coinpairs = spider.get_coinpairs()
    depth_url = "https://api.binance.com/api/v1/depth?limit=100&symbol="
    url_list = [[depth_url + coinpair.replace("/", ""), coinpair] for coinpair in coinpairs]
    spider.add_task(url_list)

    spider.run()

Ejemplo n.º 4

from core.factory import AsyncioSpiderFactory
from component.cleaners import PoloniexDepthCleaner
from component.limiters import StandardLimiter
from component.ip_controllers import StandardController
from component.exception_handler import PoloniexDepthExceptionHandler

if __name__ == "__main__":
    limiter = StandardLimiter("Poloniex")
    ip_controller = StandardController()
    cleaner = PoloniexDepthCleaner()
    exception_handler = PoloniexDepthExceptionHandler()

    spider = AsyncioSpiderFactory.make_spider(
        "PoloniexDepthSpider",
        "Poloniex",
        limiter=limiter,
        cleaner=cleaner,
        exception_handler=exception_handler,
        ip_controller=ip_controller)

    coinpairs = spider.get_coinpairs()
    depth_url = "https://poloniex.com/public?command=returnOrderBook&depth=30&currencyPair="
    url_list = [[depth_url + "_".join(coinpair.split("/")[::-1]), coinpair]
                for coinpair in coinpairs]
    spider.add_task(url_list)

    spider.run()