def __init__(self, config):
self.db = MongoDB(config)
self.scrapyd = 'http://%s:%s/' % (environ.get('HYPHE_CRAWLER_HOST', config['host']), int(environ.get('HYPHE_CRAWLER_PORT', config['scrapy_port'])))
self.db_name = config["db_name"]
self.queue = None
self.depiler = LoopingCall(self.depile)
self.depiler.start(0.2, True)
def __init__(self, config):
self.db = MongoDB(config)
self.scrapyd = 'http://%s:%s/' % (config['host'], config['scrapy_port'])
self.queue = None
self.depiler = LoopingCall(self.depile)
self.depiler.start(1, True)
