def set_stop_stats(spider_id):
key = '{}_STATS'.format(spider_id)
BaseRedis.set(key, 'stopped')
def clean():
BaseRedis.delete('PROXIES')
def clean(spider_id):
key = '{}_CRAWL_STATUS'.format(spider_id)
BaseRedis.delete(key)
def get_status(spider_id):
key = '{}_CRAWL_STATUS'.format(spider_id)
return BaseRedis.get(key) or 0
def increase(spider_id):
key = '{}_CRAWL_STATUS'.format(spider_id)
crawl_count = int((BaseRedis.get(key) or 0)) + 1
BaseRedis.set(key, crawl_count)
return crawl_count
def is_stop_stats(spider_id):
key = '{}_STATS'.format(spider_id)
return BaseRedis.get(key) == 'stopped'