def process_snapshot(index):
'''数据处理'''
mqkey = 'snapshot'
procname = "spider-%s-%s" % (mqkey, index)
setproctitle(procname)
from common import mongoMq
from utils import mq as Mq
from utils.logger import logger
import business.snapshot as bSnapshot
start = time()
while True:
#如果停止，退出程序
if time() - start > 10:
if not _checkActive(mongoMq, procname[7:]): exit()
start = time()
row = Mq.consume('snapshot')
logger.debug("snapshot::::%s" % json.dumps(row))
if not row:
sleep(10)
continue
bSnapshot.generate(row)
Mq.finish(row['id'], 'snapshot')
def process_mirror(index):
'''镜像的进程'''
mqkey = 'mirror'
procname = "spider-%s-%s" % (mqkey, index)
setproctitle(procname)
from common import mongoMq
from utils import mq as Mq
import business.mirror as bMirror
start = time()
while True:
#如果停止，退出程序
if time() - start > 10:
if not _checkActive(mongoMq, procname[7:]): exit()
start = time()
row = Mq.consume('mirror')
if not row:
sleep(10)
continue
bMirror.generate(row['id'])
Mq.finish(row['id'], 'mirror')
def process_darklink(index):
'''数据处理'''
mqkey = 'darklink'
procname = "spider-%s-%s" % (mqkey, index)
setproctitle(procname)
from common import mongoMq
from utils import mq as Mq
start = time()
while True:
#如果停止，退出程序
if time() - start > 10:
if not _checkActive(mongoMq, procname[7:]): exit()
start = time()
row = Mq.consume('darklink')
if not row:
sleep(10)
continue
bSnapshot.generate(row)
Mq.finish(row['id'], 'darklink')
def process_piping(index):
'''数据处理'''
mqkey = 'piping'
procname = "spider-%s-%s" % (mqkey, index)
setproctitle(procname)
from common import mongoMq
from utils import mq as Mq
import business.piping as bPiping
start = time()
while True:
#如果停止，退出程序
if time() - start > 10:
if not _checkActive(mongoMq, procname[7:]): exit()
start = time()
row = Mq.consume('piping')
if not row:
sleep(10)
continue
bPiping.piping_all(row['id'])
Mq.finish(row['id'], 'piping')
def process_notify(index):
'''蜘蛛'''
mqkey = 'notify'
procname = "spider-%s-%s" % (mqkey, index)
setproctitle(procname)
from common import mongoMq
from utils import mq as Mq
import business.notify as bNotify
start = time()
while True:
#如果停止，退出程序
if time() - start > 10:
if not _checkActive(mongoMq, procname[7:]): exit()
start = time()
row = Mq.consume('notify')
if not row:
sleep(10)
continue
result = bNotify.send(row['id'])
Mq.finish(row['id'], 'notify')
def process_spider(index):
'''蜘蛛'''
mqkey = 'spider'
procname = "spider-%s-%s" % (mqkey, index)
setproctitle(procname)
from common import mongoMq
from utils import mq as Mq
from business.crawler import crawl
start = time()
while True:
#如果停止，退出程序
if time() - start > 10:
if not _checkActive(mongoMq, procname[7:]): exit()
start = time()
row = Mq.consume('spider')
if not row:
sleep(10)
continue
crawl(row)
Mq.finish(row['id'], 'spider')