def masterstart():
master = Master()
redisCon = Redis(host=conf.REDIS_HOST,
port=conf.REDIS_PORT,
password=conf.REDIS_PASSWD)
try:
logger.log(CUSTOM_LOGGING.SYSINFO, 'starting rq server...')
master.start()
except KeyboardInterrupt, ke:
finishedJob = redisCon.scard('visited')
leftJob = redisCon.llen('visit')
keMsg = """
you input KeyboardInterrupt(Ctrl+C was pressed), so quit,
finished: %d
left: %d
""" % (finishedJob, leftJob)
logger.log(CUSTOM_LOGGING.WARNING, keMsg)
def masterstart():
master = Master()
redisCon = Redis(host=conf.REDIS_HOST, port=conf.REDIS_PORT, password=conf.REDIS_PASSWD)
try:
logger.log(CUSTOM_LOGGING.SYSINFO, "starting rq server...")
master.start()
except KeyboardInterrupt, ke:
finishedJob = redisCon.scard("visited")
leftJob = redisCon.llen("visit")
keMsg = """
you input KeyboardInterrupt(Ctrl+C was pressed), so quit,
finished: %d
left: %d
""" % (
finishedJob,
leftJob,
)
logger.log(CUSTOM_LOGGING.WARNING, keMsg)
def testCrawlerDist():
master = Master()
redisCon = Redis(host=conf.REDIS_HOST,
port=conf.REDIS_PORT,
password=conf.REDIS_PASSWD)
try:
logger.log(CUSTOM_LOGGING.SYSINFO, 'starting rq server...')
master.start()
except KeyboardInterrupt, ke:
totalJob = len(redisCon.keys('rq:job:*'))
failedJob = len(redisCon.keys('rq:fail*'))
finishedJob = redisCon.scard('visited')
leftJob = redisCon.llen('visit')
# leftJob = totalJob - failedJob
keMsg = """
you input KeyboardInterrupt(Ctrl+C was pressed), so quit,
finished: %d
left: %d
""" % (finishedJob, leftJob)
logger.log(CUSTOM_LOGGING.WARNING, keMsg)
def testCrawlerDist():
master = Master()
redisCon = Redis(host=conf.REDIS_HOST,
port=conf.REDIS_PORT,
password=conf.REDIS_PASSWD)
try:
logger.log(CUSTOM_LOGGING.SYSINFO, 'starting rq server...')
master.start()
except KeyboardInterrupt, ke:
totalJob = len(redisCon.keys('rq:job:*'))
failedJob = len(redisCon.keys('rq:fail*'))
finishedJob = redisCon.scard('visited')
leftJob = redisCon.llen('visit')
# leftJob = totalJob - failedJob
keMsg = """
you input KeyboardInterrupt(Ctrl+C was pressed), so quit,
finished: %d
left: %d
""" % (finishedJob, leftJob)
logger.log(CUSTOM_LOGGING.WARNING, keMsg)