Exemplo n.º 1
0
def masterstart():
    master = Master()
    redisCon = Redis(host=conf.REDIS_HOST,
                     port=conf.REDIS_PORT,
                     password=conf.REDIS_PASSWD)

    try:
        logger.log(CUSTOM_LOGGING.SYSINFO, 'starting rq server...')
        master.start()
    except KeyboardInterrupt, ke:
        finishedJob = redisCon.scard('visited')
        leftJob = redisCon.llen('visit')
        keMsg = """
you input KeyboardInterrupt(Ctrl+C was pressed), so quit,
finished: %d
left: %d
                """ % (finishedJob, leftJob)
        logger.log(CUSTOM_LOGGING.WARNING, keMsg)
Exemplo n.º 2
0
def masterstart():
    master = Master()
    redisCon = Redis(host=conf.REDIS_HOST, port=conf.REDIS_PORT, password=conf.REDIS_PASSWD)

    try:
        logger.log(CUSTOM_LOGGING.SYSINFO, "starting rq server...")
        master.start()
    except KeyboardInterrupt, ke:
        finishedJob = redisCon.scard("visited")
        leftJob = redisCon.llen("visit")
        keMsg = """
you input KeyboardInterrupt(Ctrl+C was pressed), so quit,
finished: %d
left: %d
                """ % (
            finishedJob,
            leftJob,
        )
        logger.log(CUSTOM_LOGGING.WARNING, keMsg)
Exemplo n.º 3
0
def testCrawlerDist():
    master = Master()
    redisCon = Redis(host=conf.REDIS_HOST,
                     port=conf.REDIS_PORT,
                     password=conf.REDIS_PASSWD)

    try:
        logger.log(CUSTOM_LOGGING.SYSINFO, 'starting rq server...')
        master.start()
    except KeyboardInterrupt, ke:
        totalJob = len(redisCon.keys('rq:job:*'))
        failedJob = len(redisCon.keys('rq:fail*'))
        finishedJob = redisCon.scard('visited')
        leftJob = redisCon.llen('visit')
        # leftJob = totalJob - failedJob
        keMsg = """
you input KeyboardInterrupt(Ctrl+C was pressed), so quit,
finished: %d
left: %d
                """ % (finishedJob, leftJob)
        logger.log(CUSTOM_LOGGING.WARNING, keMsg)
Exemplo n.º 4
0
def testCrawlerDist():
    master = Master()
    redisCon = Redis(host=conf.REDIS_HOST,
                     port=conf.REDIS_PORT,
                     password=conf.REDIS_PASSWD)

    try:
        logger.log(CUSTOM_LOGGING.SYSINFO, 'starting rq server...')
        master.start()
    except KeyboardInterrupt, ke:
        totalJob = len(redisCon.keys('rq:job:*'))
        failedJob = len(redisCon.keys('rq:fail*'))
        finishedJob = redisCon.scard('visited')
        leftJob = redisCon.llen('visit')
        # leftJob = totalJob - failedJob
        keMsg = """
you input KeyboardInterrupt(Ctrl+C was pressed), so quit,
finished: %d
left: %d
                """ % (finishedJob, leftJob)
        logger.log(CUSTOM_LOGGING.WARNING, keMsg)