Python Master.start Examples

Example #1

def masterstart():
    master = Master()
    redisCon = Redis(host=conf.REDIS_HOST,
                     port=conf.REDIS_PORT,
                     password=conf.REDIS_PASSWD)

    try:
        logger.log(CUSTOM_LOGGING.SYSINFO, 'starting rq server...')
        master.start()
    except KeyboardInterrupt, ke:
        finishedJob = redisCon.scard('visited')
        leftJob = redisCon.llen('visit')
        keMsg = """
you input KeyboardInterrupt(Ctrl+C was pressed), so quit,
finished: %d
left: %d
                """ % (finishedJob, leftJob)
        logger.log(CUSTOM_LOGGING.WARNING, keMsg)

Example #2

File: master.py Project: xinali/penework

def masterstart():
    master = Master()
    redisCon = Redis(host=conf.REDIS_HOST, port=conf.REDIS_PORT, password=conf.REDIS_PASSWD)

    try:
        logger.log(CUSTOM_LOGGING.SYSINFO, "starting rq server...")
        master.start()
    except KeyboardInterrupt, ke:
        finishedJob = redisCon.scard("visited")
        leftJob = redisCon.llen("visit")
        keMsg = """
you input KeyboardInterrupt(Ctrl+C was pressed), so quit,
finished: %d
left: %d
                """ % (
            finishedJob,
            leftJob,
        )
        logger.log(CUSTOM_LOGGING.WARNING, keMsg)

Example #3

def testCrawlerDist():
    master = Master()
    redisCon = Redis(host=conf.REDIS_HOST,
                     port=conf.REDIS_PORT,
                     password=conf.REDIS_PASSWD)

    try:
        logger.log(CUSTOM_LOGGING.SYSINFO, 'starting rq server...')
        master.start()
    except KeyboardInterrupt, ke:
        totalJob = len(redisCon.keys('rq:job:*'))
        failedJob = len(redisCon.keys('rq:fail*'))
        finishedJob = redisCon.scard('visited')
        leftJob = redisCon.llen('visit')
        # leftJob = totalJob - failedJob
        keMsg = """
you input KeyboardInterrupt(Ctrl+C was pressed), so quit,
finished: %d
left: %d
                """ % (finishedJob, leftJob)
        logger.log(CUSTOM_LOGGING.WARNING, keMsg)

Example #4

def testCrawlerDist():
    master = Master()
    redisCon = Redis(host=conf.REDIS_HOST,
                     port=conf.REDIS_PORT,
                     password=conf.REDIS_PASSWD)

    try:
        logger.log(CUSTOM_LOGGING.SYSINFO, 'starting rq server...')
        master.start()
    except KeyboardInterrupt, ke:
        totalJob = len(redisCon.keys('rq:job:*'))
        failedJob = len(redisCon.keys('rq:fail*'))
        finishedJob = redisCon.scard('visited')
        leftJob = redisCon.llen('visit')
        # leftJob = totalJob - failedJob
        keMsg = """
you input KeyboardInterrupt(Ctrl+C was pressed), so quit,
finished: %d
left: %d
                """ % (finishedJob, leftJob)
        logger.log(CUSTOM_LOGGING.WARNING, keMsg)