Example #1
0
File: worker.py Project: B-Rich/mtq
def aux(args):
    
    logger = logging.getLogger('mq.Worker')
    logger.setLevel(logging.INFO)
    hdlr = ColorStreamHandler()
    logger.addHandler(hdlr)

    config = config_dict(args.config)
    
    tags = config.get('TAGS', ()) or args.tags
    queues = config.get('QUEUES', ()) or args.queues
    
    factory = MTQConnection.from_config(config)
    
    worker = factory.new_worker(queues=queues, tags=tags, log_worker_output=args.log_output, 
                                poll_interval=args.poll_interval)
    
    if config.get('exception_handler'):
        worker.push_exception_handler(config['exception_handler'])
    if config.get('pre_call'):
        worker._pre_call = config.get('pre_call')
    if config.get('post_call'):
        worker._post_call = config.get('post_call')

    if args.job_id:
        job = factory.get_job(args.job_id)
        if job is None:
            worker.logger.error('No job %s' % args.job_id)
            return
        
        worker.process_job(job)
        return
        
    worker.work(one=args.one, batch=args.batch)
Example #2
0
def main():

    logger = logging.getLogger("mtq")
    logger.setLevel(logging.INFO)
    hdlr = ColorStreamHandler()
    logger.addHandler(hdlr)

    parser = ArgumentParser(
        description=__doc__,
        version="Mongo Task Queue (mtq) v%s" % mtq.__version__,
        formatter_class=RawDescriptionHelpFormatter,
    )
    parser.add_argument("-c", "--config", help="Python module containing MTQ settings.")

    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument("-a", "--add", help="Schedule a new job", action="store_true")
    group.add_argument("-n", "--now", help="Run a job now!", metavar="TASK")
    group.add_argument("--remove", help="Remove a rule", type=ObjectId)
    group.add_argument("-u", "--update", help="Update a rule", type=ObjectId)
    group.add_argument("-l", "--list", help="List rules", action="store_true")
    group.add_argument("-s", "--serve-forever", "--run", help="List rules", action="store_true", dest="run")
    parser.add_argument(
        "-r", "--rule", help="Schedule rule based on the iCal RFC (http://www.ietf.org/rfc/rfc2445.txt)"
    )
    parser.add_argument(
        "-t", "--task", help="importable string of the task to be run. Must be a callable object with no arguments"
    )
    parser.add_argument("-q", "--queue", help="name of the queue (default: default)")
    parser.add_argument("--tags", help="tag the job with these tags")
    args = parser.parse_args()

    config = config_dict(args.config)
    factory = mtq.from_config(config)

    scheduler = factory.scheduler()

    if args.rule:
        test_rule(args.rule)

    if args.add:
        if args.task is None:
            raise Exception("must specify task")
        _id = scheduler.add_job(args.rule, args.task, args.queue or "default", args.tags)
        print("Added new scheduled task _id=%s" % (_id,))
    if args.update:
        scheduler.update_job(args.update, args.rule, args.task, args.queue, args.tags)
    elif args.list:
        pprint_scheduler(scheduler)
    elif args.remove:
        scheduler.remove_job(args.remove)
        print("Remove scheduled task _id=%s" % (args.remove,))
    elif args.now:
        queue = factory.queue(args.queue, tags=args.tags)
        queue.enqueue_call(args.now)
    elif args.run:
        scheduler.run()
Example #3
0
File: ctrl.py Project: B-Rich/mtq
def main():
    
    parser = ArgumentParser(description=__doc__, version='Mongo Task Queue (mtq) v%s' % mtq.__version__)
    
    parser.add_argument('-c', '--config', help='Python module containing MTQ settings.')

    sp = parser.add_subparsers()
    
    wparser = sp.add_parser('working',
                            help=('Set all workers as not working. '
                                  'Live workers will reset working status'
                                  'on next check-in'))
    wparser.set_defaults(main=working)
    
    fparser = sp.add_parser('failed', help='Flag failed jobs as fixed')
    fparser.set_defaults(main=failed)
    group = fparser.add_mutually_exclusive_group(required=True)
    group.add_argument('-a', '--all', action='store_true',
                       help='All jobs')
    group.add_argument('-f', '--func',
                       help='All jobs with function name')
    group.add_argument('-i', '--id', type=ObjectId,
                       help='Job Id')

    fparser = sp.add_parser('finish', help='Flag unfinished jobs as finished (use with caution)')
    fparser.set_defaults(main=finish)
    fparser.add_argument('-i', '--id',
                         type=ObjectId, required=True,
                         help='Job Id')
    
    sparser = sp.add_parser('shutdown',
                            help=('Schedule workers for shutdown. '
                                  'Workers will self-terminate on next check-in'))
    sparser.add_argument('-s', '--status', type=int,
                       help='Status code', default=1)

    sparser.set_defaults(main=shutdown)
    
    group = sparser.add_mutually_exclusive_group(required=True)
    
    group.add_argument('-a', '--all', action='store_true',
                       help='All workers')
    group.add_argument('-k', '--hostname',
                       help='Workers only running on host')
    group.add_argument('-n', '--name',
                       help='Worker name')
    group.add_argument('-i', '--id', type=ObjectId,
                       help='Worker Id')
    
    args = parser.parse_args()
    
    config = config_dict(args.config)
    factory = MTQConnection.from_config(config)
    
    args.main(factory, args)
Example #4
0
def main():

    logger = logging.getLogger('mtq')
    logger.setLevel(logging.INFO)
    hdlr = ColorStreamHandler()
    logger.addHandler(hdlr)

    parser = ArgumentParser(description=__doc__, version='Mongo Task Queue (mtq) v%s' % mtq.__version__,
                            formatter_class=RawDescriptionHelpFormatter)
    parser.add_argument('-c', '--config', help='Python module containing MTQ settings.')
    
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('-a', '--add', help='Schedule a new job', action='store_true')
    group.add_argument('-n', '--now', help='Run a job now!', metavar='TASK')
    group.add_argument('--remove', help='Remove a rule', type=ObjectId)
    group.add_argument('-u', '--update', help='Update a rule', type=ObjectId)
    group.add_argument('-l', '--list', help='List rules', action='store_true')
    group.add_argument('-s', '--serve-forever', '--run', help='List rules', action='store_true',
                       dest='run')
    parser.add_argument('-r', '--rule', help='Schedule rule based on the iCal RFC (http://www.ietf.org/rfc/rfc2445.txt)')
    parser.add_argument('-t', '--task', help='importable string of the task to be run. Must be a callable object with no arguments')
    parser.add_argument('-q', '--queue', help='name of the queue (default: default)')
    parser.add_argument('--tags', help='tag the job with these tags')
    parser.add_argument('--timeout', type=int, default=None,
                        help='Timeout after N seconds', metavar='N')
    args = parser.parse_args()
    
    config = config_dict(args.config)
    factory = mtq.from_config(config)
        
    scheduler = factory.scheduler()
    
    if args.rule:
        test_rule(args.rule)
        
    if args.add:
        if args.task is None:
            raise Exception('must specify task')
        _id = scheduler.add_job(args.rule, args.task, args.queue or 'default', args.tags, args.timeout)
        print('Added new scheduled task _id=%s' % (_id,))
    if args.update:
        scheduler.update_job(args.update, args.rule, args.task, args.queue, args.tags)
    elif args.list:
        pprint_scheduler(scheduler)
    elif args.remove:
        scheduler.remove_job(args.remove)
        print('Remove scheduled task _id=%s' % (args.remove,))
    elif args.now:
        queue = factory.queue(args.queue, tags=args.tags)
        queue.enqueue_call(args.now, timeout=args.timeout)
    elif args.run:
        scheduler.run()
Example #5
0
def main():
    parser = ArgumentParser(description=__doc__, version='Mongo Task Queue (mtq) v%s'
                            % mtq.__version__, add_help=False)
    parser.add_argument('-c', '--config', help='Python module containing MTQ settings.')

    args, _ = parser.parse_known_args()
    config = config_dict(args.config)
    add_extra_arguments = config.get('extra_arguments')
    init_config = config.get('init')

    parser = ArgumentParser(description=__doc__,
                            version='Mongo Task Queue (mtq) v%s' % mtq.__version__, add_help=True)
    parser.add_argument('-c', '--config', help='Python module containing MTQ settings.')

    parser.add_argument('queues', nargs='*', default=['default'], help='The queues to listen on (default: %(default)r)')
    parser.add_argument('-r', '--reloader', action='store_true', help='Reload the worker when it detects a change')
    parser.add_argument('-p', '--poll-interval', help='Sleep interval to check for jobs', default=3, type=int)
    parser.add_argument('-t', '--tags', nargs='*', help='only process jobs which contain all of the tags', default=[])
    parser.add_argument('-l', '--log-output', action='store_true', help='Store job and woker ouput in the db, seealso mtq-tail')
    parser.add_argument('-1', '--one', action='store_true',
                        help='Process only the first job')
    parser.add_argument('--backlog', '--list-jobs', action='store_true',
                        help='List backlog of jobs and exit', dest='backlog')
    parser.add_argument('-b', '--batch', action='store_true',
                        help='Process jobs until the queue is empty, then exit')
    parser.add_argument('-j', '--job-id', type=object_id,
                        help='Process the job (even if it has already been processed)')
    parser.add_argument('-f', '--failed', action='store_true',
                        help='Process failed jobs')

    if add_extra_arguments:
        add_extra_arguments(parser)

    args = parser.parse_args()

    if init_config:
        init_config(args)

    if args.reloader:
        from werkzeug.serving import run_with_reloader
        run_with_reloader(lambda: aux(args))
    else:
        aux(args)
Example #6
0
def main():
    
    parser = ArgumentParser(description=__doc__, version='0.0')
    parser.add_argument('-c', '--config', help='Python module containing MTQ settings.')
    parser.add_argument('-m', '--max-age',
                        help='Maximum age of jobs (e.g. 1d) unit may be one of s (seconds), m (minutes), h (hours) or d (days) ',
                        type=max_age, dest='since', metavar='AGE',
                        default=None)
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('-q', '--queues', action='store_const',
                       const=queue_stats, dest='action',
                       help='print stats on queues')
    group.add_argument('-w', '--worker', action='store_const', const=worker_stats, dest='action',
                       help='print stats on workers')
    group.add_argument('-j', '--jobs', action='store_const', const=print_job_stats, dest='action',
                       help='print stats on jobs')
    group.add_argument('-s', '--storage', '--db', action='store_const', const=print_db_stats, dest='action',
                       help='print stats on database')
    
    args = parser.parse_args()
    
    config = config_dict(args.config)
    factory = MTQConnection.from_config(config)
    args.action(factory, args)
Example #7
0
def aux(args):

    logger = logging.getLogger('mq.Worker')
    logger.setLevel(logging.INFO)
    hdlr = ColorStreamHandler()
    logger.addHandler(hdlr)

    config = config_dict(args.config)

    tags = config.get('TAGS', ()) or args.tags
    queues = config.get('QUEUES', ()) or args.queues

    factory = MTQConnection.from_config(config)

    worker = factory.new_worker(queues=queues,
                                tags=tags,
                                log_worker_output=args.log_output,
                                poll_interval=args.poll_interval)

    if config.get('exception_handler'):
        worker.push_exception_handler(config['exception_handler'])
    if config.get('pre_call'):
        worker._pre_call = config.get('pre_call')
    if config.get('post_call'):
        worker._post_call = config.get('post_call')

    if args.job_id:
        job = factory.get_job(args.job_id)
        if job is None:
            worker.logger.error('No job %s' % args.job_id)
            return

        worker.process_job(job)
        return

    worker.work(one=args.one, batch=args.batch)
Example #8
0
def main():

    logger = logging.getLogger('mtq')
    logger.setLevel(logging.INFO)
    hdlr = ColorStreamHandler()
    logger.addHandler(hdlr)

    parser = ArgumentParser(description=__doc__,
                            version='Mongo Task Queue (mtq) v%s' %
                            mtq.__version__,
                            formatter_class=RawDescriptionHelpFormatter)
    parser.add_argument('-c',
                        '--config',
                        help='Python module containing MTQ settings.')

    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('-a',
                       '--add',
                       help='Schedule a new job',
                       action='store_true')
    group.add_argument('-n', '--now', help='Run a job now!', metavar='TASK')
    group.add_argument('--remove', help='Remove a rule', type=ObjectId)
    group.add_argument('-u', '--update', help='Update a rule', type=ObjectId)
    group.add_argument('-l', '--list', help='List rules', action='store_true')
    group.add_argument('-s',
                       '--serve-forever',
                       '--run',
                       help='List rules',
                       action='store_true',
                       dest='run')
    parser.add_argument(
        '-r',
        '--rule',
        help=
        'Schedule rule based on the iCal RFC (http://www.ietf.org/rfc/rfc2445.txt)'
    )
    parser.add_argument(
        '-t',
        '--task',
        help=
        'importable string of the task to be run. Must be a callable object with no arguments'
    )
    parser.add_argument('-q',
                        '--queue',
                        help='name of the queue (default: default)')
    parser.add_argument('--tags', help='tag the job with these tags')
    args = parser.parse_args()

    config = config_dict(args.config)
    factory = mtq.from_config(config)

    scheduler = factory.scheduler()

    if args.rule:
        test_rule(args.rule)

    if args.add:
        if args.task is None:
            raise Exception('must specify task')
        _id = scheduler.add_job(args.rule, args.task, args.queue or 'default',
                                args.tags)
        print('Added new scheduled task _id=%s' % (_id, ))
    if args.update:
        scheduler.update_job(args.update, args.rule, args.task, args.queue,
                             args.tags)
    elif args.list:
        pprint_scheduler(scheduler)
    elif args.remove:
        scheduler.remove_job(args.remove)
        print('Remove scheduled task _id=%s' % (args.remove, ))
    elif args.now:
        queue = factory.queue(args.queue, tags=args.tags)
        queue.enqueue_call(args.now)
    elif args.run:
        scheduler.run()