if __name__ == '__main__': parser = argparse.ArgumentParser() _arg = parser.add_argument _arg('--redis', metavar='PATH|HOST:PORT', type=str, default=DEFAULT_REDIS, action='store', help='Redis endpoint specified as host:port - ' 'default: %s' % DEFAULT_REDIS) _arg('--flushdb', action='store_const', default=False, const=True) _arg('jobs', metavar='JOBTYPE_USERDATA_:ARGS', type=str, nargs='*') args = parser.parse_args() endpoint = args.redis.split(':', 1) if len(endpoint) == 2: job_queue = jobs.connect_to_queue(endpoint[0], int(endpoint[1])) else: job_queue = jobs.connect_to_unix_socket_queue(endpoint[0]) if args.flushdb: job_queue.redis_connection.flushall() if len(args.jobs) == 0: sys.exit(0) def job_from_def(job_def): count, job_type, userdata, args = job_def count = int(count or '1') assert count >= 1 job_ids = [] for i_job in xrange(count): job_ids.append(job_queue.push(job_type, userdata=userdata,
'(default=%s)' % DEFAULT_ORPHANED_TIMEOUT) _arg('mapping', metavar='N@JOBTYPE|OPTIONS|:COMMAND', type=str, nargs='+', help='Job specification - can be specified multiple times.\n' 'For example \'4@parse[auto_heartbeat=yes]:/bin/parser\' ' 'would run the command /bin/parser up to a maximum of 4 ' 'times in parallel with the job type identified as \'parse\', using ' 'automatic heartbeats. COMMAND can also contain fixed arguments') args = parser.parse_args() if (args.redis_unix_socket != '') and (args.redis_tcp != ''): print('error: both TCP and unix socket endpoints specified, please' ' choose one or the other') sys.exit(1) try: if args.redis_unix_socket: job_queue = jobs.connect_to_unix_socket_queue( args.redis_unix_socket) else: endpoint = args.redis_tcp or DEFAULT_REDIS job_queue = jobs.connect_to_queue(**parse_hostport(endpoint)) except redis.exceptions.ConnectionError, error: print('error: Could not connect to job queue: %s' % error) sys.exit(1) runner_args = parse_mappings(job_queue, args.mapping) greenlets = [gevent.spawn(runner_greenlet, *argtuple) for argtuple in runner_args] greenlets.append(gevent.spawn(monitor_greenlet, job_queue, args.monitor_interval, args.orphaned_timeout)) gevent.signal(signal.SIGINT, lambda: signal_handler(job_queue, greenlets))