Example #1
0
    def __init__(self, job_poll_interval = 10):
        self.reaper_id = uuid.uuid1().hex
        self.messagebus = MessageBus()
        self.chan = self.messagebus.channel
        self.is_registered = None
        
        self.JOB_POLL_INTERVAL = job_poll_interval
        self.CONTROL_QUEUE_NAME = "rpc.reaper.%s" % self.reaper_id
        self.REPLY_QUEUE_NAME = "reply.reaper.%s" % self.reaper_id # queue for RPC responses
        self.logger = logging.getLogger('reaper.%s' % self.reaper_id)
        #self.logger.setLevel(logging.DEBUG)
        

        # Accept control commands via the control exchange:
        self.chan.exchange_declare(self.CONTROL_EXCHANGE_NAME, type='direct')
        self.chan.queue_declare(queue=self.CONTROL_QUEUE_NAME, durable=False, auto_delete=True)
        self.chan.queue_bind(queue=self.CONTROL_QUEUE_NAME, exchange=self.CONTROL_EXCHANGE_NAME, routing_key=self.CONTROL_QUEUE_NAME)
        
        # RPC Service to dispatch

        self.dispatch = DispatchService(reply_queue=self.REPLY_QUEUE_NAME, )

        # Init threads to handle message consumption
        self.shutdown_event = threading.Event()
        self.control_listener = ConsumptionThread(mode='GET', shutdown_event=self.shutdown_event, name="control_listener")
Example #2
0
class Reaper(object):

    JOB_POLL_INTERVAL = 10 #seconds

    REAPER_TYPE = 'generic'
    CONTROL_EXCHANGE_NAME = 'Control_Exchange'
    commands = {}


    def __init__(self, job_poll_interval = 10):
        self.reaper_id = uuid.uuid1().hex
        self.messagebus = MessageBus()
        self.chan = self.messagebus.channel
        self.is_registered = None
        
        self.JOB_POLL_INTERVAL = job_poll_interval
        self.CONTROL_QUEUE_NAME = "rpc.reaper.%s" % self.reaper_id
        self.REPLY_QUEUE_NAME = "reply.reaper.%s" % self.reaper_id # queue for RPC responses
        self.logger = logging.getLogger('reaper.%s' % self.reaper_id)
        #self.logger.setLevel(logging.DEBUG)
        

        # Accept control commands via the control exchange:
        self.chan.exchange_declare(self.CONTROL_EXCHANGE_NAME, type='direct')
        self.chan.queue_declare(queue=self.CONTROL_QUEUE_NAME, durable=False, auto_delete=True)
        self.chan.queue_bind(queue=self.CONTROL_QUEUE_NAME, exchange=self.CONTROL_EXCHANGE_NAME, routing_key=self.CONTROL_QUEUE_NAME)
        
        # RPC Service to dispatch

        self.dispatch = DispatchService(reply_queue=self.REPLY_QUEUE_NAME, )

        # Init threads to handle message consumption
        self.shutdown_event = threading.Event()
        self.control_listener = ConsumptionThread(mode='GET', shutdown_event=self.shutdown_event, name="control_listener")

                
     
    def job_request_loop(self):
        job = None
        while not self.shutdown_event.is_set():
            if self.is_registered:
                self.logger.debug("Requesting job.")
                job = self.dispatch.get_a_job(self.reaper_id)
                self.logger.debug("Job request returned.")
                if job:
                    if job.command in self.commands:  # only commands allowed by the configuration will be executed
                        args = self.commands[job.command].split(' ')  + list(job.args or [])
                        self.logger.debug("ARGS: %s" % str(args))
                        self.logger.info("Executing %s" % ' '.join(args))
                        start_time = datetime.utcnow()
                        try:
                            output=""
                            p = Popen(args, stdout=PIPE, stderr=STDOUT)
                            self.dispatch.report_job_start(self.reaper_id, job, p.pid, start_time) # note that "job" here is a Protobuf object
                            while True:
                                line = p.stdout.readline()
                                if line == '' and p.poll() != None:
                                    break
                                output += line
                                sys.stdout.write(line)
                            resultcode = p.wait()
                        except OSError as oserr:
                            resultcode = -1
                            output += "OSError: %s" % oserr.strerror
                        end_time = datetime.utcnow()
                        if resultcode == 0:
                            state = 'complete'
                        elif resultcode == 129:
                            state = 'failed_nonblocking'
                        else:
                            state = 'failed'

                        if options.max_output_capture >= 0 and len(output) > options.max_output_capture:
                            halfmax = options.max_output_capture / 2
                            fill = "\n\n***** OUTPUT TO LONG.  %d CHARACTERS TRUNCATED BY REAPER *****\n\n" % (len(output) - options.max_output_capture)
                            output = output[:halfmax] + fill + output[-halfmax:]

                        self.logger.info("Job %s: %s" % (job.uuid[:8], state) )
                        self.dispatch.report_job_end(job, state, end_time, output)
                    else:
                        end_time = datetime.utcnow()
                        self.logger.error("Command: '%s' not found in amq_config's list of valid commands." % job.command)
                        self.dispatch.report_job_end(job, 'failed', end_time, "Command: '%s' not found in the list of valid commands for reaper %s" % (job.command, self.reaper_id))
                else:
                    self.logger.info("No jobs available.")
                    time.sleep(self.JOB_POLL_INTERVAL)
                self.logger.debug("Reached end of job loop.")
                
            else:
                # not registered yet
                self.logger.debug("Waiting for registration.")
                sleep(0.1)
    
    
    ####
    #  Control Commands
    #  TODO: refactor this to an RpcService subclass
    ####
    
    def _rpc_status(self, msg):
        return protocols.pack(protobuf.ReaperStatusResponse, {'status': 'OK'})
    
    def _rpc_shutdown(self, msg):
        response = protocols.pack(protobuf.ReaperStatusResponse, {'status': 'shutdown'})
        self.shutdown(delay=1)
        return response
    
    CONTROL_COMMAND_MAP = {
        'GetStatus': _rpc_status,
        'Shutdown': _rpc_shutdown
    }
    
    def control_command_handler(self, msg, command_map=CONTROL_COMMAND_MAP):
        """ Unpack a message and process commands 
            Speaks the RpcRequest protocol.
        """
        self.logger.debug("command_handler got a message.")
        request = protocols.unpack(protobuf.RpcRequestWrapper, msg.body)
        self.logger.debug("command msg contents: %s" % str(request))
        response = dotdict()
        response.sequence_number = request.sequence_number
        if request.method in command_map:
            try:
                response.payload = command_map[request.method].__call__(self, request.payload)
                response.error = False
            except Exception, e:
                #self.logger.error("Error in command '%s': %s %s" % (request.method, str(Exception),  e))
                sys.excepthook(*sys.exc_info())
                #traceback.print_tb(sys.last_traceback)
                response.payload = ''
                response.error = True
                response.error_string = str(e)
        else: