예제 #1
0
    def run(self):
        """
        Initialize and run ESProcess.
        """
        try:
            logger.info("starting ES HPOExecutor with thread ident: %s" % self.ident)
            if self.is_set_payload():
                payload = self.get_payload()
            elif self.is_retrieve_payload():
                payload = self.retrieve_payload()
            else:
                logger.error("Payload is not set but is_retrieve_payload is also not set. No payloads.")

            logger.info("payload: %s" % payload)

            logger.info("Starting ESProcess")
            proc = ESProcess(payload, waiting_time=999999)
            self.proc = proc
            logger.info("ESProcess initialized")

            proc.set_get_event_ranges_hook(self.get_event_ranges)
            proc.set_handle_out_message_hook(self.handle_out_message)

            logger.info('ESProcess starts to run')
            proc.start()
            logger.info('ESProcess started to run')

            exit_code = None
            try:
                iteration = long(0)  # Python 2  # noqa: F821
            except Exception:
                iteration = 0  # Python 3
            while proc.is_alive():
                iteration += 1
                if self.is_stop():
                    logger.info('Stop is set. breaking -- stop process pid=%s' % proc.pid)
                    proc.stop()
                    break
                self.stageout_es()

                exit_code = proc.poll()
                if iteration % 60 == 0:
                    logger.info('running: iteration=%d pid=%s exit_code=%s' % (iteration, proc.pid, exit_code))
                time.sleep(5)

            while proc.is_alive():
                time.sleep(1)
            logger.info("ESProcess finished")

            self.stageout_es(force=True)
            self.clean()

            self.exit_code = proc.poll()

        except Exception as e:
            logger.error('Execute payload failed: %s, %s' % (e, traceback.format_exc()))
            self.clean()
            self.exit_code = -1
        logger.info('ES HPO executor finished')
예제 #2
0
    def run(self):
        """
        Initialize and run ESProcess.
        """
        try:
            if self.is_set_payload():
                payload = self.get_payload()
            elif self.is_retrieve_payload():
                payload = self.retrieve_payload()
            else:
                logger.error(
                    "Payload is not set but is_retrieve_payload is also not set. No payloads."
                )

            job = self.get_job()
            log = get_logger(job.jobid, logger)
            log.info("payload: %s" % payload)

            log.info("Starting ESProcess")
            proc = ESProcess(payload)
            self.proc = proc
            log.info("ESProcess initialized")

            proc.set_get_event_ranges_hook(self.get_event_ranges)
            proc.set_handle_out_message_hook(self.handle_out_message)

            log.info('ESProcess starts to run')
            proc.start()
            log.info('ESProcess started to run')

            exit_code = None
            iteration = 0L
            while proc.is_alive():
                iteration += 1
                if self.is_stop():
                    log.info('Stop is set. breaking -- stop process pid=%s' %
                             proc.pid)
                    proc.stop()
                    break
                self.stageout_es()

                exit_code = proc.poll()
                if iteration % 300 == 0:
                    log.info('running: iteration=%d pid=%s exit_code=%s' %
                             (iteration, proc.pid, exit_code))
                time.sleep(1)

            while proc.is_alive():
                time.sleep(0.1)
            log.info("ESProcess finished")

            self.stageout_es(force=True)
            self.clean()

            self.exit_code = proc.poll()

        except Exception as e:
            logger.error('Execute payload failed: %s, %s' %
                         (str(e), traceback.format_exc()))
            self.clean()
            self.exit_code = -1
        logger.info('ES generic executor finished')