def run_payload(self, job, out, err): """ (add description) :param job: :param out: :param err: :return: """ log = get_logger(job.jobid, logger) self.pre_setup(job) # get the payload command from the user specific code pilot_user = os.environ.get('PILOT_USER', 'atlas').lower() user = __import__('pilot.user.%s.common' % pilot_user, globals(), locals(), [pilot_user], -1) self.post_setup(job) self.utility_before_payload(job) self.utility_with_payload(job) try: executable = user.get_payload_command(job) except exception.PilotException as e: log.fatal('could not define payload command') return None log.info("payload execution command: %s" % executable) try: payload = { 'executable': executable, 'workdir': job.workdir, 'output_file': out, 'error_file': err, 'job': job } log.debug("payload: %s" % payload) log.info("Starting EventService WorkExecutor") executor = WorkExecutor() executor.set_payload(payload) executor.start() log.info("EventService WorkExecutor started") log.info("ESProcess started with pid: %s" % executor.get_pid()) job.pid = executor.get_pid() if job.pid: job.pgrp = os.getpgid(job.pid) self.utility_after_payload_started(job) except Exception as e: log.error('could not execute: %s' % str(e)) return None return executor
def test_workexecutor_update_events(self): """ Make sure that no exceptions to run work executor. """ try: executor = WorkExecutor() self.executor = executor executor.set_payload(self._payload) executor.start() ret = executor.get_event_ranges() logger.debug(ret) update_events = [] for event in ret: event_range = { "eventRangeID": event['eventRangeID'], "eventStatus": 'finished' } update_events.append(event_range) event_range_status = [{ "zipFile": { "numEvents": len(update_events), "objstoreID": 1318, "adler32": '000000', "lfn": 'test_file', "fsize": 100, "pathConvention": 1000 }, "eventRanges": update_events }] import json event_range_message = { 'version': 1, 'eventRanges': json.dumps(event_range_status) } ret = executor.update_events(event_range_message) logger.debug(ret) executor.stop() except Exception as ex: if self.executor: self.executor.stop() raise ex
def run_payload(self, job, cmd, out, err): """ (add description) :param job: job object. :param cmd: (unused in ES mode) :param out: stdout file object. :param err: stderr file object. :return: """ self.pre_setup(job) # get the payload command from the user specific code pilot_user = os.environ.get('PILOT_USER', 'atlas').lower() user = __import__('pilot.user.%s.common' % pilot_user, globals(), locals(), [pilot_user], 0) # Python 2/3 self.post_setup(job) self.utility_before_payload(job) self.utility_with_payload(job) try: executable = user.get_payload_command(job) except exception.PilotException: logger.fatal('could not define payload command') return None logger.info("payload execution command: %s", executable) try: payload = {'executable': executable, 'workdir': job.workdir, 'output_file': out, 'error_file': err, 'job': job} logger.debug("payload: %s", payload) logger.info("Starting EventService WorkExecutor") executor_type = self.get_executor_type() executor = WorkExecutor(args=executor_type) executor.set_payload(payload) executor.start() logger.info("EventService WorkExecutor started") logger.info("ESProcess started with pid: %s", executor.get_pid()) job.pid = executor.get_pid() if job.pid: job.pgrp = os.getpgid(job.pid) self.utility_after_payload_started(job) except Exception as error: logger.error('could not execute: %s', str(error)) return None return executor
def test_workexecutor_generic(self): """ Make sure that no exceptions to run work executor. """ try: executor = WorkExecutor() self.executor = executor executor.set_payload(self._payload) executor.start() t_start = time.time() t1 = time.time() while executor.is_alive(): if time.time() > t1 + 300: logging.info("work executor is running") t1 = time.time() time.sleep(1) if time.time() > t_start + 20 * 60: executor.stop() break while executor.is_alive(): time.sleep(0.1) exit_code = executor.get_exit_code() self.assertEqual(exit_code, 0) except Exception as ex: logger.debug("Exception: %s, %s" % (ex, traceback.format_exc())) if self.executor: self.executor.stop() while self.executor.is_alive(): time.sleep(0.1) raise ex