def run_payload(self, job, out, err):
"""
(add description)
:param job:
:param out:
:param err:
:return:
"""
log = get_logger(job.jobid, logger)
self.pre_setup(job)
# get the payload command from the user specific code
pilot_user = os.environ.get('PILOT_USER', 'atlas').lower()
user = __import__('pilot.user.%s.common' % pilot_user, globals(),
locals(), [pilot_user], -1)
self.post_setup(job)
self.utility_before_payload(job)
self.utility_with_payload(job)
try:
executable = user.get_payload_command(job)
except exception.PilotException as e:
log.fatal('could not define payload command')
return None
log.info("payload execution command: %s" % executable)
try:
payload = {
'executable': executable,
'workdir': job.workdir,
'output_file': out,
'error_file': err,
'job': job
}
log.debug("payload: %s" % payload)
log.info("Starting EventService WorkExecutor")
executor = WorkExecutor()
executor.set_payload(payload)
executor.start()
log.info("EventService WorkExecutor started")
log.info("ESProcess started with pid: %s" % executor.get_pid())
job.pid = executor.get_pid()
if job.pid:
job.pgrp = os.getpgid(job.pid)
self.utility_after_payload_started(job)
except Exception as e:
log.error('could not execute: %s' % str(e))
return None
return executor
def test_workexecutor_update_events(self):
"""
Make sure that no exceptions to run work executor.
"""
try:
executor = WorkExecutor()
self.executor = executor
executor.set_payload(self._payload)
executor.start()
ret = executor.get_event_ranges()
logger.debug(ret)
update_events = []
for event in ret:
event_range = {
"eventRangeID": event['eventRangeID'],
"eventStatus": 'finished'
}
update_events.append(event_range)
event_range_status = [{
"zipFile": {
"numEvents": len(update_events),
"objstoreID": 1318,
"adler32": '000000',
"lfn": 'test_file',
"fsize": 100,
"pathConvention": 1000
},
"eventRanges": update_events
}]
import json
event_range_message = {
'version': 1,
'eventRanges': json.dumps(event_range_status)
}
ret = executor.update_events(event_range_message)
logger.debug(ret)
executor.stop()
except Exception as ex:
if self.executor:
self.executor.stop()
raise ex
def run_payload(self, job, cmd, out, err):
"""
(add description)
:param job: job object.
:param cmd: (unused in ES mode)
:param out: stdout file object.
:param err: stderr file object.
:return:
"""
self.pre_setup(job)
# get the payload command from the user specific code
pilot_user = os.environ.get('PILOT_USER', 'atlas').lower()
user = __import__('pilot.user.%s.common' % pilot_user, globals(), locals(), [pilot_user], 0) # Python 2/3
self.post_setup(job)
self.utility_before_payload(job)
self.utility_with_payload(job)
try:
executable = user.get_payload_command(job)
except exception.PilotException:
logger.fatal('could not define payload command')
return None
logger.info("payload execution command: %s", executable)
try:
payload = {'executable': executable, 'workdir': job.workdir, 'output_file': out, 'error_file': err, 'job': job}
logger.debug("payload: %s", payload)
logger.info("Starting EventService WorkExecutor")
executor_type = self.get_executor_type()
executor = WorkExecutor(args=executor_type)
executor.set_payload(payload)
executor.start()
logger.info("EventService WorkExecutor started")
logger.info("ESProcess started with pid: %s", executor.get_pid())
job.pid = executor.get_pid()
if job.pid:
job.pgrp = os.getpgid(job.pid)
self.utility_after_payload_started(job)
except Exception as error:
logger.error('could not execute: %s', str(error))
return None
return executor
def test_workexecutor_generic(self):
"""
Make sure that no exceptions to run work executor.
"""
try:
executor = WorkExecutor()
self.executor = executor
executor.set_payload(self._payload)
executor.start()
t_start = time.time()
t1 = time.time()
while executor.is_alive():
if time.time() > t1 + 300:
logging.info("work executor is running")
t1 = time.time()
time.sleep(1)
if time.time() > t_start + 20 * 60:
executor.stop()
break
while executor.is_alive():
time.sleep(0.1)
exit_code = executor.get_exit_code()
self.assertEqual(exit_code, 0)
except Exception as ex:
logger.debug("Exception: %s, %s" % (ex, traceback.format_exc()))
if self.executor:
self.executor.stop()
while self.executor.is_alive():
time.sleep(0.1)
raise ex