def _start(self):
"""Detach a process from the controlling terminal and run it in the
background as a daemon.
We don't use pksubprocess. This method is not called from the MainThread
so can't set signals.
"""
env = _safe_env()
env['SIREPO_MPI_CORES'] = str(mpi.cfg.cores)
try:
pid = os.fork()
except OSError as e:
pkdlog('{}: fork OSError: {} errno={}', self.jid, e.strerror, e.errno)
reraise
if pid != 0:
pkdlog('{}: started: pid={} cmd={}', self.jid, pid, self.cmd)
self.__pid = pid
return
try:
os.chdir(str(self.run_dir))
#Don't os.setsid() so signals propagate properly
maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
if (maxfd == resource.RLIM_INFINITY):
maxfd = runner.MAX_OPEN_FILES
for fd in range(0, maxfd):
try:
os.close(fd)
except OSError:
pass
sys.stdin = open(template_common.RUN_LOG, 'a+')
assert sys.stdin.fileno() == 0
os.dup2(0, 1)
sys.stdout = os.fdopen(1, 'a+')
os.dup2(0, 2)
sys.stderr = os.fdopen(2, 'a+')
pkdlog('{}: child will exec: {}', self.jid, self.cmd)
sys.stderr.flush()
try:
simulation_db.write_status('running', self.run_dir)
os.execvpe(self.cmd[0], self.cmd, env=env)
except BaseException as e:
pkdlog(
'{}: execvp error: {} errno={}',
self.jid,
e.strerror if hasattr(e, 'strerror') else '',
e.errno if hasattr(e, 'errno') else '',
)
finally:
sys.exit(1)
except BaseException as e:
# NOTE: there's no lock here so just append to the log. This
# really shouldn't happen, but it might (out of memory) so just
# log to the run log and hope somebody notices
self._error_during_start(e, pkdexc())
raise
def _start(self):
"""Detach a process from the controlling terminal and run it in the
background as a daemon.
We don't use pksubprocess. This method is not called from the MainThread
so can't set signals.
"""
env = _safe_env()
env['SIREPO_MPI_CORES'] = str(mpi.cfg.cores)
try:
pid = os.fork()
except OSError as e:
pkdlog('{}: fork OSError: {} errno={}', self.jid, e.strerror, e.errno)
reraise
if pid != 0:
pkdlog('{}: started: pid={} cmd={}', self.jid, pid, self.cmd)
self.__pid = pid
return
try:
os.chdir(str(self.run_dir))
#Don't os.setsid() so signals propagate properly
maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
if (maxfd == resource.RLIM_INFINITY):
maxfd = runner.MAX_OPEN_FILES
for fd in range(0, maxfd):
try:
os.close(fd)
except OSError:
pass
sys.stdin = open(template_common.RUN_LOG, 'a+')
assert sys.stdin.fileno() == 0
os.dup2(0, 1)
sys.stdout = os.fdopen(1, 'a+')
os.dup2(0, 2)
sys.stderr = os.fdopen(2, 'a+')
pkdlog('{}: child will exec: {}', self.jid, self.cmd)
sys.stderr.flush()
try:
simulation_db.write_status('running', self.run_dir)
os.execvpe(self.cmd[0], self.cmd, env=env)
except BaseException as e:
pkdlog(
'{}: execvp error: {} errno={}',
self.jid,
e.strerror if hasattr(e, 'strerror') else '',
e.errno if hasattr(e, 'errno') else '',
)
finally:
sys.exit(1)
except BaseException as e:
# NOTE: there's no lock here so just append to the log. This
# really shouldn't happen, but it might (out of memory) so just
# log to the run log and hope somebody notices
self._error_during_start(e, pkdexc())
raise
def _start(self):
"""Detach a process from the controlling terminal and run it in the
background as a daemon.
We don't use pksubprocess. This method is not called from the MainThread
so can't set signals.
"""
try:
pid = os.fork()
except OSError as e:
pkdlog('{}: fork OSError: {} errno={}', self.jid, e.strerror,
e.errno)
reraise
if pid != 0:
pkdlog('{}: started: pid={} cmd={}', self.jid, pid, self.cmd)
self.pid = pid
return
try:
os.chdir(str(self.run_dir))
#Don't os.setsid() so signals propagate properly
import resource
maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
if (maxfd == resource.RLIM_INFINITY):
maxfd = _MAX_OPEN_FILES
for fd in range(0, maxfd):
try:
os.close(fd)
except OSError:
pass
sys.stdin = open(template_common.RUN_LOG, 'a+')
assert sys.stdin.fileno() == 0
os.dup2(0, 1)
sys.stdout = os.fdopen(1, 'a+')
os.dup2(0, 2)
sys.stderr = os.fdopen(2, 'a+')
pkdlog('{}: child will exec: {}', self.jid, self.cmd)
sys.stderr.flush()
try:
simulation_db.write_status('running', self.run_dir)
os.execvp(self.cmd[0], self.cmd)
finally:
pkdlog('{}: execvp error: {} errno={}', self.jid, e.strerror,
e.errno)
sys.exit(1)
except BaseException as e:
with open(str(self.run_dir.join(template_common.RUN_LOG)),
'a') as f:
f.write('{}: error starting simulation: {}'.format(
self.jid, e))
raise
def _slot_start(self, slot):
"""Have a slot so now ask docker to run the job
POSIT: Job locked by caller
"""
# __host is sentinel of the start attempt
self.__host = slot.host
ctx = pkcollections.Dict(
kill_secs=runner.KILL_TIMEOUT_SECS,
run_dir=self.run_dir,
run_log=self.run_dir.join(template_common.RUN_LOG),
run_secs=self.run_secs(),
sh_cmd=self.__sh_cmd(),
)
self.__image = _image()
script = str(self.run_dir.join('runner-docker.sh'))
with open(str(script), 'wb') as f:
f.write(pkjinja.render_resource('runner/docker.sh', ctx))
cmd = _RUN_PREFIX + (
'--cpus={}'.format(slot.cores),
'--detach',
#TODO(robnagler) other environ vars required?
'--env=SIREPO_MPI_CORES={}'.format(slot.cores),
'--init',
'--memory={}g'.format(slot.gigabytes),
'--name={}'.format(self.__cname),
'--network=none',
#TODO(robnagler) this doesn't do anything
# '--ulimit=cpu=1',
# do not use a user name, because that may not map inside the
# container properly. /etc/passwd on the host and guest are
# different.
'--user={}'.format(os.getuid()),
) + self.__volumes() + (
#TODO(robnagler) make this configurable per code (would be structured)
self.__image,
'bash',
script,
)
self.__cid = _cmd(slot.host, cmd)
simulation_db.write_status('running', self.run_dir)
pkdlog(
'{}: started slot={} cid={} dir={} cmd={}',
self.__cname,
slot,
self.__cid,
self.run_dir,
cmd,
)
def _start_job(self):
"""Detach a process from the controlling terminal and run it in the
background as a daemon.
We don't use pksubprocess. This method is not called from the MainThread
so can't set signals.
"""
try:
pid = os.fork()
except OSError as e:
pkdlog('{}: fork OSError: {} errno={}', self.jid, e.strerror, e.errno)
reraise
if pid != 0:
pkdlog('{}: started: pid={} cmd={}', self.jid, pid, self.cmd)
return pid
try:
os.chdir(str(self.run_dir))
#Don't os.setsid() so signals propagate properly
import resource
maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
if (maxfd == resource.RLIM_INFINITY):
maxfd = 1024
for fd in range(0, maxfd):
try:
os.close(fd)
except OSError:
pass
sys.stdin = open(template_common.RUN_LOG, 'a+')
assert sys.stdin.fileno() == 0
os.dup2(0, 1)
sys.stdout = os.fdopen(1, 'a+')
os.dup2(0, 2)
sys.stderr = os.fdopen(2, 'a+')
pkdlog('{}: child will exec: {}', self.jid, self.cmd)
sys.stderr.flush()
try:
simulation_db.write_status('running', self.run_dir)
os.execvp(self.cmd[0], self.cmd)
finally:
pkdlog('{}: execvp error: {} errno={}', self.jid, e.strerror, e.errno)
sys.exit(1)
except BaseException as e:
with open(str(self.run_dir.join(template_common.RUN_LOG)), 'a') as f:
f.write('{}: error starting simulation: {}'.format(self.jid, e))
raise
def _error_during_start(self, exception, stack):
"""An exception happened, log what you can.
Callback from implementations
POSIT: job already locked or in subprocess (see Background._start)
"""
try:
with open(str(self.run_dir.join(template_common.RUN_LOG)), 'a') as f:
f.write(
'{}: error starting simulation: {}\n{}'.format(
self.jid,
exception,
stack,
),
)
simulation_db.write_status('error', self.run_dir)
pkdlog('{}: unable to start job: {} {}', self.jid, exception, stack)
except Exception:
pass