def submit_job(self, app):
"""
Run an `Application` instance as a local process.
:see: `LRMS.submit_job`
"""
# Update current resource usage to check how many jobs are
# running in there. Please note that for consistency with
# other backends, these updated information are not kept!
try:
self.transport.connect()
except gc3libs.exceptions.TransportError as ex:
raise gc3libs.exceptions.LRMSSubmitError(
"Unable to access shellcmd resource at %s: %s" %
(self.frontend, str(ex)))
job_infos = self._get_persisted_resource_state()
free_slots = self.max_cores - self._compute_used_cores(job_infos)
available_memory = self.total_memory - \
self._compute_used_memory(job_infos)
if self.free_slots == 0 or free_slots == 0:
# XXX: We shouldn't check for self.free_slots !
raise gc3libs.exceptions.LRMSSubmitError(
"Resource %s already running maximum allowed number of jobs"
" (%s). Increase 'max_cores' to raise." %
(self.name, self.max_cores))
if app.requested_memory and \
(available_memory < app.requested_memory or
self.available_memory < app.requested_memory):
raise gc3libs.exceptions.LRMSSubmitError(
"Resource %s does not have enough available memory:"
" %s requested, but only %s available."
% (self.name,
app.requested_memory.to_str('%g%s', unit=Memory.MB),
available_memory.to_str('%g%s', unit=Memory.MB),)
)
log.debug("Executing local command '%s' ...",
str.join(" ", app.arguments))
# Check if spooldir is a valid directory
if not self.spooldir:
ex, stdout, stderr = self.transport.execute_command(
'cd "$TMPDIR" && pwd')
if ex != 0 or stdout.strip() == '' or not stdout[0] == '/':
log.debug(
"Unable to recover a valid absolute path for spooldir."
" Using `/var/tmp`.")
self.spooldir = '/var/tmp'
else:
self.spooldir = stdout.strip()
# determine execution directory
exit_code, stdout, stderr = self.transport.execute_command(
"mktemp -d %s " % posixpath.join(
self.spooldir, 'gc3libs.XXXXXX'))
if exit_code != 0:
log.error(
"Error creating temporary directory on host %s: %s",
self.frontend, stderr)
log.debug('Freeing resources used by failed application')
self.free(app)
raise gc3libs.exceptions.LRMSSubmitError(
"Error creating temporary directory on host %s: %s",
self.frontend, stderr)
execdir = stdout.strip()
app.execution.lrms_execdir = execdir
# Copy input files to remote dir
for local_path, remote_path in app.inputs.items():
if local_path.scheme != 'file':
continue
remote_path = posixpath.join(execdir, remote_path)
remote_parent = os.path.dirname(remote_path)
try:
if (remote_parent not in ['', '.']
and not self.transport.exists(remote_parent)):
log.debug("Making remote directory '%s'", remote_parent)
self.transport.makedirs(remote_parent)
log.debug("Transferring file '%s' to '%s'",
local_path.path, remote_path)
self.transport.put(local_path.path, remote_path)
# preserve execute permission on input files
if os.access(local_path.path, os.X_OK):
self.transport.chmod(remote_path, 0o755)
except:
log.critical(
"Copying input file '%s' to remote host '%s' failed",
local_path.path, self.frontend)
log.debug('Cleaning up failed application')
self.free(app)
raise
# try to ensure that a local executable really has
# execute permissions, but ignore failures (might be a
# link to a file we do not own)
if app.arguments[0].startswith('./'):
try:
self.transport.chmod(
posixpath.join(execdir, app.arguments[0][2:]),
0o755)
# os.chmod(app.arguments[0], 0755)
except:
log.error(
"Failed setting execution flag on remote file '%s'",
posixpath.join(execdir, app.arguments[0]))
# set up redirection
redirection_arguments = ''
if app.stdin is not None:
# stdin = open(app.stdin, 'r')
redirection_arguments += " <%s" % app.stdin
if app.stdout is not None:
redirection_arguments += " >%s" % app.stdout
stdout_dir = os.path.dirname(app.stdout)
if stdout_dir:
self.transport.makedirs(posixpath.join(execdir, stdout_dir))
if app.join:
redirection_arguments += " 2>&1"
else:
if app.stderr is not None:
redirection_arguments += " 2>%s" % app.stderr
stderr_dir = os.path.dirname(app.stderr)
if stderr_dir:
self.transport.makedirs(posixpath.join(execdir, stderr_dir))
# set up environment
env_commands = []
for k, v in app.environment.iteritems():
env_commands.append(
"export {k}={v};"
.format(k=sh_quote_safe(k), v=sh_quote_unsafe(v)))
# Create the directory in which pid, output and wrapper script
# files will be stored
wrapper_dir = posixpath.join(
execdir,
ShellcmdLrms.WRAPPER_DIR)
if not self.transport.isdir(wrapper_dir):
try:
self.transport.makedirs(wrapper_dir)
except:
log.error("Failed creating remote folder '%s'"
% wrapper_dir)
self.free(app)
raise
# Set up scripts to download/upload the swift/http files
downloadfiles = []
uploadfiles = []
wrapper_downloader_filename = posixpath.join(
wrapper_dir,
ShellcmdLrms.WRAPPER_DOWNLOADER)
for url, outfile in app.inputs.items():
if url.scheme in ['swift', 'swifts', 'swt', 'swts', 'http', 'https']:
downloadfiles.append("python '%s' download '%s' '%s'" % (wrapper_downloader_filename, str(url), outfile))
for infile, url in app.outputs.items():
if url.scheme in ['swift', 'swt', 'swifts', 'swts']:
uploadfiles.append("python '%s' upload '%s' '%s'" % (wrapper_downloader_filename, str(url), infile))
if downloadfiles or uploadfiles:
# Also copy the downloader.
with open(resource_filename(Requirement.parse("gc3pie"),
"gc3libs/etc/downloader.py")) as fd:
wrapper_downloader = self.transport.open(
wrapper_downloader_filename, 'w')
wrapper_downloader.write(fd.read())
wrapper_downloader.close()
# Build
pidfilename = posixpath.join(wrapper_dir,
ShellcmdLrms.WRAPPER_PID)
wrapper_output_filename = posixpath.join(
wrapper_dir,
ShellcmdLrms.WRAPPER_OUTPUT_FILENAME)
wrapper_script_fname = posixpath.join(
wrapper_dir,
ShellcmdLrms.WRAPPER_SCRIPT)
try:
# Create the wrapper script
wrapper_script = self.transport.open(
wrapper_script_fname, 'w')
commands = (
r"""#!/bin/sh
echo $$ >{pidfilename}
cd {execdir}
exec {redirections}
{environment}
{downloadfiles}
'{time_cmd}' -o '{wrapper_out}' -f '{fmt}' {command}
rc=$?
{uploadfiles}
rc2=$?
if [ $rc -ne 0 ]; then exit $rc; else exit $rc2; fi
""".format(
pidfilename=pidfilename,
execdir=execdir,
time_cmd=self.time_cmd,
wrapper_out=wrapper_output_filename,
fmt=ShellcmdLrms.TIMEFMT,
redirections=redirection_arguments,
environment=str.join('\n', env_commands),
downloadfiles=str.join('\n', downloadfiles),
uploadfiles=str.join('\n', uploadfiles),
command=(str.join(' ',
(sh_quote_unsafe(arg)
for arg in app.arguments))),
))
wrapper_script.write(commands)
wrapper_script.close()
#log.info("Wrapper script: <<<%s>>>", commands)
except gc3libs.exceptions.TransportError:
log.error("Freeing resources used by failed application")
self.free(app)
raise
try:
self.transport.chmod(wrapper_script_fname, 0o755)
# Execute the script in background
self.transport.execute_command(wrapper_script_fname, detach=True)
except gc3libs.exceptions.TransportError:
log.error("Freeing resources used by failed application")
self.free(app)
raise
# Just after the script has been started the pidfile should be
# filled in with the correct pid.
#
# However, the script can have not been able to write the
# pidfile yet, so we have to wait a little bit for it...
pidfile = None
for retry in gc3libs.utils.ExponentialBackoff():
try:
pidfile = self.transport.open(pidfilename, 'r')
break
except gc3libs.exceptions.TransportError as ex:
if '[Errno 2]' in str(ex): # no such file or directory
time.sleep(retry)
continue
else:
raise
if pidfile is None:
# XXX: probably self.free(app) should go here as well
raise gc3libs.exceptions.LRMSSubmitError(
"Unable to get PID file of submitted process from"
" execution directory `%s`: %s"
% (execdir, pidfilename))
pid = pidfile.read().strip()
try:
pid = int(pid)
except ValueError:
# XXX: probably self.free(app) should go here as well
pidfile.close()
raise gc3libs.exceptions.LRMSSubmitError(
"Invalid pid `%s` in pidfile %s." % (pid, pidfilename))
pidfile.close()
# Update application and current resources
app.execution.lrms_jobid = pid
# We don't need to update free_slots since its value is
# checked at runtime.
if app.requested_memory:
self.available_memory -= app.requested_memory
self.job_infos[pid] = {
'requested_cores': app.requested_cores,
'requested_memory': app.requested_memory,
'execution_dir': execdir,
'terminated': False,
}
self._update_job_resource_file(pid, self.job_infos[pid])
return app
def submit_job(self, app):
"""This method will create a remote directory to store job's
sandbox, and will copy the sandbox in there.
"""
job = app.execution
# Create the remote directory.
try:
self.transport.connect()
cmd = "mkdir -p $HOME/.gc3pie_jobs;" \
" mktemp -d $HOME/.gc3pie_jobs/lrms_job.XXXXXXXXXX"
log.info("Creating remote temporary folder: command '%s' " % cmd)
exit_code, stdout, stderr = self.transport.execute_command(cmd)
if exit_code == 0:
ssh_remote_folder = stdout.split('\n')[0]
else:
raise gc3libs.exceptions.LRMSError(
"Failed executing command '%s' on resource '%s';"
" exit code: %d, stderr: '%s'."
% (cmd, self.name, exit_code, stderr))
except gc3libs.exceptions.TransportError:
raise
except:
raise
# Copy the input file(s) to remote directory.
for local_path, remote_path in app.inputs.items():
remote_path = os.path.join(ssh_remote_folder, remote_path)
remote_parent = os.path.dirname(remote_path)
try:
if remote_parent not in ['', '.']:
log.debug("Making remote directory '%s'",
remote_parent)
self.transport.makedirs(remote_parent)
log.debug("Transferring file '%s' to '%s'",
local_path.path, remote_path)
self.transport.put(local_path.path, remote_path)
# preserve execute permission on input files
if os.access(local_path.path, os.X_OK):
self.transport.chmod(remote_path, 0o755)
except:
log.critical(
"Copying input file '%s' to remote cluster '%s' failed",
local_path.path, self.frontend)
raise
if app.arguments[0].startswith('./'):
gc3libs.log.debug("Making remote path '%s' executable.",
app.arguments[0])
self.transport.chmod(os.path.join(ssh_remote_folder,
app.arguments[0]), 0o755)
# if STDOUT/STDERR should be saved in a directory, ensure it
# exists (see Issue 495 for details)
for dest in (app.stdout, app.stderr):
if dest:
destdir = os.path.dirname(dest)
if destdir:
self.transport.makedirs(
posixpath.join(ssh_remote_folder, destdir))
try:
sub_cmd, aux_script = self._submit_command(app)
if aux_script != '':
# create temporary script name
script_filename = ('./script.%s.sh' % uuid.uuid4())
# save script to a temporary file and submit that one instead
local_script_file = tempfile.NamedTemporaryFile()
local_script_file.write('#!/bin/sh\n')
# Add preamble file
prologue = self.get_prologue_script(app)
if prologue:
local_script_file.write(prologue)
local_script_file.write(aux_script)
# Add epilogue files
epilogue = self.get_epilogue_script(app)
if epilogue:
local_script_file.write(epilogue)
local_script_file.flush()
# upload script to remote location
self.transport.put(
local_script_file.name,
os.path.join(ssh_remote_folder, script_filename))
# set execution mode on remote script
self.transport.chmod(
os.path.join(ssh_remote_folder, script_filename), 0o755)
# cleanup
local_script_file.close()
if os.path.exists(local_script_file.name):
os.unlink(local_script_file.name)
else:
# we still need a script name even if there is no
# script to submit
script_filename = ''
# Submit it
exit_code, stdout, stderr = self.transport.execute_command(
"/bin/sh -c %s" % sh_quote_safe('cd %s && %s %s' % (
ssh_remote_folder, sub_cmd, script_filename)))
if exit_code != 0:
raise gc3libs.exceptions.LRMSError(
"Failed executing command 'cd %s && %s %s' on resource"
" '%s'; exit code: %d, stderr: '%s'."
% (ssh_remote_folder, sub_cmd, script_filename,
self.name, exit_code, stderr))
jobid = self._parse_submit_output(stdout)
log.debug('Job submitted with jobid: %s', jobid)
job.execution_target = self.frontend
job.lrms_jobid = jobid
job.lrms_jobname = jobid
try:
if app.jobname:
job.lrms_jobname = app.jobname
except:
pass
if 'stdout' in app:
job.stdout_filename = app.stdout
else:
job.stdout_filename = '%s.o%s' % (job.lrms_jobname, jobid)
if app.join:
job.stderr_filename = job.stdout_filename
else:
if 'stderr' in app:
job.stderr_filename = app.stderr
else:
job.stderr_filename = '%s.e%s' % (job.lrms_jobname, jobid)
job.history.append('Submitted to %s @ %s, got jobid %s'
% (self._batchsys_name, self.name, jobid))
job.history.append("Submission command output:\n"
" === stdout ===\n%s"
" === stderr ===\n%s"
" === end ===\n"
% (stdout, stderr), 'pbs', 'qsub')
job.ssh_remote_folder = ssh_remote_folder
return job
except:
log.critical(
"Failure submitting job to resource '%s' - "
"see log file for errors", self.name)
raise
def submit_job(self, app):
"""This method will create a remote directory to store job's
sandbox, and will copy the sandbox in there.
"""
job = app.execution
# Create the remote directory.
self.transport.connect()
cmd = ("mkdir -p {0};"
" mktemp -d {0}/batch_job.XXXXXXXXXX".format(self.spooldir))
exit_code, stdout, stderr = self.transport.execute_command(cmd)
if exit_code != 0:
raise gc3libs.exceptions.SpoolDirError(
"Cannot create temporary job working directory"
" on resource '%s'; command '%s' exited"
" with code: %d and stderr: '%s'." %
(self.name, cmd, exit_code, stderr))
ssh_remote_folder = stdout.split('\n')[0]
# Copy the input file(s) to remote directory.
for local_path, remote_path in list(app.inputs.items()):
remote_path = os.path.join(ssh_remote_folder, remote_path)
remote_parent = os.path.dirname(remote_path)
try:
if remote_parent not in ['', '.']:
log.debug("Making remote directory '%s'", remote_parent)
self.transport.makedirs(remote_parent)
log.debug("Transferring file '%s' to '%s'", local_path.path,
remote_path)
self.transport.put(local_path.path, remote_path)
# preserve execute permission on input files
if os.access(local_path.path, os.X_OK):
self.transport.chmod(remote_path, 0o755)
except:
log.critical(
"Copying input file '%s' to remote cluster '%s' failed",
local_path.path, self.frontend)
raise
if app.arguments[0].startswith('./'):
gc3libs.log.debug("Making remote path '%s' executable.",
app.arguments[0])
self.transport.chmod(
os.path.join(ssh_remote_folder, app.arguments[0]), 0o755)
# if STDOUT/STDERR should be saved in a directory, ensure it
# exists (see Issue 495 for details)
for dest in (app.stdout, app.stderr):
if dest:
destdir = os.path.dirname(dest)
if destdir:
self.transport.makedirs(
posixpath.join(ssh_remote_folder, destdir))
try:
sub_cmd, aux_script = self._submit_command(app)
if aux_script != '':
# create temporary script name
script_filename = ('./script.%s.sh' % uuid.uuid4())
# save script to a temporary file and submit that one instead
local_script_file = tempfile.NamedTemporaryFile(mode='wt')
local_script_file.write('#!/bin/sh\n')
# Add preamble file
prologue = self.get_prologue_script(app)
if prologue:
local_script_file.write(prologue)
local_script_file.write(aux_script)
# Add epilogue files
epilogue = self.get_epilogue_script(app)
if epilogue:
local_script_file.write(epilogue)
local_script_file.flush()
# upload script to remote location
self.transport.put(
local_script_file.name,
os.path.join(ssh_remote_folder, script_filename))
# set execution mode on remote script
self.transport.chmod(
os.path.join(ssh_remote_folder, script_filename), 0o755)
# cleanup
local_script_file.close()
if os.path.exists(local_script_file.name):
os.unlink(local_script_file.name)
else:
# we still need a script name even if there is no
# script to submit
script_filename = ''
# Submit it
exit_code, stdout, stderr = self.transport.execute_command(
"/bin/sh -c %s" %
sh_quote_safe('cd %s && %s %s' %
(ssh_remote_folder, sub_cmd, script_filename)))
if exit_code != 0:
raise gc3libs.exceptions.LRMSError(
"Failed executing command 'cd %s && %s %s' on resource"
" '%s'; exit code: %d, stderr: '%s'." %
(ssh_remote_folder, sub_cmd, script_filename, self.name,
exit_code, stderr))
jobid = self._parse_submit_output(stdout)
log.debug('Job submitted with jobid: %s', jobid)
job.execution_target = self.frontend
job.lrms_jobid = jobid
job.lrms_jobname = jobid
try:
if app.jobname:
job.lrms_jobname = app.jobname
except:
pass
if 'stdout' in app:
job.stdout_filename = app.stdout
else:
job.stdout_filename = '%s.o%s' % (job.lrms_jobname, jobid)
if app.join:
job.stderr_filename = job.stdout_filename
else:
if 'stderr' in app:
job.stderr_filename = app.stderr
else:
job.stderr_filename = '%s.e%s' % (job.lrms_jobname, jobid)
job.history.append('Submitted to %s @ %s, got jobid %s' %
(self._batchsys_name, self.name, jobid))
job.history.append(
"Submission command output:\n"
" === stdout ===\n%s"
" === stderr ===\n%s"
" === end ===\n" % (stdout, stderr), 'pbs', 'qsub')
job.ssh_remote_folder = ssh_remote_folder
return job
except:
log.critical(
"Failure submitting job to resource '%s' - "
"see log file for errors", self.name)
raise
def submit_job(self, app):
"""
Run an `Application` instance as a local process.
:see: `LRMS.submit_job`
"""
# Update current resource usage to check how many jobs are
# running in there. Please note that for consistency with
# other backends, these updated information are not kept!
try:
self.transport.connect()
except gc3libs.exceptions.TransportError as ex:
raise gc3libs.exceptions.LRMSSubmitError(
"Unable to access shellcmd resource at %s: %s" %
(self.frontend, str(ex)))
job_infos = self._get_persisted_resource_state()
free_slots = self.max_cores - self._compute_used_cores(job_infos)
available_memory = self.total_memory - \
self._compute_used_memory(job_infos)
if self.free_slots == 0 or free_slots == 0:
# XXX: We shouldn't check for self.free_slots !
raise gc3libs.exceptions.LRMSSubmitError(
"Resource %s already running maximum allowed number of jobs"
" (%s). Increase 'max_cores' to raise." %
(self.name, self.max_cores))
if app.requested_memory and \
(available_memory < app.requested_memory or
self.available_memory < app.requested_memory):
raise gc3libs.exceptions.LRMSSubmitError(
"Resource %s does not have enough available memory:"
" %s requested, but only %s available." % (
self.name,
app.requested_memory.to_str('%g%s', unit=Memory.MB),
available_memory.to_str('%g%s', unit=Memory.MB),
))
log.debug("Executing local command '%s' ...",
str.join(" ", app.arguments))
# Check if spooldir is a valid directory
if not self.spooldir:
ex, stdout, stderr = self.transport.execute_command(
'cd "$TMPDIR" && pwd')
if ex != 0 or stdout.strip() == '' or not stdout[0] == '/':
log.debug(
"Unable to recover a valid absolute path for spooldir."
" Using `/var/tmp`.")
self.spooldir = '/var/tmp'
else:
self.spooldir = stdout.strip()
# determine execution directory
exit_code, stdout, stderr = self.transport.execute_command(
"mktemp -d %s " % posixpath.join(self.spooldir, 'gc3libs.XXXXXX'))
if exit_code != 0:
log.error("Error creating temporary directory on host %s: %s",
self.frontend, stderr)
log.debug('Freeing resources used by failed application')
self.free(app)
raise gc3libs.exceptions.LRMSSubmitError(
"Error creating temporary directory on host %s: %s",
self.frontend, stderr)
execdir = stdout.strip()
app.execution.lrms_execdir = execdir
# Copy input files to remote dir
for local_path, remote_path in app.inputs.items():
if local_path.scheme != 'file':
continue
remote_path = posixpath.join(execdir, remote_path)
remote_parent = os.path.dirname(remote_path)
try:
if (remote_parent not in ['', '.']
and not self.transport.exists(remote_parent)):
log.debug("Making remote directory '%s'", remote_parent)
self.transport.makedirs(remote_parent)
log.debug("Transferring file '%s' to '%s'", local_path.path,
remote_path)
self.transport.put(local_path.path, remote_path)
# preserve execute permission on input files
if os.access(local_path.path, os.X_OK):
self.transport.chmod(remote_path, 0o755)
except:
log.critical(
"Copying input file '%s' to remote host '%s' failed",
local_path.path, self.frontend)
log.debug('Cleaning up failed application')
self.free(app)
raise
# try to ensure that a local executable really has
# execute permissions, but ignore failures (might be a
# link to a file we do not own)
if app.arguments[0].startswith('./'):
try:
self.transport.chmod(
posixpath.join(execdir, app.arguments[0][2:]), 0o755)
# os.chmod(app.arguments[0], 0755)
except:
log.error("Failed setting execution flag on remote file '%s'",
posixpath.join(execdir, app.arguments[0]))
# set up redirection
redirection_arguments = ''
if app.stdin is not None:
# stdin = open(app.stdin, 'r')
redirection_arguments += " <%s" % app.stdin
if app.stdout is not None:
redirection_arguments += " >%s" % app.stdout
stdout_dir = os.path.dirname(app.stdout)
if stdout_dir:
self.transport.makedirs(posixpath.join(execdir, stdout_dir))
if app.join:
redirection_arguments += " 2>&1"
else:
if app.stderr is not None:
redirection_arguments += " 2>%s" % app.stderr
stderr_dir = os.path.dirname(app.stderr)
if stderr_dir:
self.transport.makedirs(posixpath.join(
execdir, stderr_dir))
# set up environment
env_commands = []
for k, v in app.environment.iteritems():
env_commands.append("export {k}={v};".format(k=sh_quote_safe(k),
v=sh_quote_unsafe(v)))
# Create the directory in which pid, output and wrapper script
# files will be stored
wrapper_dir = posixpath.join(execdir, ShellcmdLrms.WRAPPER_DIR)
if not self.transport.isdir(wrapper_dir):
try:
self.transport.makedirs(wrapper_dir)
except:
log.error("Failed creating remote folder '%s'" % wrapper_dir)
self.free(app)
raise
# Set up scripts to download/upload the swift/http files
downloadfiles = []
uploadfiles = []
wrapper_downloader_filename = posixpath.join(
wrapper_dir, ShellcmdLrms.WRAPPER_DOWNLOADER)
for url, outfile in app.inputs.items():
if url.scheme in [
'swift', 'swifts', 'swt', 'swts', 'http', 'https'
]:
downloadfiles.append(
"python '%s' download '%s' '%s'" %
(wrapper_downloader_filename, str(url), outfile))
for infile, url in app.outputs.items():
if url.scheme in ['swift', 'swt', 'swifts', 'swts']:
uploadfiles.append(
"python '%s' upload '%s' '%s'" %
(wrapper_downloader_filename, str(url), infile))
if downloadfiles or uploadfiles:
# Also copy the downloader.
with open(
resource_filename(Requirement.parse("gc3pie"),
"gc3libs/etc/downloader.py")) as fd:
wrapper_downloader = self.transport.open(
wrapper_downloader_filename, 'w')
wrapper_downloader.write(fd.read())
wrapper_downloader.close()
# Build
pidfilename = posixpath.join(wrapper_dir, ShellcmdLrms.WRAPPER_PID)
wrapper_output_filename = posixpath.join(
wrapper_dir, ShellcmdLrms.WRAPPER_OUTPUT_FILENAME)
wrapper_script_fname = posixpath.join(wrapper_dir,
ShellcmdLrms.WRAPPER_SCRIPT)
try:
# Create the wrapper script
wrapper_script = self.transport.open(wrapper_script_fname, 'w')
commands = (r"""#!/bin/sh
echo $$ >{pidfilename}
cd {execdir}
exec {redirections}
{environment}
{downloadfiles}
'{time_cmd}' -o '{wrapper_out}' -f '{fmt}' {command}
rc=$?
{uploadfiles}
rc2=$?
if [ $rc -ne 0 ]; then exit $rc; else exit $rc2; fi
""".format(
pidfilename=pidfilename,
execdir=execdir,
time_cmd=self.time_cmd,
wrapper_out=wrapper_output_filename,
fmt=ShellcmdLrms.TIMEFMT,
redirections=redirection_arguments,
environment=str.join('\n', env_commands),
downloadfiles=str.join('\n', downloadfiles),
uploadfiles=str.join('\n', uploadfiles),
command=(str.join(' ', (sh_quote_unsafe(arg)
for arg in app.arguments))),
))
wrapper_script.write(commands)
wrapper_script.close()
#log.info("Wrapper script: <<<%s>>>", commands)
except gc3libs.exceptions.TransportError:
log.error("Freeing resources used by failed application")
self.free(app)
raise
try:
self.transport.chmod(wrapper_script_fname, 0o755)
# Execute the script in background
self.transport.execute_command(wrapper_script_fname, detach=True)
except gc3libs.exceptions.TransportError:
log.error("Freeing resources used by failed application")
self.free(app)
raise
# Just after the script has been started the pidfile should be
# filled in with the correct pid.
#
# However, the script can have not been able to write the
# pidfile yet, so we have to wait a little bit for it...
pidfile = None
for retry in gc3libs.utils.ExponentialBackoff():
try:
pidfile = self.transport.open(pidfilename, 'r')
break
except gc3libs.exceptions.TransportError as ex:
if '[Errno 2]' in str(ex): # no such file or directory
time.sleep(retry)
continue
else:
raise
if pidfile is None:
# XXX: probably self.free(app) should go here as well
raise gc3libs.exceptions.LRMSSubmitError(
"Unable to get PID file of submitted process from"
" execution directory `%s`: %s" % (execdir, pidfilename))
pid = pidfile.read().strip()
try:
pid = int(pid)
except ValueError:
# XXX: probably self.free(app) should go here as well
pidfile.close()
raise gc3libs.exceptions.LRMSSubmitError(
"Invalid pid `%s` in pidfile %s." % (pid, pidfilename))
pidfile.close()
# Update application and current resources
app.execution.lrms_jobid = pid
# We don't need to update free_slots since its value is
# checked at runtime.
if app.requested_memory:
self.available_memory -= app.requested_memory
self.job_infos[pid] = {
'requested_cores': app.requested_cores,
'requested_memory': app.requested_memory,
'execution_dir': execdir,
'terminated': False,
}
self._update_job_resource_file(pid, self.job_infos[pid])
return app
