def post(self, user, pipeline_identifier):
(boutiques_descriptor_path, descriptor_type
), error = get_original_descriptor_path_and_type(pipeline_identifier)
if error:
return marshal(error), 400
body = request.get_json(force=True)
invocation = body['invocation']
queries = body['queries']
inv_tmp = tempfile.NamedTemporaryFile(mode="r+", delete=True)
try:
json.dump(invocation, inv_tmp)
inv_tmp.flush()
executor = LocalExecutor(boutiques_descriptor_path, inv_tmp.name, {
"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True
})
query_results = []
for query in queries:
query_results += [evaluateEngine(executor, query)]
inv_tmp.close()
return query_results[0] if len(query_results) == 1 else query_results, 200
except Exception as error:
inv_tmp.close()
raise error
def evaluate(*params):
parser = ArgumentParser("Evaluates parameter values for a descriptor"
" and invocation")
parser.add_argument("descriptor", action="store",
help="The Boutiques descriptor as a JSON file, JSON "
"string or Zenodo ID (prefixed by 'zenodo.').")
parser.add_argument("invocation", action="store",
help="Input JSON complying to invocation.")
parser.add_argument("query", action="store", nargs="*",
help="The query to be performed. Simply request keys "
"from the descriptor (i.e. output-files), and chain "
"together queries (i.e. id=myfile or optional=false) "
"slashes between them and commas connecting them. "
"(i.e. output-files/optional=false,id=myfile). "
"Perform multiple queries by separating them with a "
"space.")
result = parser.parse_args(params)
# Generate object that will parse the invocation and descriptor
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(result.descriptor, result.invocation,
{"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True,
"skipDataCollect": True})
from boutiques.evaluate import evaluateEngine
query_results = []
for query in result.query:
query_results += [evaluateEngine(executor, query)]
return query_results[0] if len(query_results) == 1 else query_results
def evaluate(*params):
parser = parser_evaluate()
result = parser.parse_args(params)
# Generate object that will parse the invocation and descriptor
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(result.descriptor, result.invocation,
{"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True,
"skipDataCollect": True})
from boutiques.evaluate import evaluateEngine
query_results = []
for query in result.query:
query_results += [evaluateEngine(executor, query)]
return query_results[0] if len(query_results) == 1 else query_results
def execute(self, mount_strings):
'''
The execute method runs the generated command line
(from either generateRandomParams or readInput)
If docker is specified, it will attempt to use it, instead
of local execution.
After execution, it checks for output file existence.
'''
command, exit_code, con = self.cmdLine[0], None, self.con or {}
# Check for Container image
conType, conImage = con.get('type'), con.get('image'),
conIndex = con.get("index")
conIsPresent = (conImage is not None)
# Export environment variables, if they are specified in the descriptor
envVars = {}
if 'environment-variables' in list(self.desc_dict.keys()):
variables = [(p['name'], p['value']) for p in
self.desc_dict['environment-variables']]
for (envVarName, envVarValue) in variables:
os.environ[envVarName] = envVarValue
envVars[envVarName] = envVarValue
# Container script constant name
# Note that docker/singularity cannot do a local volume
# mount of files starting with a '.', hence this one does not
millitime = int(time.time()*1000)
dsname = ('temp-' +
str(random.SystemRandom().randint(0, int(millitime))) +
"-" + str(millitime) + '.localExec.boshjob.sh')
dsname = op.realpath(dsname)
# If container is present, alter the command template accordingly
container_location = ""
container_command = ""
if conIsPresent:
if conType == 'docker':
# Pull the docker image
if self._localExecute("docker pull " + str(conImage))[1]:
container_location = "Local copy"
elif conType == 'singularity':
if not conIndex:
conIndex = "shub://"
elif not conIndex.endswith("://"):
conIndex = conIndex + "://"
conName = conImage.replace("/", "-").replace(":", "-") + ".simg"
if conName not in os.listdir('./'):
pull_loc = "\"{0}\" {1}{2}".format(conName,
conIndex,
conImage)
container_location = ("Pulled from {1} ({0} not found "
"in current"
"working director").format(conName,
pull_loc)
# Pull the singularity image
if self._localExecute("singularity pull --name " +
pull_loc)[1]:
raise ExecutorError("Could not pull Singularity image")
else:
container_location = "Local ({0})".format(conName)
conName = op.abspath(conName)
else:
raise ExecutorError('Unrecognized container'
' type: \"%s\"' % conType)
# Generate command script
uname, uid = pwd.getpwuid(os.getuid())[0], str(os.getuid())
# Adds the user to the container before executing
# the templated command line
userchange = '' if not self.changeUser else ("useradd --uid " +
uid + ' ' + uname +
"\n")
# If --changeUser was desired, run with su so that
# any output files are owned by the user instead of root
# Get the supported shell by the docker or singularity
if self.changeUser:
command = 'su ' + uname + ' -c ' + "\"{0}\"".format(command)
cmdString = "#!"+self.shell+" -l\n" + userchange + str(command)
with open(dsname, "w") as scrFile:
scrFile.write(cmdString)
# Ensure the script is executable
self._localExecute("chmod 755 " + dsname)
# Prepare extra environment variables
envString = ""
if envVars:
for (key, val) in list(envVars.items()):
envString += "SINGULARITYENV_{0}='{1}' ".format(key, val)
# Change launch (working) directory if desired
launchDir = self.launchDir
if launchDir is None:
launchDir = op.realpath('./')
launchDir = op.realpath(launchDir)
# Run it in docker
mount_strings = [] if not mount_strings else mount_strings
mount_strings = [op.realpath(m.split(":")[0])+":"+m.split(":")[1]
for m in mount_strings]
mount_strings.append(op.realpath('./') + ':' + launchDir)
if conType == 'docker':
envString = " "
if envVars:
for (key, val) in list(envVars.items()):
envString += " -e {0}='{1}' ".format(key, val)
# export mounts to docker string
docker_mounts = " -v ".join(m for m in mount_strings)
container_command = ('docker run --entrypoint=' + self.shell +
' --rm' + envString +
' -v ' + docker_mounts +
' -w ' + launchDir + ' ' +
str(conImage) + ' ' + dsname)
elif conType == 'singularity':
envString = ""
if envVars:
for (key, val) in list(envVars.items()):
envString += "SINGULARITYENV_{0}='{1}' ".format(key,
val)
# TODO: Singularity 2.4.6 default configuration binds: /proc,
# /sys, /dev, ${HOME}, /tmp, /var/tmp, /etc/localtime, and
# /etc/hosts. This means that any path down-stream shouldn't
# be bound on the command-line, as this will currently raise
# an exception. See:
# https://github.com/singularityware/singularity/issues/1469
#
# Previous bind string:
# singularity_mounts = " -B ".join(m for m in mount_strings)
def_mounts = ["/proc", "/sys", "/dev", "/tmp", "/var/tmp",
"/etc/localtime", "/etc/hosts",
op.realpath(op.expanduser('~')),
op.expanduser('~')]
# Ensures the set of paths provided has no overlap
compaths = list()
for idxm, m in enumerate(mount_strings):
for n in mount_strings[idxm:]:
if n != m:
tmp = op.dirname(op.commonprefix([n, m]))
if tmp != '/':
compaths += [tmp]
if not any(m.startswith(c) for c in compaths):
compaths += [m]
mount_strings = set(compaths)
# Only adds mount points for those not already included
singularity_mounts = ""
for m in mount_strings:
if not any(d in m for d in def_mounts):
singularity_mounts += "-B {0} ".format(m)
container_command = (envString + 'singularity exec '
'--cleanenv ' +
singularity_mounts +
' -W ' + launchDir + ' ' +
str(conName) + ' ' + dsname)
else:
raise ExecutorError('Unrecognized container type: '
'\"%s\"' % conType)
(stdout, stderr), exit_code = self._localExecute(container_command)
# Otherwise, just run command locally
else:
(stdout, stderr), exit_code = self._localExecute(command)
time.sleep(0.5) # Give the OS a (half) second to finish writing
# Destroy temporary docker script, if desired.
# By default, keep the script so the dev can look at it.
if conIsPresent and self.destroyTempScripts:
if os.path.isfile(dsname):
os.remove(dsname)
# Check for output files
missing_files = []
output_files = []
all_files = evaluateEngine(self, "output-files")
required_files = evaluateEngine(self, "output-files/optional=False")
optional_files = evaluateEngine(self, "output-files/optional=True")
for f in all_files.keys():
file_name = all_files[f]
fd = FileDescription(f, file_name, False)
if op.exists(file_name):
output_files.append(fd)
else: # file does not exist
if f in required_files.keys():
missing_files.append(fd)
# Set error messages
desc_err = ''
if 'error-codes' in list(self.desc_dict.keys()):
for err_elem in self.desc_dict['error-codes']:
if err_elem['code'] == exit_code:
desc_err = err_elem['description']
break
return ExecutorOutput(stdout,
stderr,
exit_code,
desc_err,
output_files,
missing_files,
command,
container_command, container_location)
