def post(self, user, pipeline_identifier):
(boutiques_descriptor_path, descriptor_type
), error = get_original_descriptor_path_and_type(pipeline_identifier)
if error:
return marshal(error), 400
body = request.get_json(force=True)
invocation = body['invocation']
queries = body['queries']
inv_tmp = tempfile.NamedTemporaryFile(mode="r+", delete=True)
try:
json.dump(invocation, inv_tmp)
inv_tmp.flush()
executor = LocalExecutor(boutiques_descriptor_path, inv_tmp.name, {
"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True
})
query_results = []
for query in queries:
query_results += [evaluateEngine(executor, query)]
inv_tmp.close()
return query_results[0] if len(query_results) == 1 else query_results, 200
except Exception as error:
inv_tmp.close()
raise error
def evaluate(*params):
parser = ArgumentParser("Evaluates parameter values for a descriptor"
" and invocation")
parser.add_argument("descriptor",
action="store",
help="The Boutiques descriptor as a JSON file, JSON "
"string or Zenodo ID (prefixed by 'zenodo.').")
parser.add_argument("invocation",
action="store",
help="Input JSON complying to invocation.")
parser.add_argument("query",
action="store",
nargs="*",
help="The query to be performed. Simply request keys "
"from the descriptor (i.e. output-files), and chain "
"together queries (i.e. id=myfile or optional=false) "
"slashes between them and commas connecting them. "
"(i.e. output-files/optional=false,id=myfile). "
"Perform multiple queries by separating them with a "
"space.")
result = parser.parse_args(params)
# Generate object that will parse the invocation and descriptor
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(result.descriptor, result.invocation, {
"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True
})
from boutiques.evaluate import evaluateEngine
query_results = []
for query in result.query:
query_results += [evaluateEngine(executor, query)]
return query_results[0] if len(query_results) == 1 else query_results
def test_example_requires_group_complete_x10(self):
descriptor = op.join(op.split(bfile)[0],
'tests/docopt/valid/test_valid.json')
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(descriptor, None,
{"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True,
"skipDataCollect": True,
"requireComplete": True})
# Can't create descriptors with mutex group but only one valid example
# Bosh example is inherently random,
# Couldn't even inject prederemined input to executor.in_dict
# because _randomFillInDict clears it
executor.generateRandomParams(100)
self.assertGreater(len(executor.in_dict), 0)
def example(*params):
parser = parser_example()
results = parser.parse_args(params)
descriptor = results.descriptor
valid = invocation(descriptor)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(descriptor, None,
{"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True,
"skipDataCollect": True,
"requireComplete": results.complete})
executor.generateRandomParams(1)
return json.dumps(executor.in_dict, indent=4, sort_keys=True)
def example(*params):
parser = ArgumentParser("Generates example invocation from a valid"
"descriptor")
parser.add_argument("descriptor", action="store",
help="The Boutiques descriptor as a JSON file, "
"JSON string or Zenodo ID (prefixed by 'zenodo.').")
parser.add_argument("-c", "--complete", action="store_true",
help="Include optional parameters.")
results = parser.parse_args(params)
descriptor = results.descriptor
valid = invocation(descriptor)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(descriptor, None,
{"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True,
"skipDataCollect": True,
"requireComplete": results.complete})
executor.generateRandomParams(1)
return json.dumps(executor.in_dict, indent=4, sort_keys=True)
def evaluate(*params):
parser = parser_evaluate()
result = parser.parse_args(params)
# Generate object that will parse the invocation and descriptor
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(result.descriptor, result.invocation,
{"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True,
"skipDataCollect": True})
from boutiques.evaluate import evaluateEngine
query_results = []
for query in result.query:
query_results += [evaluateEngine(executor, query)]
return query_results[0] if len(query_results) == 1 else query_results
def get_bosh_cmdline(descriptor_filepath, invocation_filepath):
executor = LocalExecutor(descriptor_filepath, {
"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True,
})
executor.readInput(invocation_filepath)
old_stdout = sys.stdout
sys.stdout = mystdout = StringIO()
executor.printCmdLine()
sys.stdout = old_stdout
return mystdout.getvalue()[19:]
def execute(*params):
parser = ArgumentParser("Boutiques local executor", add_help=False)
parser.add_argument("mode",
action="store",
help="Mode of operation to use. Launch: takes a "
"set of inputs compliant with invocation schema "
"and launches the tool. Simulate: shows sample "
"command-lines based on the provided descriptor"
" based on provided or randomly generated inputs. "
"Prepare: pulls the Docker or Singularity container "
"image for a given descriptor. ",
choices=["launch", "simulate", "prepare"])
parser.add_argument("--help",
"-h",
action="store_true",
help="show this help message and exit")
helps = any([True for ht in ["--help", "-h"] if ht in params])
if len(params) <= 1 and helps:
parser.print_help()
raise SystemExit
args, params = parser.parse_known_args(params)
mode = args.mode
params += ["--help"] if args.help is True else []
if mode == "launch":
parser = ArgumentParser("Launches an invocation.")
parser.add_argument(
"descriptor",
action="store",
help="The Boutiques descriptor as a JSON file, "
"JSON string or Zenodo ID (prefixed by 'zenodo.').")
parser.add_argument("invocation",
action="store",
help="Input JSON complying to invocation.")
parser.add_argument("-v",
"--volumes",
action="append",
type=str,
help="Volumes to mount when launching the "
"container. Format consistently the following:"
" /a:/b will mount local directory /a to "
"container directory /b.")
parser.add_argument("-x",
"--debug",
action="store_true",
help="Keeps temporary scripts used during "
"execution, and prints additional debug "
"messages.")
parser.add_argument("-u",
"--user",
action="store_true",
help="Runs the container as local user ({0})"
" instead of root.".format(os.getenv("USER")))
parser.add_argument("-s",
"--stream",
action="store_true",
help="Streams stdout and stderr in real time "
"during execution.")
parser.add_argument("--imagepath",
action="store",
help="Path to Singularity image. "
"If not specified, will use current directory.")
parser.add_argument("--skip-data-collection",
action="store_true",
help="Skips execution data collection and saving"
"to cache.")
parser.add_argument("--provenance",
action="store",
type=json.loads,
help="Append JSON object to the generated record.")
force_group = parser.add_mutually_exclusive_group()
force_group.add_argument("--force-docker",
action="store_true",
help="Tries to run Singularity images with "
"Docker. This only works if the image is on"
"Docker Hub, i.e. has index docker://")
force_group.add_argument("--force-singularity",
action="store_true",
help="Tries to run Docker images with "
"Singularity.")
results = parser.parse_args(params)
descriptor = results.descriptor
inp = results.invocation
# Validate invocation and descriptor
valid = invocation(descriptor, '-i', inp)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, inp, {
"forcePathType": True,
"debug": results.debug,
"changeUser": results.user,
"stream": results.stream,
"imagePath": results.imagepath,
"skipDataCollect": results.skip_data_collection,
"forceDocker": results.force_docker,
"forceSingularity": results.force_singularity,
"provenance": results.provenance
})
# Execute it
return executor.execute(results.volumes)
if mode == "simulate":
parser = ArgumentParser("Simulates an invocation.")
parser.add_argument(
"descriptor",
action="store",
help="The Boutiques descriptor as a JSON file, "
"JSON string or Zenodo ID (prefixed by 'zenodo.').")
parser.add_argument("-i",
"--input",
action="store",
help="Input JSON complying to invocation.")
parser.add_argument("-j",
"--json",
action="store_true",
help="Flag to generate invocation in JSON format.")
parser.add_argument("-c",
"--complete",
action="store_true",
help="Include optional parameters.")
results = parser.parse_args(params)
descriptor = results.descriptor
# Do some basic input scrubbing
inp = results.input
arguments = [descriptor]
if inp:
arguments.append('-i')
arguments.append(inp)
valid = invocation(*arguments)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, inp, {
"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True,
"skipDataCollect": True,
"requireComplete": results.complete
})
if not inp:
executor.generateRandomParams(1)
if results.json:
sout = [json.dumps(executor.in_dict, indent=4, sort_keys=True)]
print(sout[0])
else:
executor.printCmdLine()
sout = executor.cmd_line
# for consistency with execute
# Adding hide to "container location" field since it's an invalid
# value, can parse that to hide the summary print
return ExecutorOutput(os.linesep.join(sout), "", 0, "", [], [],
os.linesep.join(sout), "", "hide")
if mode == "prepare":
parser = ArgumentParser("Pulls the container image for a given "
"descriptor")
parser.add_argument(
"descriptor",
action="store",
help="The Boutiques descriptor as a JSON file, "
"JSON string or Zenodo ID (prefixed by 'zenodo.').")
parser.add_argument("-x",
"--debug",
action="store_true",
help="Keeps temporary scripts used during "
"execution, and prints additional debug "
"messages.")
parser.add_argument("-s",
"--stream",
action="store_true",
help="Streams stdout and stderr in real time "
"during execution.")
parser.add_argument("--imagepath",
action="store",
help="Path to Singularity image. "
"If not specified, will use current directory.")
results = parser.parse_args(params)
descriptor = results.descriptor
# Validate descriptor
valid = invocation(descriptor)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, None, {
"forcePathType": True,
"debug": results.debug,
"stream": results.stream,
"imagePath": results.imagepath,
"skipDataCollect": True
})
container_location = executor.prepare()[1]
print("Container location: " + container_location)
# Adding hide to "container location" field since it's an invalid
# value, and we can parse that to hide the summary print
return ExecutorOutput(container_location, "", 0, "", [], [], "", "",
"hide")
def execute(*params):
parser = ArgumentParser("Boutiques local executor", add_help=False)
parser.add_argument("mode",
action="store",
help="Mode of operation to use. Launch: takes a "
"set of inputs compliant with invocation schema "
"and launches the tool. Simulate: shows sample "
"command-lines based on the provided descriptor"
" based on provided or randomly generated "
"inputs.",
choices=["launch", "simulate"])
parser.add_argument("--help",
"-h",
action="store_true",
help="show this help message and exit")
helps = any([True for ht in ["--help", "-h"] if ht in params])
if len(params) <= 1 and helps:
parser.print_help()
raise SystemExit
args, params = parser.parse_known_args(params)
mode = args.mode
params += ["--help"] if args.help is True else []
if mode == "launch":
parser = ArgumentParser("Launches an invocation.")
parser.add_argument("descriptor",
action="store",
help="The Boutiques descriptor.")
parser.add_argument("input",
action="store",
help="Input JSON complying to invocation.")
parser.add_argument("-v",
"--volumes",
action="store",
type=str,
help="Volumes to mount when launching the "
"container. Format consistently the following:"
" /a:/b will mount local direcotry /a to "
"container directory /b.",
nargs="*")
parser.add_argument("-x",
"--debug",
action="store_true",
help="Keeps temporary scripts used during "
"execution.")
parser.add_argument(
"-u",
"--user",
action="store_true",
help="Runs the container as local user ({0}) instead of root.".
format(os.getenv("USER")))
results = parser.parse_args(params)
descriptor = results.descriptor
# Do some basic input scrubbing
inp = results.input
if not os.path.isfile(inp):
raise SystemExit("Input file {} does not exist".format(inp))
if not inp.endswith(".json"):
raise SystemExit("Input file {} must end in json".format(inp))
if not os.path.isfile(descriptor):
raise SystemExit(
"JSON descriptor {} does not exist".format(descriptor))
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, {
"forcePathType": True,
"destroyTempScripts": not results.debug,
"changeUser": results.user
})
executor.readInput(inp)
# Execute it
exit_code = executor.execute(results.volumes)
if exit_code:
raise SystemExit(exit_code)
if mode == "simulate":
parser = ArgumentParser("Simulates an invocation.")
parser.add_argument("descriptor",
action="store",
help="The Boutiques descriptor.")
parser.add_argument("-i",
"--input",
action="store",
help="Input JSON complying to invocation.")
parser.add_argument("-r",
"--random",
action="store",
type=int,
nargs="*",
help="Generate random set of inputs.")
results = parser.parse_args(params)
descriptor = results.descriptor
# Do some basic input scrubbing
inp = results.input
rand = results.random is not None
numb = results.random[0] if rand and len(results.random) > 0 else 1
if numb and numb < 1:
raise SystemExit("--number value must be positive.")
if rand and inp:
raise SystemExit(
"--random setting and --input value cannot be used together.")
if inp and not os.path.isfile(inp):
raise SystemExit("Input file {} does not exist.".format(inp))
if inp and not inp.endswith(".json"):
raise SystemExit("Input file {} must end in 'json'.".format(inp))
if not os.path.isfile(descriptor):
raise SystemExit(
"JSON descriptor {} does not seem to exist.".format(
descriptor))
if not rand and not inp:
raise SystemExit("The default mode requires an input (-i).")
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(descriptor, {
"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True
})
if rand:
executor.generateRandomParams(numb)
executor.printCmdLine()
else:
executor.readInput(inp)
executor.printCmdLine()
def execute(*params):
parser = ArgumentParser("Boutiques local executor", add_help=False)
parser.add_argument("mode",
action="store",
help="Mode of operation to use. Launch: takes a "
"set of inputs compliant with invocation schema "
"and launches the tool. Simulate: shows sample "
"command-lines based on the provided descriptor"
" based on provided or randomly generated inputs. "
"Prepare: pulls the Docker or Singularity container "
"image for a given descriptor. ",
choices=["launch", "simulate", "prepare"])
parser.add_argument("--help",
"-h",
action="store_true",
help="show this help message and exit")
helps = any([True for ht in ["--help", "-h"] if ht in params])
if len(params) <= 1 and helps:
parser.print_help()
raise SystemExit
args, params = parser.parse_known_args(params)
mode = args.mode
params += ["--help"] if args.help is True else []
if mode == "launch":
parser = ArgumentParser("Launches an invocation.")
parser.add_argument(
"descriptor",
action="store",
help="The Boutiques descriptor as a JSON file, "
"JSON string or Zenodo ID (prefixed by 'zenodo.').")
parser.add_argument("invocation",
action="store",
help="Input JSON complying to invocation.")
parser.add_argument("-v",
"--volumes",
action="store",
type=str,
help="Volumes to mount when launching the "
"container. Format consistently the following:"
" /a:/b will mount local directory /a to "
"container directory /b.",
nargs="*")
parser.add_argument("-x",
"--debug",
action="store_true",
help="Keeps temporary scripts used during "
"execution, and prints additional debug "
"messages.")
parser.add_argument("-u",
"--user",
action="store_true",
help="Runs the container as local user ({0})"
" instead of root.".format(os.getenv("USER")))
parser.add_argument("-s",
"--stream",
action="store_true",
help="Streams stdout and stderr in real time "
"during execution.")
parser.add_argument("--imagepath",
action="store",
help="Location of Singularity image "
"(default is current directory).")
results = parser.parse_args(params)
descriptor = results.descriptor
inp = results.invocation
# Validate invocation and descriptor
valid = invocation(descriptor, '-i', inp)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, inp, {
"forcePathType": True,
"debug": results.debug,
"changeUser": results.user,
"stream": results.stream,
"imagePath": results.imagepath
})
# Execute it
return executor.execute(results.volumes)
if mode == "simulate":
parser = ArgumentParser("Simulates an invocation.")
parser.add_argument(
"descriptor",
action="store",
help="The Boutiques descriptor as a JSON file, "
"JSON string or Zenodo ID (prefixed by 'zenodo.').")
parser.add_argument("-i",
"--input",
action="store",
help="Input JSON complying to invocation.")
parser.add_argument("-r",
"--random",
action="store",
type=int,
nargs="*",
help="Generate random set of inputs.")
results = parser.parse_args(params)
descriptor = results.descriptor
# Do some basic input scrubbing
inp = results.input
rand = results.random is not None
numb = results.random[0] if rand and len(results.random) > 0 else 1
if numb and numb < 1:
raise_error(SystemExit, "--number value must be positive.")
# raise SystemExit("--number value must be positive.")
if rand and inp:
raise_error(
SystemExit, "--random setting and --input value cannot "
"be used together.")
if inp and not os.path.isfile(inp):
raise_error(SystemExit,
"Input file {} does not exist.".format(inp))
if inp and not inp.endswith(".json"):
raise_error(SystemExit,
"Input file {} must end in 'json'.".format(inp))
if not rand and not inp:
raise_error(SystemExit, "The default mode requires an input (-i).")
valid = invocation(descriptor, '-i', inp) if inp else\
invocation(descriptor)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(descriptor, inp, {
"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True
})
if rand:
executor.generateRandomParams(numb)
executor.printCmdLine()
# for consistency with execute
# Adding hide to "container location" field since it's an invalid
# value, and we can parse that to hide the summary print
return ExecutorOutput(os.linesep.join(executor.cmd_line), "", 0, "",
[], [], "", "", "hide")
if mode == "prepare":
parser = ArgumentParser("Pulls the container image for a given "
"descriptor")
parser.add_argument(
"descriptor",
action="store",
help="The Boutiques descriptor as a JSON file, "
"JSON string or Zenodo ID (prefixed by 'zenodo.').")
parser.add_argument("-x",
"--debug",
action="store_true",
help="Keeps temporary scripts used during "
"execution, and prints additional debug "
"messages.")
parser.add_argument("-s",
"--stream",
action="store_true",
help="Streams stdout and stderr in real time "
"during execution.")
parser.add_argument("--imagepath",
action="store",
help="Location of Singularity image "
"(default is current directory).")
results = parser.parse_args(params)
descriptor = results.descriptor
# Validate descriptor
valid = invocation(descriptor)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, None, {
"forcePathType": True,
"debug": results.debug,
"stream": results.stream,
"imagePath": results.imagepath
})
container_location = executor.prepare()[1]
print("Container location: " + container_location)
# Adding hide to "container location" field since it's an invalid
# value, and we can parse that to hide the summary print
return ExecutorOutput(container_location, "", 0, "", [], [], "", "",
"hide")
def execute(*params):
parser = parser_execute()
helps = any([True for ht in ["--help", "-h"] if ht in params])
if len(params) <= 1 and helps:
parser.print_help()
raise SystemExit
args, params = parser.parse_known_args(params)
mode = args.mode
params += ["--help"] if args.help is True else []
if mode == "launch":
parser = parser_executeLaunch()
results = parser.parse_args(params)
descriptor = results.descriptor
inp = results.invocation
# Validate invocation and descriptor
arguments = [descriptor, '-i', inp]
if results.sandbox:
arguments.append('--sandbox')
valid = invocation(*arguments)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, inp, {
"forcePathType": True,
"debug": results.debug,
"changeUser": results.user,
"stream": results.stream,
"imagePath": results.imagepath,
"skipDataCollect": results.skip_data_collection,
"forceDocker": results.force_docker,
"forceSingularity": results.force_singularity,
"provenance": results.provenance,
"noContainer": results.no_container,
"sandbox": results.sandbox
})
# Execute it
return executor.execute(results.volumes)
if mode == "simulate":
parser = parser_executeSimulate()
results = parser.parse_args(params)
descriptor = results.descriptor
# Do some basic input scrubbing
inp = results.input
arguments = [descriptor]
if inp:
arguments.append('-i')
arguments.append(inp)
if results.sandbox:
arguments.append('--sandbox')
valid = invocation(*arguments)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, inp, {
"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True,
"skipDataCollect": True,
"requireComplete": results.complete,
"sandbox": results.sandbox
})
if not inp:
# Add optional inputs with default-value to inputs_dict,
# which is then populated with random params
executor.in_dict = addDefaultValues(executor.desc_dict, {})
executor.generateRandomParams(generateCmdLineFromInDict=True)
if results.json:
sout = [
json.dumps(customSortInvocationByInput(executor.in_dict,
descriptor),
indent=4)
]
print(sout[0])
else:
executor.printCmdLine()
sout = executor.cmd_line
# for consistency with execute
# Adding hide to "container location" field since it's an invalid
# value, can parse that to hide the summary print
return ExecutorOutput(os.linesep.join(sout), "", 0, "", [], [],
os.linesep.join(sout), "", "hide")
if mode == "prepare":
parser = parser_executePrepare()
results = parser.parse_args(params)
descriptor = results.descriptor
# Validate descriptor
arguments = [descriptor]
if results.sandbox:
arguments.append('--sandbox')
valid = invocation(*arguments)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, None, {
"forcePathType": True,
"debug": results.debug,
"stream": results.stream,
"imagePath": results.imagepath,
"skipDataCollect": True,
"sandbox": results.sandbox
})
container_location = executor.prepare()[1]
print("Container location: " + container_location)
# Adding hide to "container location" field since it's an invalid
# value, and we can parse that to hide the summary print
return ExecutorOutput(container_location, "", 0, "", [], [], "", "",
"hide")
