def example(*params):
parser = parser_example()
results = parser.parse_args(params)
descriptor = results.descriptor
arguments = [descriptor]
if results.sandbox:
arguments.append('--sandbox')
valid = invocation(*arguments)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, None, {
"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True,
"skipDataCollect": True,
"requireComplete": results.complete,
"sandbox": results.sandbox
})
executor.generateRandomParams()
return json.dumps(customSortInvocationByInput(executor.in_dict,
descriptor),
indent=4)
def example(*params):
parser = ArgumentParser("Generates example invocation from a valid"
"descriptor")
parser.add_argument("descriptor",
action="store",
help="The Boutiques descriptor as a JSON file, "
"JSON string or Zenodo ID (prefixed by 'zenodo.').")
parser.add_argument("-c",
"--complete",
action="store_true",
help="Include optional parameters.")
results = parser.parse_args(params)
descriptor = results.descriptor
valid = invocation(descriptor)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, None, {
"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True,
"skipDataCollect": True,
"requireComplete": results.complete
})
executor.generateRandomParams(1)
return json.dumps(executor.in_dict, indent=4, sort_keys=True)
def test_example_requires_group_complete_x10(self):
descriptor = op.join(op.split(bfile)[0],
'tests/docopt/valid/test_valid.json')
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(descriptor, None,
{"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True,
"skipDataCollect": True,
"requireComplete": True})
# Can't create descriptors with mutex group but only one valid example
# Bosh example is inherently random,
# Couldn't even inject prederemined input to executor.in_dict
# because _randomFillInDict clears it
executor.generateRandomParams(100)
self.assertGreater(len(executor.in_dict), 0)
def example(*params):
parser = parser_example()
results = parser.parse_args(params)
descriptor = results.descriptor
valid = invocation(descriptor)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(descriptor, None,
{"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True,
"skipDataCollect": True,
"requireComplete": results.complete})
executor.generateRandomParams(1)
return json.dumps(executor.in_dict, indent=4, sort_keys=True)
def execute(*params):
parser = ArgumentParser("Boutiques local executor", add_help=False)
parser.add_argument("mode",
action="store",
help="Mode of operation to use. Launch: takes a "
"set of inputs compliant with invocation schema "
"and launches the tool. Simulate: shows sample "
"command-lines based on the provided descriptor"
" based on provided or randomly generated inputs. "
"Prepare: pulls the Docker or Singularity container "
"image for a given descriptor. ",
choices=["launch", "simulate", "prepare"])
parser.add_argument("--help",
"-h",
action="store_true",
help="show this help message and exit")
helps = any([True for ht in ["--help", "-h"] if ht in params])
if len(params) <= 1 and helps:
parser.print_help()
raise SystemExit
args, params = parser.parse_known_args(params)
mode = args.mode
params += ["--help"] if args.help is True else []
if mode == "launch":
parser = ArgumentParser("Launches an invocation.")
parser.add_argument(
"descriptor",
action="store",
help="The Boutiques descriptor as a JSON file, "
"JSON string or Zenodo ID (prefixed by 'zenodo.').")
parser.add_argument("invocation",
action="store",
help="Input JSON complying to invocation.")
parser.add_argument("-v",
"--volumes",
action="append",
type=str,
help="Volumes to mount when launching the "
"container. Format consistently the following:"
" /a:/b will mount local directory /a to "
"container directory /b.")
parser.add_argument("-x",
"--debug",
action="store_true",
help="Keeps temporary scripts used during "
"execution, and prints additional debug "
"messages.")
parser.add_argument("-u",
"--user",
action="store_true",
help="Runs the container as local user ({0})"
" instead of root.".format(os.getenv("USER")))
parser.add_argument("-s",
"--stream",
action="store_true",
help="Streams stdout and stderr in real time "
"during execution.")
parser.add_argument("--imagepath",
action="store",
help="Path to Singularity image. "
"If not specified, will use current directory.")
parser.add_argument("--skip-data-collection",
action="store_true",
help="Skips execution data collection and saving"
"to cache.")
parser.add_argument("--provenance",
action="store",
type=json.loads,
help="Append JSON object to the generated record.")
force_group = parser.add_mutually_exclusive_group()
force_group.add_argument("--force-docker",
action="store_true",
help="Tries to run Singularity images with "
"Docker. This only works if the image is on"
"Docker Hub, i.e. has index docker://")
force_group.add_argument("--force-singularity",
action="store_true",
help="Tries to run Docker images with "
"Singularity.")
results = parser.parse_args(params)
descriptor = results.descriptor
inp = results.invocation
# Validate invocation and descriptor
valid = invocation(descriptor, '-i', inp)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, inp, {
"forcePathType": True,
"debug": results.debug,
"changeUser": results.user,
"stream": results.stream,
"imagePath": results.imagepath,
"skipDataCollect": results.skip_data_collection,
"forceDocker": results.force_docker,
"forceSingularity": results.force_singularity,
"provenance": results.provenance
})
# Execute it
return executor.execute(results.volumes)
if mode == "simulate":
parser = ArgumentParser("Simulates an invocation.")
parser.add_argument(
"descriptor",
action="store",
help="The Boutiques descriptor as a JSON file, "
"JSON string or Zenodo ID (prefixed by 'zenodo.').")
parser.add_argument("-i",
"--input",
action="store",
help="Input JSON complying to invocation.")
parser.add_argument("-j",
"--json",
action="store_true",
help="Flag to generate invocation in JSON format.")
parser.add_argument("-c",
"--complete",
action="store_true",
help="Include optional parameters.")
results = parser.parse_args(params)
descriptor = results.descriptor
# Do some basic input scrubbing
inp = results.input
arguments = [descriptor]
if inp:
arguments.append('-i')
arguments.append(inp)
valid = invocation(*arguments)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, inp, {
"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True,
"skipDataCollect": True,
"requireComplete": results.complete
})
if not inp:
executor.generateRandomParams(1)
if results.json:
sout = [json.dumps(executor.in_dict, indent=4, sort_keys=True)]
print(sout[0])
else:
executor.printCmdLine()
sout = executor.cmd_line
# for consistency with execute
# Adding hide to "container location" field since it's an invalid
# value, can parse that to hide the summary print
return ExecutorOutput(os.linesep.join(sout), "", 0, "", [], [],
os.linesep.join(sout), "", "hide")
if mode == "prepare":
parser = ArgumentParser("Pulls the container image for a given "
"descriptor")
parser.add_argument(
"descriptor",
action="store",
help="The Boutiques descriptor as a JSON file, "
"JSON string or Zenodo ID (prefixed by 'zenodo.').")
parser.add_argument("-x",
"--debug",
action="store_true",
help="Keeps temporary scripts used during "
"execution, and prints additional debug "
"messages.")
parser.add_argument("-s",
"--stream",
action="store_true",
help="Streams stdout and stderr in real time "
"during execution.")
parser.add_argument("--imagepath",
action="store",
help="Path to Singularity image. "
"If not specified, will use current directory.")
results = parser.parse_args(params)
descriptor = results.descriptor
# Validate descriptor
valid = invocation(descriptor)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, None, {
"forcePathType": True,
"debug": results.debug,
"stream": results.stream,
"imagePath": results.imagepath,
"skipDataCollect": True
})
container_location = executor.prepare()[1]
print("Container location: " + container_location)
# Adding hide to "container location" field since it's an invalid
# value, and we can parse that to hide the summary print
return ExecutorOutput(container_location, "", 0, "", [], [], "", "",
"hide")
def execute(*params):
parser = ArgumentParser("Boutiques local executor", add_help=False)
parser.add_argument("mode",
action="store",
help="Mode of operation to use. Launch: takes a "
"set of inputs compliant with invocation schema "
"and launches the tool. Simulate: shows sample "
"command-lines based on the provided descriptor"
" based on provided or randomly generated "
"inputs.",
choices=["launch", "simulate"])
parser.add_argument("--help",
"-h",
action="store_true",
help="show this help message and exit")
helps = any([True for ht in ["--help", "-h"] if ht in params])
if len(params) <= 1 and helps:
parser.print_help()
raise SystemExit
args, params = parser.parse_known_args(params)
mode = args.mode
params += ["--help"] if args.help is True else []
if mode == "launch":
parser = ArgumentParser("Launches an invocation.")
parser.add_argument("descriptor",
action="store",
help="The Boutiques descriptor.")
parser.add_argument("input",
action="store",
help="Input JSON complying to invocation.")
parser.add_argument("-v",
"--volumes",
action="store",
type=str,
help="Volumes to mount when launching the "
"container. Format consistently the following:"
" /a:/b will mount local direcotry /a to "
"container directory /b.",
nargs="*")
parser.add_argument("-x",
"--debug",
action="store_true",
help="Keeps temporary scripts used during "
"execution.")
parser.add_argument(
"-u",
"--user",
action="store_true",
help="Runs the container as local user ({0}) instead of root.".
format(os.getenv("USER")))
results = parser.parse_args(params)
descriptor = results.descriptor
# Do some basic input scrubbing
inp = results.input
if not os.path.isfile(inp):
raise SystemExit("Input file {} does not exist".format(inp))
if not inp.endswith(".json"):
raise SystemExit("Input file {} must end in json".format(inp))
if not os.path.isfile(descriptor):
raise SystemExit(
"JSON descriptor {} does not exist".format(descriptor))
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, {
"forcePathType": True,
"destroyTempScripts": not results.debug,
"changeUser": results.user
})
executor.readInput(inp)
# Execute it
exit_code = executor.execute(results.volumes)
if exit_code:
raise SystemExit(exit_code)
if mode == "simulate":
parser = ArgumentParser("Simulates an invocation.")
parser.add_argument("descriptor",
action="store",
help="The Boutiques descriptor.")
parser.add_argument("-i",
"--input",
action="store",
help="Input JSON complying to invocation.")
parser.add_argument("-r",
"--random",
action="store",
type=int,
nargs="*",
help="Generate random set of inputs.")
results = parser.parse_args(params)
descriptor = results.descriptor
# Do some basic input scrubbing
inp = results.input
rand = results.random is not None
numb = results.random[0] if rand and len(results.random) > 0 else 1
if numb and numb < 1:
raise SystemExit("--number value must be positive.")
if rand and inp:
raise SystemExit(
"--random setting and --input value cannot be used together.")
if inp and not os.path.isfile(inp):
raise SystemExit("Input file {} does not exist.".format(inp))
if inp and not inp.endswith(".json"):
raise SystemExit("Input file {} must end in 'json'.".format(inp))
if not os.path.isfile(descriptor):
raise SystemExit(
"JSON descriptor {} does not seem to exist.".format(
descriptor))
if not rand and not inp:
raise SystemExit("The default mode requires an input (-i).")
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(descriptor, {
"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True
})
if rand:
executor.generateRandomParams(numb)
executor.printCmdLine()
else:
executor.readInput(inp)
executor.printCmdLine()
def execute(*params):
parser = ArgumentParser("Boutiques local executor", add_help=False)
parser.add_argument("mode",
action="store",
help="Mode of operation to use. Launch: takes a "
"set of inputs compliant with invocation schema "
"and launches the tool. Simulate: shows sample "
"command-lines based on the provided descriptor"
" based on provided or randomly generated inputs. "
"Prepare: pulls the Docker or Singularity container "
"image for a given descriptor. ",
choices=["launch", "simulate", "prepare"])
parser.add_argument("--help",
"-h",
action="store_true",
help="show this help message and exit")
helps = any([True for ht in ["--help", "-h"] if ht in params])
if len(params) <= 1 and helps:
parser.print_help()
raise SystemExit
args, params = parser.parse_known_args(params)
mode = args.mode
params += ["--help"] if args.help is True else []
if mode == "launch":
parser = ArgumentParser("Launches an invocation.")
parser.add_argument(
"descriptor",
action="store",
help="The Boutiques descriptor as a JSON file, "
"JSON string or Zenodo ID (prefixed by 'zenodo.').")
parser.add_argument("invocation",
action="store",
help="Input JSON complying to invocation.")
parser.add_argument("-v",
"--volumes",
action="store",
type=str,
help="Volumes to mount when launching the "
"container. Format consistently the following:"
" /a:/b will mount local directory /a to "
"container directory /b.",
nargs="*")
parser.add_argument("-x",
"--debug",
action="store_true",
help="Keeps temporary scripts used during "
"execution, and prints additional debug "
"messages.")
parser.add_argument("-u",
"--user",
action="store_true",
help="Runs the container as local user ({0})"
" instead of root.".format(os.getenv("USER")))
parser.add_argument("-s",
"--stream",
action="store_true",
help="Streams stdout and stderr in real time "
"during execution.")
parser.add_argument("--imagepath",
action="store",
help="Location of Singularity image "
"(default is current directory).")
results = parser.parse_args(params)
descriptor = results.descriptor
inp = results.invocation
# Validate invocation and descriptor
valid = invocation(descriptor, '-i', inp)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, inp, {
"forcePathType": True,
"debug": results.debug,
"changeUser": results.user,
"stream": results.stream,
"imagePath": results.imagepath
})
# Execute it
return executor.execute(results.volumes)
if mode == "simulate":
parser = ArgumentParser("Simulates an invocation.")
parser.add_argument(
"descriptor",
action="store",
help="The Boutiques descriptor as a JSON file, "
"JSON string or Zenodo ID (prefixed by 'zenodo.').")
parser.add_argument("-i",
"--input",
action="store",
help="Input JSON complying to invocation.")
parser.add_argument("-r",
"--random",
action="store",
type=int,
nargs="*",
help="Generate random set of inputs.")
results = parser.parse_args(params)
descriptor = results.descriptor
# Do some basic input scrubbing
inp = results.input
rand = results.random is not None
numb = results.random[0] if rand and len(results.random) > 0 else 1
if numb and numb < 1:
raise_error(SystemExit, "--number value must be positive.")
# raise SystemExit("--number value must be positive.")
if rand and inp:
raise_error(
SystemExit, "--random setting and --input value cannot "
"be used together.")
if inp and not os.path.isfile(inp):
raise_error(SystemExit,
"Input file {} does not exist.".format(inp))
if inp and not inp.endswith(".json"):
raise_error(SystemExit,
"Input file {} must end in 'json'.".format(inp))
if not rand and not inp:
raise_error(SystemExit, "The default mode requires an input (-i).")
valid = invocation(descriptor, '-i', inp) if inp else\
invocation(descriptor)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(descriptor, inp, {
"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True
})
if rand:
executor.generateRandomParams(numb)
executor.printCmdLine()
# for consistency with execute
# Adding hide to "container location" field since it's an invalid
# value, and we can parse that to hide the summary print
return ExecutorOutput(os.linesep.join(executor.cmd_line), "", 0, "",
[], [], "", "", "hide")
if mode == "prepare":
parser = ArgumentParser("Pulls the container image for a given "
"descriptor")
parser.add_argument(
"descriptor",
action="store",
help="The Boutiques descriptor as a JSON file, "
"JSON string or Zenodo ID (prefixed by 'zenodo.').")
parser.add_argument("-x",
"--debug",
action="store_true",
help="Keeps temporary scripts used during "
"execution, and prints additional debug "
"messages.")
parser.add_argument("-s",
"--stream",
action="store_true",
help="Streams stdout and stderr in real time "
"during execution.")
parser.add_argument("--imagepath",
action="store",
help="Location of Singularity image "
"(default is current directory).")
results = parser.parse_args(params)
descriptor = results.descriptor
# Validate descriptor
valid = invocation(descriptor)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, None, {
"forcePathType": True,
"debug": results.debug,
"stream": results.stream,
"imagePath": results.imagepath
})
container_location = executor.prepare()[1]
print("Container location: " + container_location)
# Adding hide to "container location" field since it's an invalid
# value, and we can parse that to hide the summary print
return ExecutorOutput(container_location, "", 0, "", [], [], "", "",
"hide")
def execute(*params):
parser = parser_execute()
helps = any([True for ht in ["--help", "-h"] if ht in params])
if len(params) <= 1 and helps:
parser.print_help()
raise SystemExit
args, params = parser.parse_known_args(params)
mode = args.mode
params += ["--help"] if args.help is True else []
if mode == "launch":
parser = parser_executeLaunch()
results = parser.parse_args(params)
descriptor = results.descriptor
inp = results.invocation
# Validate invocation and descriptor
arguments = [descriptor, '-i', inp]
if results.sandbox:
arguments.append('--sandbox')
valid = invocation(*arguments)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, inp, {
"forcePathType": True,
"debug": results.debug,
"changeUser": results.user,
"stream": results.stream,
"imagePath": results.imagepath,
"skipDataCollect": results.skip_data_collection,
"forceDocker": results.force_docker,
"forceSingularity": results.force_singularity,
"provenance": results.provenance,
"noContainer": results.no_container,
"sandbox": results.sandbox
})
# Execute it
return executor.execute(results.volumes)
if mode == "simulate":
parser = parser_executeSimulate()
results = parser.parse_args(params)
descriptor = results.descriptor
# Do some basic input scrubbing
inp = results.input
arguments = [descriptor]
if inp:
arguments.append('-i')
arguments.append(inp)
if results.sandbox:
arguments.append('--sandbox')
valid = invocation(*arguments)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, inp, {
"forcePathType": True,
"destroyTempScripts": True,
"changeUser": True,
"skipDataCollect": True,
"requireComplete": results.complete,
"sandbox": results.sandbox
})
if not inp:
# Add optional inputs with default-value to inputs_dict,
# which is then populated with random params
executor.in_dict = addDefaultValues(executor.desc_dict, {})
executor.generateRandomParams(generateCmdLineFromInDict=True)
if results.json:
sout = [
json.dumps(customSortInvocationByInput(executor.in_dict,
descriptor),
indent=4)
]
print(sout[0])
else:
executor.printCmdLine()
sout = executor.cmd_line
# for consistency with execute
# Adding hide to "container location" field since it's an invalid
# value, can parse that to hide the summary print
return ExecutorOutput(os.linesep.join(sout), "", 0, "", [], [],
os.linesep.join(sout), "", "hide")
if mode == "prepare":
parser = parser_executePrepare()
results = parser.parse_args(params)
descriptor = results.descriptor
# Validate descriptor
arguments = [descriptor]
if results.sandbox:
arguments.append('--sandbox')
valid = invocation(*arguments)
# Generate object that will perform the commands
from boutiques.localExec import LocalExecutor
executor = LocalExecutor(
descriptor, None, {
"forcePathType": True,
"debug": results.debug,
"stream": results.stream,
"imagePath": results.imagepath,
"skipDataCollect": True,
"sandbox": results.sandbox
})
container_location = executor.prepare()[1]
print("Container location: " + container_location)
# Adding hide to "container location" field since it's an invalid
# value, and we can parse that to hide the summary print
return ExecutorOutput(container_location, "", 0, "", [], [], "", "",
"hide")