def post(self, user, pipeline_identifier): (boutiques_descriptor_path, descriptor_type ), error = get_original_descriptor_path_and_type(pipeline_identifier) if error: return marshal(error), 400 body = request.get_json(force=True) invocation = body['invocation'] queries = body['queries'] inv_tmp = tempfile.NamedTemporaryFile(mode="r+", delete=True) try: json.dump(invocation, inv_tmp) inv_tmp.flush() executor = LocalExecutor(boutiques_descriptor_path, inv_tmp.name, { "forcePathType": True, "destroyTempScripts": True, "changeUser": True }) query_results = [] for query in queries: query_results += [evaluateEngine(executor, query)] inv_tmp.close() return query_results[0] if len(query_results) == 1 else query_results, 200 except Exception as error: inv_tmp.close() raise error
def evaluate(*params): parser = ArgumentParser("Evaluates parameter values for a descriptor" " and invocation") parser.add_argument("descriptor", action="store", help="The Boutiques descriptor as a JSON file, JSON " "string or Zenodo ID (prefixed by 'zenodo.').") parser.add_argument("invocation", action="store", help="Input JSON complying to invocation.") parser.add_argument("query", action="store", nargs="*", help="The query to be performed. Simply request keys " "from the descriptor (i.e. output-files), and chain " "together queries (i.e. id=myfile or optional=false) " "slashes between them and commas connecting them. " "(i.e. output-files/optional=false,id=myfile). " "Perform multiple queries by separating them with a " "space.") result = parser.parse_args(params) # Generate object that will parse the invocation and descriptor from boutiques.localExec import LocalExecutor executor = LocalExecutor(result.descriptor, result.invocation, { "forcePathType": True, "destroyTempScripts": True, "changeUser": True }) from boutiques.evaluate import evaluateEngine query_results = [] for query in result.query: query_results += [evaluateEngine(executor, query)] return query_results[0] if len(query_results) == 1 else query_results
def test_example_requires_group_complete_x10(self): descriptor = op.join(op.split(bfile)[0], 'tests/docopt/valid/test_valid.json') from boutiques.localExec import LocalExecutor executor = LocalExecutor(descriptor, None, {"forcePathType": True, "destroyTempScripts": True, "changeUser": True, "skipDataCollect": True, "requireComplete": True}) # Can't create descriptors with mutex group but only one valid example # Bosh example is inherently random, # Couldn't even inject prederemined input to executor.in_dict # because _randomFillInDict clears it executor.generateRandomParams(100) self.assertGreater(len(executor.in_dict), 0)
def example(*params): parser = parser_example() results = parser.parse_args(params) descriptor = results.descriptor valid = invocation(descriptor) # Generate object that will perform the commands from boutiques.localExec import LocalExecutor executor = LocalExecutor(descriptor, None, {"forcePathType": True, "destroyTempScripts": True, "changeUser": True, "skipDataCollect": True, "requireComplete": results.complete}) executor.generateRandomParams(1) return json.dumps(executor.in_dict, indent=4, sort_keys=True)
def example(*params): parser = ArgumentParser("Generates example invocation from a valid" "descriptor") parser.add_argument("descriptor", action="store", help="The Boutiques descriptor as a JSON file, " "JSON string or Zenodo ID (prefixed by 'zenodo.').") parser.add_argument("-c", "--complete", action="store_true", help="Include optional parameters.") results = parser.parse_args(params) descriptor = results.descriptor valid = invocation(descriptor) # Generate object that will perform the commands from boutiques.localExec import LocalExecutor executor = LocalExecutor(descriptor, None, {"forcePathType": True, "destroyTempScripts": True, "changeUser": True, "skipDataCollect": True, "requireComplete": results.complete}) executor.generateRandomParams(1) return json.dumps(executor.in_dict, indent=4, sort_keys=True)
def evaluate(*params): parser = parser_evaluate() result = parser.parse_args(params) # Generate object that will parse the invocation and descriptor from boutiques.localExec import LocalExecutor executor = LocalExecutor(result.descriptor, result.invocation, {"forcePathType": True, "destroyTempScripts": True, "changeUser": True, "skipDataCollect": True}) from boutiques.evaluate import evaluateEngine query_results = [] for query in result.query: query_results += [evaluateEngine(executor, query)] return query_results[0] if len(query_results) == 1 else query_results
def get_bosh_cmdline(descriptor_filepath, invocation_filepath): executor = LocalExecutor(descriptor_filepath, { "forcePathType": True, "destroyTempScripts": True, "changeUser": True, }) executor.readInput(invocation_filepath) old_stdout = sys.stdout sys.stdout = mystdout = StringIO() executor.printCmdLine() sys.stdout = old_stdout return mystdout.getvalue()[19:]
def execute(*params): parser = ArgumentParser("Boutiques local executor", add_help=False) parser.add_argument("mode", action="store", help="Mode of operation to use. Launch: takes a " "set of inputs compliant with invocation schema " "and launches the tool. Simulate: shows sample " "command-lines based on the provided descriptor" " based on provided or randomly generated inputs. " "Prepare: pulls the Docker or Singularity container " "image for a given descriptor. ", choices=["launch", "simulate", "prepare"]) parser.add_argument("--help", "-h", action="store_true", help="show this help message and exit") helps = any([True for ht in ["--help", "-h"] if ht in params]) if len(params) <= 1 and helps: parser.print_help() raise SystemExit args, params = parser.parse_known_args(params) mode = args.mode params += ["--help"] if args.help is True else [] if mode == "launch": parser = ArgumentParser("Launches an invocation.") parser.add_argument( "descriptor", action="store", help="The Boutiques descriptor as a JSON file, " "JSON string or Zenodo ID (prefixed by 'zenodo.').") parser.add_argument("invocation", action="store", help="Input JSON complying to invocation.") parser.add_argument("-v", "--volumes", action="append", type=str, help="Volumes to mount when launching the " "container. Format consistently the following:" " /a:/b will mount local directory /a to " "container directory /b.") parser.add_argument("-x", "--debug", action="store_true", help="Keeps temporary scripts used during " "execution, and prints additional debug " "messages.") parser.add_argument("-u", "--user", action="store_true", help="Runs the container as local user ({0})" " instead of root.".format(os.getenv("USER"))) parser.add_argument("-s", "--stream", action="store_true", help="Streams stdout and stderr in real time " "during execution.") parser.add_argument("--imagepath", action="store", help="Path to Singularity image. " "If not specified, will use current directory.") parser.add_argument("--skip-data-collection", action="store_true", help="Skips execution data collection and saving" "to cache.") parser.add_argument("--provenance", action="store", type=json.loads, help="Append JSON object to the generated record.") force_group = parser.add_mutually_exclusive_group() force_group.add_argument("--force-docker", action="store_true", help="Tries to run Singularity images with " "Docker. This only works if the image is on" "Docker Hub, i.e. has index docker://") force_group.add_argument("--force-singularity", action="store_true", help="Tries to run Docker images with " "Singularity.") results = parser.parse_args(params) descriptor = results.descriptor inp = results.invocation # Validate invocation and descriptor valid = invocation(descriptor, '-i', inp) # Generate object that will perform the commands from boutiques.localExec import LocalExecutor executor = LocalExecutor( descriptor, inp, { "forcePathType": True, "debug": results.debug, "changeUser": results.user, "stream": results.stream, "imagePath": results.imagepath, "skipDataCollect": results.skip_data_collection, "forceDocker": results.force_docker, "forceSingularity": results.force_singularity, "provenance": results.provenance }) # Execute it return executor.execute(results.volumes) if mode == "simulate": parser = ArgumentParser("Simulates an invocation.") parser.add_argument( "descriptor", action="store", help="The Boutiques descriptor as a JSON file, " "JSON string or Zenodo ID (prefixed by 'zenodo.').") parser.add_argument("-i", "--input", action="store", help="Input JSON complying to invocation.") parser.add_argument("-j", "--json", action="store_true", help="Flag to generate invocation in JSON format.") parser.add_argument("-c", "--complete", action="store_true", help="Include optional parameters.") results = parser.parse_args(params) descriptor = results.descriptor # Do some basic input scrubbing inp = results.input arguments = [descriptor] if inp: arguments.append('-i') arguments.append(inp) valid = invocation(*arguments) # Generate object that will perform the commands from boutiques.localExec import LocalExecutor executor = LocalExecutor( descriptor, inp, { "forcePathType": True, "destroyTempScripts": True, "changeUser": True, "skipDataCollect": True, "requireComplete": results.complete }) if not inp: executor.generateRandomParams(1) if results.json: sout = [json.dumps(executor.in_dict, indent=4, sort_keys=True)] print(sout[0]) else: executor.printCmdLine() sout = executor.cmd_line # for consistency with execute # Adding hide to "container location" field since it's an invalid # value, can parse that to hide the summary print return ExecutorOutput(os.linesep.join(sout), "", 0, "", [], [], os.linesep.join(sout), "", "hide") if mode == "prepare": parser = ArgumentParser("Pulls the container image for a given " "descriptor") parser.add_argument( "descriptor", action="store", help="The Boutiques descriptor as a JSON file, " "JSON string or Zenodo ID (prefixed by 'zenodo.').") parser.add_argument("-x", "--debug", action="store_true", help="Keeps temporary scripts used during " "execution, and prints additional debug " "messages.") parser.add_argument("-s", "--stream", action="store_true", help="Streams stdout and stderr in real time " "during execution.") parser.add_argument("--imagepath", action="store", help="Path to Singularity image. " "If not specified, will use current directory.") results = parser.parse_args(params) descriptor = results.descriptor # Validate descriptor valid = invocation(descriptor) # Generate object that will perform the commands from boutiques.localExec import LocalExecutor executor = LocalExecutor( descriptor, None, { "forcePathType": True, "debug": results.debug, "stream": results.stream, "imagePath": results.imagepath, "skipDataCollect": True }) container_location = executor.prepare()[1] print("Container location: " + container_location) # Adding hide to "container location" field since it's an invalid # value, and we can parse that to hide the summary print return ExecutorOutput(container_location, "", 0, "", [], [], "", "", "hide")
def execute(*params): parser = ArgumentParser("Boutiques local executor", add_help=False) parser.add_argument("mode", action="store", help="Mode of operation to use. Launch: takes a " "set of inputs compliant with invocation schema " "and launches the tool. Simulate: shows sample " "command-lines based on the provided descriptor" " based on provided or randomly generated " "inputs.", choices=["launch", "simulate"]) parser.add_argument("--help", "-h", action="store_true", help="show this help message and exit") helps = any([True for ht in ["--help", "-h"] if ht in params]) if len(params) <= 1 and helps: parser.print_help() raise SystemExit args, params = parser.parse_known_args(params) mode = args.mode params += ["--help"] if args.help is True else [] if mode == "launch": parser = ArgumentParser("Launches an invocation.") parser.add_argument("descriptor", action="store", help="The Boutiques descriptor.") parser.add_argument("input", action="store", help="Input JSON complying to invocation.") parser.add_argument("-v", "--volumes", action="store", type=str, help="Volumes to mount when launching the " "container. Format consistently the following:" " /a:/b will mount local direcotry /a to " "container directory /b.", nargs="*") parser.add_argument("-x", "--debug", action="store_true", help="Keeps temporary scripts used during " "execution.") parser.add_argument( "-u", "--user", action="store_true", help="Runs the container as local user ({0}) instead of root.". format(os.getenv("USER"))) results = parser.parse_args(params) descriptor = results.descriptor # Do some basic input scrubbing inp = results.input if not os.path.isfile(inp): raise SystemExit("Input file {} does not exist".format(inp)) if not inp.endswith(".json"): raise SystemExit("Input file {} must end in json".format(inp)) if not os.path.isfile(descriptor): raise SystemExit( "JSON descriptor {} does not exist".format(descriptor)) # Generate object that will perform the commands from boutiques.localExec import LocalExecutor executor = LocalExecutor( descriptor, { "forcePathType": True, "destroyTempScripts": not results.debug, "changeUser": results.user }) executor.readInput(inp) # Execute it exit_code = executor.execute(results.volumes) if exit_code: raise SystemExit(exit_code) if mode == "simulate": parser = ArgumentParser("Simulates an invocation.") parser.add_argument("descriptor", action="store", help="The Boutiques descriptor.") parser.add_argument("-i", "--input", action="store", help="Input JSON complying to invocation.") parser.add_argument("-r", "--random", action="store", type=int, nargs="*", help="Generate random set of inputs.") results = parser.parse_args(params) descriptor = results.descriptor # Do some basic input scrubbing inp = results.input rand = results.random is not None numb = results.random[0] if rand and len(results.random) > 0 else 1 if numb and numb < 1: raise SystemExit("--number value must be positive.") if rand and inp: raise SystemExit( "--random setting and --input value cannot be used together.") if inp and not os.path.isfile(inp): raise SystemExit("Input file {} does not exist.".format(inp)) if inp and not inp.endswith(".json"): raise SystemExit("Input file {} must end in 'json'.".format(inp)) if not os.path.isfile(descriptor): raise SystemExit( "JSON descriptor {} does not seem to exist.".format( descriptor)) if not rand and not inp: raise SystemExit("The default mode requires an input (-i).") # Generate object that will perform the commands from boutiques.localExec import LocalExecutor executor = LocalExecutor(descriptor, { "forcePathType": True, "destroyTempScripts": True, "changeUser": True }) if rand: executor.generateRandomParams(numb) executor.printCmdLine() else: executor.readInput(inp) executor.printCmdLine()
def execute(*params): parser = ArgumentParser("Boutiques local executor", add_help=False) parser.add_argument("mode", action="store", help="Mode of operation to use. Launch: takes a " "set of inputs compliant with invocation schema " "and launches the tool. Simulate: shows sample " "command-lines based on the provided descriptor" " based on provided or randomly generated inputs. " "Prepare: pulls the Docker or Singularity container " "image for a given descriptor. ", choices=["launch", "simulate", "prepare"]) parser.add_argument("--help", "-h", action="store_true", help="show this help message and exit") helps = any([True for ht in ["--help", "-h"] if ht in params]) if len(params) <= 1 and helps: parser.print_help() raise SystemExit args, params = parser.parse_known_args(params) mode = args.mode params += ["--help"] if args.help is True else [] if mode == "launch": parser = ArgumentParser("Launches an invocation.") parser.add_argument( "descriptor", action="store", help="The Boutiques descriptor as a JSON file, " "JSON string or Zenodo ID (prefixed by 'zenodo.').") parser.add_argument("invocation", action="store", help="Input JSON complying to invocation.") parser.add_argument("-v", "--volumes", action="store", type=str, help="Volumes to mount when launching the " "container. Format consistently the following:" " /a:/b will mount local directory /a to " "container directory /b.", nargs="*") parser.add_argument("-x", "--debug", action="store_true", help="Keeps temporary scripts used during " "execution, and prints additional debug " "messages.") parser.add_argument("-u", "--user", action="store_true", help="Runs the container as local user ({0})" " instead of root.".format(os.getenv("USER"))) parser.add_argument("-s", "--stream", action="store_true", help="Streams stdout and stderr in real time " "during execution.") parser.add_argument("--imagepath", action="store", help="Location of Singularity image " "(default is current directory).") results = parser.parse_args(params) descriptor = results.descriptor inp = results.invocation # Validate invocation and descriptor valid = invocation(descriptor, '-i', inp) # Generate object that will perform the commands from boutiques.localExec import LocalExecutor executor = LocalExecutor( descriptor, inp, { "forcePathType": True, "debug": results.debug, "changeUser": results.user, "stream": results.stream, "imagePath": results.imagepath }) # Execute it return executor.execute(results.volumes) if mode == "simulate": parser = ArgumentParser("Simulates an invocation.") parser.add_argument( "descriptor", action="store", help="The Boutiques descriptor as a JSON file, " "JSON string or Zenodo ID (prefixed by 'zenodo.').") parser.add_argument("-i", "--input", action="store", help="Input JSON complying to invocation.") parser.add_argument("-r", "--random", action="store", type=int, nargs="*", help="Generate random set of inputs.") results = parser.parse_args(params) descriptor = results.descriptor # Do some basic input scrubbing inp = results.input rand = results.random is not None numb = results.random[0] if rand and len(results.random) > 0 else 1 if numb and numb < 1: raise_error(SystemExit, "--number value must be positive.") # raise SystemExit("--number value must be positive.") if rand and inp: raise_error( SystemExit, "--random setting and --input value cannot " "be used together.") if inp and not os.path.isfile(inp): raise_error(SystemExit, "Input file {} does not exist.".format(inp)) if inp and not inp.endswith(".json"): raise_error(SystemExit, "Input file {} must end in 'json'.".format(inp)) if not rand and not inp: raise_error(SystemExit, "The default mode requires an input (-i).") valid = invocation(descriptor, '-i', inp) if inp else\ invocation(descriptor) # Generate object that will perform the commands from boutiques.localExec import LocalExecutor executor = LocalExecutor(descriptor, inp, { "forcePathType": True, "destroyTempScripts": True, "changeUser": True }) if rand: executor.generateRandomParams(numb) executor.printCmdLine() # for consistency with execute # Adding hide to "container location" field since it's an invalid # value, and we can parse that to hide the summary print return ExecutorOutput(os.linesep.join(executor.cmd_line), "", 0, "", [], [], "", "", "hide") if mode == "prepare": parser = ArgumentParser("Pulls the container image for a given " "descriptor") parser.add_argument( "descriptor", action="store", help="The Boutiques descriptor as a JSON file, " "JSON string or Zenodo ID (prefixed by 'zenodo.').") parser.add_argument("-x", "--debug", action="store_true", help="Keeps temporary scripts used during " "execution, and prints additional debug " "messages.") parser.add_argument("-s", "--stream", action="store_true", help="Streams stdout and stderr in real time " "during execution.") parser.add_argument("--imagepath", action="store", help="Location of Singularity image " "(default is current directory).") results = parser.parse_args(params) descriptor = results.descriptor # Validate descriptor valid = invocation(descriptor) # Generate object that will perform the commands from boutiques.localExec import LocalExecutor executor = LocalExecutor( descriptor, None, { "forcePathType": True, "debug": results.debug, "stream": results.stream, "imagePath": results.imagepath }) container_location = executor.prepare()[1] print("Container location: " + container_location) # Adding hide to "container location" field since it's an invalid # value, and we can parse that to hide the summary print return ExecutorOutput(container_location, "", 0, "", [], [], "", "", "hide")
def execute(*params): parser = parser_execute() helps = any([True for ht in ["--help", "-h"] if ht in params]) if len(params) <= 1 and helps: parser.print_help() raise SystemExit args, params = parser.parse_known_args(params) mode = args.mode params += ["--help"] if args.help is True else [] if mode == "launch": parser = parser_executeLaunch() results = parser.parse_args(params) descriptor = results.descriptor inp = results.invocation # Validate invocation and descriptor arguments = [descriptor, '-i', inp] if results.sandbox: arguments.append('--sandbox') valid = invocation(*arguments) # Generate object that will perform the commands from boutiques.localExec import LocalExecutor executor = LocalExecutor( descriptor, inp, { "forcePathType": True, "debug": results.debug, "changeUser": results.user, "stream": results.stream, "imagePath": results.imagepath, "skipDataCollect": results.skip_data_collection, "forceDocker": results.force_docker, "forceSingularity": results.force_singularity, "provenance": results.provenance, "noContainer": results.no_container, "sandbox": results.sandbox }) # Execute it return executor.execute(results.volumes) if mode == "simulate": parser = parser_executeSimulate() results = parser.parse_args(params) descriptor = results.descriptor # Do some basic input scrubbing inp = results.input arguments = [descriptor] if inp: arguments.append('-i') arguments.append(inp) if results.sandbox: arguments.append('--sandbox') valid = invocation(*arguments) # Generate object that will perform the commands from boutiques.localExec import LocalExecutor executor = LocalExecutor( descriptor, inp, { "forcePathType": True, "destroyTempScripts": True, "changeUser": True, "skipDataCollect": True, "requireComplete": results.complete, "sandbox": results.sandbox }) if not inp: # Add optional inputs with default-value to inputs_dict, # which is then populated with random params executor.in_dict = addDefaultValues(executor.desc_dict, {}) executor.generateRandomParams(generateCmdLineFromInDict=True) if results.json: sout = [ json.dumps(customSortInvocationByInput(executor.in_dict, descriptor), indent=4) ] print(sout[0]) else: executor.printCmdLine() sout = executor.cmd_line # for consistency with execute # Adding hide to "container location" field since it's an invalid # value, can parse that to hide the summary print return ExecutorOutput(os.linesep.join(sout), "", 0, "", [], [], os.linesep.join(sout), "", "hide") if mode == "prepare": parser = parser_executePrepare() results = parser.parse_args(params) descriptor = results.descriptor # Validate descriptor arguments = [descriptor] if results.sandbox: arguments.append('--sandbox') valid = invocation(*arguments) # Generate object that will perform the commands from boutiques.localExec import LocalExecutor executor = LocalExecutor( descriptor, None, { "forcePathType": True, "debug": results.debug, "stream": results.stream, "imagePath": results.imagepath, "skipDataCollect": True, "sandbox": results.sandbox }) container_location = executor.prepare()[1] print("Container location: " + container_location) # Adding hide to "container location" field since it's an invalid # value, and we can parse that to hide the summary print return ExecutorOutput(container_location, "", 0, "", [], [], "", "", "hide")