コード例 #1
0
def cli(ctx, workflow_path, output=None, force=False, **kwds):
    """Convert Format 2 workflow to a native Galaxy workflow.
    """
    assert is_galaxy_engine(**kwds)

    kwds["no_dependency_resolution"] = True

    if workflow_path.endswith(".ga"):
        if output is None:
            output = os.path.splitext(workflow_path)[0] + ".gxwf.yml"

        with open(workflow_path, "r") as f:
            workflow_dict = json.load(f)
        format2_wrapper = from_galaxy_native(workflow_dict, json_wrapper=True)
        with open(output, "w") as f:
            f.write(format2_wrapper["yaml_content"])
    else:
        if output is None:
            output = os.path.splitext(workflow_path)[0] + ".ga"

        runnable = for_path(workflow_path)
        with engine_context(ctx, **kwds) as galaxy_engine:
            with galaxy_engine.ensure_runnables_served([runnable]) as config:
                workflow_id = config.workflow_id(workflow_path)
                output_dict = config.gi.workflows.export_workflow_dict(workflow_id)

                output_contents = json.dumps(output_dict)
                write_file(output, output_contents, force=force)
コード例 #2
0
ファイル: config.py プロジェクト: pvanheus/planemo
def _handle_job_config_file(config_directory, server_name, kwds):
    job_config_file = kwds.get("job_config_file", None)
    if not job_config_file:
        template_str = JOB_CONFIG_LOCAL
        job_config_file = os.path.join(
            config_directory,
            "job_conf.xml",
        )
        docker_enable = str(kwds.get("docker", False))
        docker_host = str(kwds.get("docker_host", docker_util.DEFAULT_HOST))
        docker_host_param = ""
        if docker_host:
            docker_host_param = """<param id="docker_host">%s</param>""" % docker_host

        conf_contents = Template(template_str).safe_substitute({
            "server_name": server_name,
            "docker_enable": docker_enable,
            "require_container": "false",
            "docker_sudo": str(kwds.get("docker_sudo", False)),
            "docker_sudo_cmd": str(kwds.get("docker_sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND)),
            "docker_cmd": str(kwds.get("docker_cmd", docker_util.DEFAULT_DOCKER_COMMAND)),
            "docker_host": docker_host_param,
        })
        write_file(job_config_file, conf_contents)
    kwds["job_config_file"] = job_config_file
コード例 #3
0
def cli(ctx, workflow_identifier, output=None, force=False, **kwds):
    """Convert Format 2 workflows to native Galaxy workflows, and vice-versa.
    """
    assert is_galaxy_engine(**kwds)

    kwds["no_dependency_resolution"] = True

    if workflow_identifier.endswith(".ga"):
        if output is None:
            output = os.path.splitext(workflow_identifier)[0] + ".gxwf.yml"

        with open(workflow_identifier, "r") as f:
            workflow_dict = json.load(f)
        format2_wrapper = from_galaxy_native(workflow_dict, json_wrapper=True)
        with open(output, "w") as f:
            f.write(format2_wrapper["yaml_content"])
    else:
        if output is None:
            output = os.path.splitext(workflow_identifier)[0] + ".ga"

        runnable = for_path(workflow_identifier)
        with engine_context(ctx, **kwds) as galaxy_engine:
            with galaxy_engine.ensure_runnables_served([runnable]) as config:
                workflow_id = config.workflow_id(workflow_identifier)
                output_dict = config.gi.workflows.export_workflow_dict(workflow_id)

                output_contents = json.dumps(output_dict, indent=4, sort_keys=True)
                write_file(output, output_contents, force=force)
コード例 #4
0
def _handle_job_config_file(config_directory, server_name, kwds):
    job_config_file = kwds.get("job_config_file", None)
    if not job_config_file:
        template_str = JOB_CONFIG_LOCAL
        job_config_file = os.path.join(
            config_directory,
            "job_conf.xml",
        )
        docker_enable = str(kwds.get("docker", False))
        docker_host = str(kwds.get("docker_host", docker_util.DEFAULT_HOST))
        docker_host_param = ""
        if docker_host:
            docker_host_param = """<param id="docker_host">%s</param>""" % docker_host

        conf_contents = Template(template_str).safe_substitute({
            "server_name":
            server_name,
            "docker_enable":
            docker_enable,
            "require_container":
            "false",
            "docker_sudo":
            str(kwds.get("docker_sudo", False)),
            "docker_sudo_cmd":
            str(kwds.get("docker_sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND)),
            "docker_cmd":
            str(kwds.get("docker_cmd", docker_util.DEFAULT_DOCKER_COMMAND)),
            "docker_host":
            docker_host_param,
        })
        write_file(job_config_file, conf_contents)
    kwds["job_config_file"] = job_config_file
コード例 #5
0
 def test_shed_diff(self):
     with self._isolate_repo("single_tool") as f:
         upload_command = ["shed_upload", "--force_repository_creation"]
         upload_command.extend(self._shed_args())
         self._check_exit_code(upload_command)
         io.write_file(join(f, "related_file"), "A related non-tool file (modified).\n")
         self._check_diff(f, True)
         self._check_diff(f, False)
コード例 #6
0
ファイル: config.py プロジェクト: lecorguille/planemo
def _configure_sheds_config_file(ctx, config_directory, **kwds):
    if "shed_target" not in kwds:
        kwds = kwds.copy()
        kwds["shed_target"] = "toolshed"
    shed_target_url = tool_shed_url(ctx, **kwds)
    contents = _sub(TOOL_SHEDS_CONF, {"shed_target_url": shed_target_url})
    tool_sheds_conf = os.path.join(config_directory, "tool_sheds_conf.xml")
    write_file(tool_sheds_conf, contents)
    return tool_sheds_conf
コード例 #7
0
def _configure_sheds_config_file(ctx, config_directory, **kwds):
    if "shed_target" not in kwds:
        kwds = kwds.copy()
        kwds["shed_target"] = "toolshed"
    shed_target_url = tool_shed_url(ctx, **kwds)
    contents = _sub(TOOL_SHEDS_CONF, {"shed_target_url": shed_target_url})
    tool_sheds_conf = os.path.join(config_directory, "tool_sheds_conf.xml")
    write_file(tool_sheds_conf, contents)
    return tool_sheds_conf
コード例 #8
0
ファイル: config.py プロジェクト: pvanheus/planemo
def _write_tool_conf(ctx, tool_paths, tool_conf_path):
    tool_definition = _tool_conf_entry_for(tool_paths)
    tool_conf_template_kwds = dict(tool_definition=tool_definition)
    tool_conf_contents = _sub(TOOL_CONF_TEMPLATE, tool_conf_template_kwds)
    write_file(tool_conf_path, tool_conf_contents)
    ctx.vlog(
        "Writing tool_conf to path %s with contents [%s]",
        tool_conf_path,
        tool_conf_contents,
    )
コード例 #9
0
def _write_tool_conf(ctx, tool_paths, tool_conf_path):
    tool_definition = _tool_conf_entry_for(tool_paths)
    tool_conf_template_kwds = dict(tool_definition=tool_definition)
    tool_conf_contents = _sub(TOOL_CONF_TEMPLATE, tool_conf_template_kwds)
    write_file(tool_conf_path, tool_conf_contents)
    ctx.vlog(
        "Writing tool_conf to path %s with contents [%s]",
        tool_conf_path,
        tool_conf_contents,
    )
コード例 #10
0
 def test_shed_diff(self):
     with self._isolate_repo("single_tool") as f:
         upload_command = ["shed_upload", "--force_repository_creation"]
         upload_command.extend(self._shed_args())
         self._check_exit_code(upload_command)
         io.write_file(
             join(f, "related_file"),
             "A related non-tool file (modified).\n",
         )
         self._check_diff(f, True)
         self._check_diff(f, False)
コード例 #11
0
    def test_diff_recursive(self):
        with self._isolate_repo("multi_repos_nested") as f:
            upload_command = ["shed_upload", "-r", "--force_repository_creation"]
            upload_command.extend(self._shed_args())
            self._check_exit_code(upload_command)

            diff_command = ["shed_diff", "-r"]
            diff_command.extend(self._shed_args(read_only=True))
            self._check_exit_code(diff_command, exit_code=0)

            io.write_file(join(f, "cat1", "related_file"), "A related non-tool file (modified).\n")
            self._check_exit_code(diff_command, exit_code=1)
コード例 #12
0
    def test_diff_recursive(self):
        with self._isolate_repo("multi_repos_nested") as f:
            self._shed_create(recursive=True)

            diff_command = ["shed_diff", "-r"]
            diff_command.extend(self._shed_args(read_only=True))
            self._check_exit_code(diff_command, exit_code=0)

            io.write_file(
                join(f, "cat1", "related_file"),
                "A related non-tool file (modified).\n",
            )
            self._check_exit_code(diff_command, exit_code=1)
コード例 #13
0
ファイル: test_shed_diff.py プロジェクト: davebx/planemo
    def test_diff_xunit(self):
        with self._isolate_repo("multi_repos_nested") as f:
            upload_command = [
                "shed_upload", "-r", "--force_repository_creation"
            ]
            upload_command.extend(self._shed_args())
            self._check_exit_code(upload_command)

            xunit_report = tempfile.NamedTemporaryFile(delete=False)
            xunit_report.flush()
            xunit_report.close()
            diff_command = ["shed_diff", "-r", "--report_xunit", xunit_report.name]
            diff_command.extend(self._shed_args(read_only=True))
            known_good_xunit_report = os.path.join(TEST_REPOS_DIR,
                                                   'multi_repos_nested.xunit.xml')
            known_bad_xunit_report = os.path.join(TEST_REPOS_DIR,
                                                  'multi_repos_nested.xunit-bad.xml')
            self._check_exit_code(diff_command, exit_code=0)

            compare = open(xunit_report.name, 'r').read()
            if not diff(
                ElementTree.parse(known_good_xunit_report).getroot(),
                ElementTree.fromstring(compare),
                reporter=sys.stdout.write
            ):
                self.assertTrue(True)
            else:
                sys.stdout.write(compare)
                self.assertTrue(False)

            io.write_file(
                join(f, "cat1", "related_file"),
                "A related non-tool file (modified).\n",
            )
            self._check_exit_code(diff_command, exit_code=1)

            compare = open(xunit_report.name, 'r').read()
            if not diff(
                ElementTree.parse(known_bad_xunit_report).getroot(),
                ElementTree.fromstring(compare),
                reporter=sys.stdout.write
            ):
                self.assertTrue(True)
            else:
                sys.stdout.write(compare)
                self.assertTrue(False)

            os.unlink(xunit_report.name)
コード例 #14
0
ファイル: test_shed_diff.py プロジェクト: davebx/planemo
    def test_diff_xunit(self):
        with self._isolate_repo("multi_repos_nested") as f:
            upload_command = [
                "shed_upload", "-r", "--force_repository_creation"
            ]
            upload_command.extend(self._shed_args())
            self._check_exit_code(upload_command)

            xunit_report = tempfile.NamedTemporaryFile(delete=False)
            xunit_report.flush()
            xunit_report.close()
            diff_command = [
                "shed_diff", "-r", "--report_xunit", xunit_report.name
            ]
            diff_command.extend(self._shed_args(read_only=True))
            known_good_xunit_report = os.path.join(
                TEST_REPOS_DIR, 'multi_repos_nested.xunit.xml')
            known_bad_xunit_report = os.path.join(
                TEST_REPOS_DIR, 'multi_repos_nested.xunit-bad.xml')
            self._check_exit_code(diff_command, exit_code=0)

            compare = open(xunit_report.name, 'r').read()
            if not diff(ElementTree.parse(known_good_xunit_report).getroot(),
                        ElementTree.fromstring(compare),
                        reporter=sys.stdout.write):
                self.assertTrue(True)
            else:
                sys.stdout.write(compare)
                self.assertTrue(False)

            io.write_file(
                join(f, "cat1", "related_file"),
                "A related non-tool file (modified).\n",
            )
            self._check_exit_code(diff_command, exit_code=1)

            compare = open(xunit_report.name, 'r').read()
            if not diff(ElementTree.parse(known_bad_xunit_report).getroot(),
                        ElementTree.fromstring(compare),
                        reporter=sys.stdout.write):
                self.assertTrue(True)
            else:
                sys.stdout.write(compare)
                self.assertTrue(False)

            os.unlink(xunit_report.name)
コード例 #15
0
    def test_diff_recusrive(self):
        with self._isolate_repo("multi_repos_nested") as f:
            upload_command = [
                "shed_upload", "-r", "--force_repository_creation"
            ]
            upload_command.extend(self._shed_args())
            self._check_exit_code(upload_command)

            diff_command = ["shed_diff", "-r"]
            diff_command.extend(self._shed_args(read_only=True))
            self._check_exit_code(diff_command, exit_code=0)

            io.write_file(
                join(f, "cat1", "related_file"),
                "A related non-tool file (modified).\n",
            )
            self._check_exit_code(diff_command, exit_code=-1)
コード例 #16
0
def cli(ctx, workflow_path, output=None, force=False, **kwds):
    """Convert Format 2 workflow to a native Galaxy workflow.
    """
    assert is_galaxy_engine(**kwds)

    kwds["no_dependency_resolution"] = True

    if output is None:
        output = os.path.splitext(workflow_path)[0] + ".ga"

    runnable = for_path(workflow_path)
    with engine_context(ctx, **kwds) as galaxy_engine:
        with galaxy_engine.serve_runnables([runnable]) as config:
            workflow_id = config.workflow_id(workflow_path)
            output_dict = config.gi.workflows.export_workflow_dict(workflow_id)

            output_contents = json.dumps(output_dict)
            write_file(output, output_contents, force=force)
コード例 #17
0
ファイル: config.py プロジェクト: natefoo/planemo
def _handle_job_config_file(config_directory, server_name, kwds):
    job_config_file = kwds.get("job_config_file", None)
    if not job_config_file:
        template_str = JOB_CONFIG_LOCAL
        job_config_file = os.path.join(
            config_directory,
            "job_conf.xml",
        )
        conf_contents = Template(template_str).safe_substitute({
            "server_name": server_name,
            "docker_enable": str(kwds.get("docker", False)),
            "docker_sudo": str(kwds.get("docker_sudo", False)),
            "docker_sudo_cmd": str(kwds.get("docker_sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND)),
            "docker_cmd": str(kwds.get("docker_cmd", docker_util.DEFAULT_DOCKER_COMMAND)),
            "docker_host": str(kwds.get("docker_host", docker_util.DEFAULT_HOST)),
        })
        write_file(job_config_file, conf_contents)
    kwds["job_config_file"] = job_config_file
コード例 #18
0
ファイル: test_shed_diff.py プロジェクト: simonbray/planemo
    def test_diff_xunit(self):
        with self._isolate_repo("multi_repos_nested") as f:
            self._shed_create(recursive=True)

            xunit_report = tempfile.NamedTemporaryFile(delete=False)
            xunit_report.flush()
            xunit_report.close()
            diff_command = ["shed_diff", "-r", "--report_xunit", xunit_report.name]
            diff_command.extend(self._shed_args(read_only=True))
            known_good_xunit_report = os.path.join(TEST_REPOS_DIR,
                                                   'multi_repos_nested.xunit.xml')
            known_bad_xunit_report = os.path.join(TEST_REPOS_DIR,
                                                  'multi_repos_nested.xunit-bad.xml')
            self._check_exit_code(diff_command, exit_code=0)

            with open(xunit_report.name, 'r') as fh:
                compare = fh.read()
            if diff(
                self._make_deterministic(ElementTree.parse(known_good_xunit_report).getroot()),
                self._make_deterministic(ElementTree.fromstring(compare)),
                reporter=sys.stdout.write
            ):
                sys.stdout.write(compare)
                assert False, "XUnit report different from multi_repos_nested.xunit.xml."

            io.write_file(
                join(f, "cat1", "related_file"),
                "A related non-tool file (modified).\n",
            )
            self._check_exit_code(diff_command, exit_code=1)

            with open(xunit_report.name, 'r') as fh:
                compare = fh.read()
            if diff(
                self._make_deterministic(ElementTree.parse(known_bad_xunit_report).getroot()),
                self._make_deterministic(ElementTree.fromstring(compare)),
                reporter=sys.stdout.write
            ):
                sys.stdout.write(compare)
                assert False, "XUnit report different from multi_repos_nested.xunit-bad.xml."

            os.unlink(xunit_report.name)
コード例 #19
0
ファイル: tool_builder.py プロジェクト: gregvonkuster/planemo
def write_tool_description(ctx, tool_description, **kwds):
    """Write a tool description to the file system guided by supplied CLI kwds."""
    tool_id = kwds.get("id")
    output = kwds.get("tool")
    if not output:
        extension = "cwl" if kwds.get("cwl") else "xml"
        output = "%s.%s" % (tool_id, extension)
    if not io.can_write_to_path(output, **kwds):
        ctx.exit(1)

    io.write_file(output, tool_description.contents)
    io.info("Tool written to %s" % output)
    test_contents = tool_description.test_contents
    if test_contents:
        sep = "-" if "-" in tool_id else "_"
        tests_path = "%s%stests.yml" % (kwds.get("id"), sep)
        if not io.can_write_to_path(tests_path, **kwds):
            ctx.exit(1)
        io.write_file(tests_path, test_contents)
        io.info("Tool tests written to %s" % tests_path)

    macros = kwds["macros"]
    macros_file = "macros.xml"
    if macros and not os.path.exists(macros_file):
        io.write_file(macros_file, tool_description.macro_contents)
    elif macros:
        io.info(REUSING_MACROS_MESSAGE)
    if tool_description.test_files:
        if not os.path.exists("test-data"):
            io.info("No test-data directory, creating one.")
            io.shell("mkdir -p 'test-data'")
        for test_file in tool_description.test_files:
            io.info("Copying test-file %s" % test_file)
            io.shell("cp '%s' 'test-data'" % test_file)
コード例 #20
0
    def test_diff_xunit(self):
        with self._isolate_repo("multi_repos_nested") as f:
            self._shed_create(recursive=True)

            xunit_report = tempfile.NamedTemporaryFile(delete=False)
            xunit_report.flush()
            xunit_report.close()
            diff_command = ["shed_diff", "-r", "--report_xunit", xunit_report.name]
            diff_command.extend(self._shed_args(read_only=True))
            known_good_xunit_report = os.path.join(TEST_REPOS_DIR,
                                                   'multi_repos_nested.xunit.xml')
            known_bad_xunit_report = os.path.join(TEST_REPOS_DIR,
                                                  'multi_repos_nested.xunit-bad.xml')
            self._check_exit_code(diff_command, exit_code=0)

            compare = open(xunit_report.name, 'r').read()
            if diff(
                self._make_deterministic(ElementTree.parse(known_good_xunit_report).getroot()),
                self._make_deterministic(ElementTree.fromstring(compare)),
                reporter=sys.stdout.write
            ):
                sys.stdout.write(compare)
                assert False, "XUnit report different from multi_repos_nested.xunit.xml."

            io.write_file(
                join(f, "cat1", "related_file"),
                "A related non-tool file (modified).\n",
            )
            self._check_exit_code(diff_command, exit_code=1)

            compare = open(xunit_report.name, 'r').read()
            if diff(
                self._make_deterministic(ElementTree.parse(known_bad_xunit_report).getroot()),
                self._make_deterministic(ElementTree.fromstring(compare)),
                reporter=sys.stdout.write
            ):
                sys.stdout.write(compare)
                assert False, "XUnit report different from multi_repos_nested.xunit-bad.xml."

            os.unlink(xunit_report.name)
コード例 #21
0
ファイル: tool_builder.py プロジェクト: stevecassidy/planemo
def write_tool_description(ctx, tool_description, **kwds):
    """Write a tool description to the file system guided by supplied CLI kwds."""
    tool_id = kwds.get("id")
    output = kwds.get("tool")
    if not output:
        extension = "cwl" if kwds.get("cwl") else "xml"
        output = "%s.%s" % (tool_id, extension)
    if not io.can_write_to_path(output, **kwds):
        ctx.exit(1)

    io.write_file(output, tool_description.contents)
    io.info("Tool written to %s" % output)
    test_contents = tool_description.test_contents
    if test_contents:
        sep = "-" if "-" in tool_id else "_"
        tests_path = "%s%stests.yml" % (kwds.get("id"), sep)
        if not io.can_write_to_path(tests_path, **kwds):
            ctx.exit(1)
        io.write_file(tests_path, test_contents)
        io.info("Tool tests written to %s" % tests_path)

    macros = kwds["macros"]
    macros_file = "macros.xml"
    if macros and not os.path.exists(macros_file):
        io.write_file(macros_file, tool_description.macro_contents)
    elif macros:
        io.info(REUSING_MACROS_MESSAGE)
    if tool_description.test_files:
        if not os.path.exists("test-data"):
            io.info("No test-data directory, creating one.")
            io.shell("mkdir -p 'test-data'")
        for test_file in tool_description.test_files:
            io.info("Copying test-file %s" % test_file)
            io.shell("cp '%s' 'test-data'" % test_file)
コード例 #22
0
ファイル: test_shed.py プロジェクト: galaxyproject/planemo
 def test_shed(self):
     shed_url = os.environ.get("TEST_TOOL_SHED_URL",
                               "http://localhost:9009")
     shed_api_key = os.environ.get("TEST_TOOL_SHED_API_KEY")
     tsi = toolshed.ToolShedInstance(shed_url, key=shed_api_key)
     owner = username(tsi)
     name = "planemotestrepo%d" % random.randint(0, 1000000)
     with self._isolate():
         shed_yml_contents = SHED_TEMPLATE.safe_substitute(
             owner=owner,
             name=name,
         )
         io.write_file(".shed.yml", shed_yml_contents)
         test_path = os.path.join(TEST_DIR, "tool_dependencies_good_1.xml")
         contents = open(test_path).read()
         io.write_file("tool_dependencies.xml", contents)
         init_cmd = [
             "shed_create",
             "--shed_key", shed_api_key,
             "--shed_target", shed_url
         ]
         self._check_exit_code(init_cmd)
         with open(".shed.yml", "r") as fh:
             contents_dict = yaml.safe_load(fh)
         contents_dict["description"] = "Update test repository."
         io.write_file(".shed.yml", yaml.dump(contents_dict))
         update_cmd = [
             "shed_update",
             "--shed_key", shed_api_key,
             "--shed_target", shed_url
         ]
         self._check_exit_code(update_cmd)
         upload_cmd = [
             "shed_upload",
             "--shed_key", shed_api_key,
             "--shed_target", shed_url
         ]
         self._check_exit_code(upload_cmd)
         download_cmd = [
             "shed_download",
             "--shed_target", shed_url,
             "--destination", "shed_download.tar.gz"
         ]
         self._check_exit_code(download_cmd)
コード例 #23
0
ファイル: test_shed.py プロジェクト: nebiolabs/planemo
 def test_shed(self):
     shed_url = os.environ.get("TEST_TOOL_SHED_URL",
                               "http://localhost:9009")
     shed_api_key = os.environ.get("TEST_TOOL_SHED_API_KEY")
     tsi = toolshed.ToolShedInstance(shed_url, key=shed_api_key)
     owner = username(tsi)
     name = "planemotestrepo%d" % random.randint(0, 1000000)
     with self._isolate():
         shed_yml_contents = SHED_TEMPLATE.safe_substitute(
             owner=owner,
             name=name,
         )
         io.write_file(".shed.yml", shed_yml_contents)
         test_path = os.path.join(TEST_DIR, "tool_dependencies_good_1.xml")
         contents = open(test_path).read()
         io.write_file("tool_dependencies.xml", contents)
         init_cmd = [
             "shed_create",
             "--shed_key", shed_api_key,
             "--shed_target", shed_url
         ]
         self._check_exit_code(init_cmd)
         contents_dict = yaml.load(open(".shed.yml", "r"))
         contents_dict["description"] = "Update test repository."
         io.write_file(".shed.yml", yaml.dump(contents_dict))
         update_cmd = [
             "shed_update",
             "--shed_key", shed_api_key,
             "--shed_target", shed_url
         ]
         self._check_exit_code(update_cmd)
         upload_cmd = [
             "shed_upload",
             "--shed_key", shed_api_key,
             "--shed_target", shed_url
         ]
         self._check_exit_code(upload_cmd)
         download_cmd = [
             "shed_download",
             "--shed_target", shed_url,
             "--destination", "shed_download.tar.gz"
         ]
         self._check_exit_code(download_cmd)
コード例 #24
0
def write_tool_description(ctx, tool_description, **kwds):
    """Write a tool description to the file system guided by supplied CLI kwds."""
    tool_id = kwds.get("id")
    output = kwds.get("tool")
    if not output:
        extension = "cwl" if kwds.get("cwl") else "xml"
        output = "%s.%s" % (tool_id, extension)
    if not io.can_write_to_path(output, **kwds):
        ctx.exit(1)

    io.write_file(output, tool_description.contents)
    io.info("Tool written to %s" % output)
    for tool_file in tool_description.tool_files:
        if tool_file.contents is None:
            continue

        path = tool_file.filename
        if not io.can_write_to_path(path, **kwds):
            ctx.exit(1)
        io.write_file(path, tool_file.contents)
        io.info("Tool %s written to %s" % (tool_file.description, path))

    macros = kwds["macros"]
    macros_file = "macros.xml"
    if macros and not os.path.exists(macros_file):
        io.write_file(macros_file, tool_description.macro_contents)
    elif macros:
        io.info(REUSING_MACROS_MESSAGE)
    if tool_description.test_files:
        if not os.path.exists("test-data"):
            io.info("No test-data directory, creating one.")
            os.makedirs('test-data')
        for test_file in tool_description.test_files:
            io.info("Copying test-file %s" % test_file)
            try:
                shutil.copy(test_file, 'test-data')
            except Exception as e:
                io.info("Copy of %s failed: %s" % (test_file, e))
コード例 #25
0
ファイル: tool_builder.py プロジェクト: martenson/planemo
def write_tool_description(ctx, tool_description, **kwds):
    """Write a tool description to the file system guided by supplied CLI kwds."""
    tool_id = kwds.get("id")
    output = kwds.get("tool")
    if not output:
        extension = "cwl" if kwds.get("cwl") else "xml"
        output = "%s.%s" % (tool_id, extension)
    if not io.can_write_to_path(output, **kwds):
        ctx.exit(1)

    io.write_file(output, tool_description.contents)
    io.info("Tool written to %s" % output)
    for tool_file in tool_description.tool_files:
        if tool_file.contents is None:
            continue

        path = tool_file.filename
        if not io.can_write_to_path(path, **kwds):
            ctx.exit(1)
        io.write_file(path, tool_file.contents)
        io.info("Tool %s written to %s" % (tool_file.description, path))

    macros = kwds["macros"]
    macros_file = "macros.xml"
    if macros and not os.path.exists(macros_file):
        io.write_file(macros_file, tool_description.macro_contents)
    elif macros:
        io.info(REUSING_MACROS_MESSAGE)
    if tool_description.test_files:
        if not os.path.exists("test-data"):
            io.info("No test-data directory, creating one.")
            os.makedirs('test-data')
        for test_file in tool_description.test_files:
            io.info("Copying test-file %s" % test_file)
            try:
                shutil.copy(test_file, 'test-data')
            except Exception as e:
                io.info("Copy of %s failed: %s" % (test_file, e))
コード例 #26
0
ファイル: galaxy_config.py プロジェクト: nebiolabs/planemo
def galaxy_config(ctx, tool_paths, for_tests=False, **kwds):
    test_data_dir = _find_test_data(tool_paths, **kwds)
    tool_data_table = _find_tool_data_table(
        tool_paths,
        test_data_dir=test_data_dir,
        **kwds
    )
    galaxy_root = _check_galaxy(ctx, **kwds)
    install_galaxy = galaxy_root is None
    config_directory = kwds.get("config_directory", None)

    def config_join(*args):
        return os.path.join(config_directory, *args)

    created_config_directory = False
    if not config_directory:
        created_config_directory = True
        config_directory = mkdtemp()
    try:
        latest_galaxy = False
        if _install_galaxy_if_needed(ctx, config_directory, kwds):
            latest_galaxy = True
            galaxy_root = config_join("galaxy-dev")

        _handle_dependency_resolution(config_directory, kwds)
        _handle_job_metrics(config_directory, kwds)
        tool_definition = _tool_conf_entry_for(tool_paths)
        empty_tool_conf = config_join("empty_tool_conf.xml")
        shed_tool_conf = _shed_tool_conf(install_galaxy, config_directory)
        tool_conf = config_join("tool_conf.xml")
        database_location = config_join("galaxy.sqlite")
        shed_tools_path = config_join("shed_tools")
        sheds_config_path = _configure_sheds_config_file(
            config_directory, **kwds
        )
        preseeded_database = True
        master_api_key = kwds.get("master_api_key", "test_key")
        dependency_dir = os.path.join(config_directory, "deps")

        try:
            _download_database_template(
                galaxy_root,
                database_location,
                latest=latest_galaxy
            )
        except Exception as e:
            print(e)
            # No network access - just roll forward from null.
            preseeded_database = False

        os.makedirs(shed_tools_path)
        server_name = "planemo%d" % random.randint(0, 100000)
        port = kwds.get("port", 9090)
        template_args = dict(
            port=port,
            host="127.0.0.1",
            server_name=server_name,
            temp_directory=config_directory,
            shed_tools_path=shed_tools_path,
            database_location=database_location,
            tool_definition=tool_definition,
            tool_conf=tool_conf,
            debug=kwds.get("debug", "true"),
            master_api_key=master_api_key,
            id_secret=kwds.get("id_secret", "test_secret"),
            log_level=kwds.get("log_level", "DEBUG"),
        )
        tool_config_file = "%s,%s" % (tool_conf, shed_tool_conf)
        properties = dict(
            tool_dependency_dir=dependency_dir,
            file_path="${temp_directory}/files",
            new_file_path="${temp_directory}/tmp",
            tool_config_file=tool_config_file,
            tool_sheds_config_file=sheds_config_path,
            check_migrate_tools="False",
            manage_dependency_relationships="False",
            job_working_directory="${temp_directory}/job_working_directory",
            template_cache_path="${temp_directory}/compiled_templates",
            citation_cache_type="file",
            citation_cache_data_dir="${temp_directory}/citations/data",
            citation_cache_lock_dir="${temp_directory}/citations/lock",
            collect_outputs_from="job_working_directory",
            database_auto_migrate="True",
            cleanup_job="never",
            master_api_key="${master_api_key}",
            id_secret="${id_secret}",
            log_level="${log_level}",
            debug="${debug}",
            watch_tools="auto",
            tool_data_table_config_path=tool_data_table,
            integrated_tool_panel_config=("${temp_directory}/"
                                          "integrated_tool_panel_conf.xml"),
            # Use in-memory database for kombu to avoid database contention
            # during tests.
            amqp_internal_connection="sqlalchemy+sqlite://",
            migrated_tools_config=empty_tool_conf,
            test_data_dir=test_data_dir,  # TODO: make gx respect this
        )
        if not for_tests:
            properties["database_connection"] = \
                "sqlite:///${database_location}?isolation_level=IMMEDIATE"

        _handle_kwd_overrides(properties, kwds)

        # TODO: consider following property
        # watch_tool = False
        # datatypes_config_file = config/datatypes_conf.xml
        # welcome_url = /static/welcome.html
        # logo_url = /
        # sanitize_all_html = True
        # serve_xss_vulnerable_mimetypes = False
        # track_jobs_in_database = None
        # outputs_to_working_directory = False
        # retry_job_output_collection = 0

        env = _build_env_for_galaxy(properties, template_args)
        if install_galaxy:
            _build_eggs_cache(ctx, env, kwds)
        _build_test_env(properties, env)
        env['GALAXY_TEST_SHED_TOOL_CONF'] = shed_tool_conf

        # No need to download twice - would GALAXY_TEST_DATABASE_CONNECTION
        # work?
        if preseeded_database:
            env["GALAXY_TEST_DB_TEMPLATE"] = os.path.abspath(database_location)
        env["GALAXY_TEST_UPLOAD_ASYNC"] = "false"
        env["GALAXY_DEVELOPMENT_ENVIRONMENT"] = "1"
        web_config = _sub(WEB_SERVER_CONFIG_TEMPLATE, template_args)
        write_file(config_join("galaxy.ini"), web_config)
        tool_conf_contents = _sub(TOOL_CONF_TEMPLATE, template_args)
        write_file(tool_conf, tool_conf_contents)
        write_file(empty_tool_conf, EMPTY_TOOL_CONF_TEMPLATE)

        shed_tool_conf_contents = _sub(SHED_TOOL_CONF_TEMPLATE, template_args)
        write_file(shed_tool_conf, shed_tool_conf_contents)

        yield GalaxyConfig(
            galaxy_root,
            config_directory,
            env,
            test_data_dir,
            port,
            server_name,
            master_api_key,
        )
    finally:
        cleanup = not kwds.get("no_cleanup", False)
        if created_config_directory and cleanup:
            shutil.rmtree(config_directory)
コード例 #27
0
def docker_galaxy_config(ctx, runnables, for_tests=False, **kwds):
    """Set up a ``GalaxyConfig`` for Docker container."""
    tool_paths = [r.path for r in runnables if r.has_tools]
    test_data_dir = _find_test_data(tool_paths, **kwds)

    with _config_directory(ctx, **kwds) as config_directory:

        def config_join(*args):
            return os.path.join(config_directory, *args)

        ensure_dependency_resolvers_conf_configured(
            ctx, kwds, os.path.join(config_directory, "resolvers_conf.xml"))
        _handle_job_metrics(config_directory, kwds)

        shed_tool_conf = "config/shed_tool_conf.xml"
        all_tool_paths = _all_tool_paths(runnables, **kwds)

        tool_directories = set([])  # Things to mount...
        for tool_path in all_tool_paths:
            directory = os.path.dirname(os.path.normpath(tool_path))
            if os.path.exists(directory):
                tool_directories.add(directory)

        # TODO: remap these.
        tool_volumes = []
        for tool_directory in tool_directories:
            volume = simple_docker_volume(tool_directory)
            tool_volumes.append(volume)

        empty_tool_conf = config_join("empty_tool_conf.xml")

        tool_conf = config_join("tool_conf.xml")

        shed_tool_path = kwds.get("shed_tool_path") or config_join(
            "shed_tools")
        _ensure_directory(shed_tool_path)

        sheds_config_path = _configure_sheds_config_file(
            ctx, config_directory, **kwds)
        port = _get_port(kwds)
        properties = _shared_galaxy_properties(config_directory,
                                               kwds,
                                               for_tests=for_tests)
        _handle_container_resolution(ctx, kwds, properties)
        master_api_key = _get_master_api_key(kwds)

        template_args = dict(
            shed_tool_path=shed_tool_path,
            tool_conf=tool_conf,
        )
        tool_config_file = "%s,%s" % (tool_conf, shed_tool_conf)

        _write_tool_conf(ctx, all_tool_paths, tool_conf)
        write_file(empty_tool_conf, EMPTY_TOOL_CONF_TEMPLATE)

        properties.update(
            dict(
                tool_config_file=tool_config_file,
                tool_sheds_config_file=sheds_config_path,
                migrated_tools_config=empty_tool_conf,
            ))

        server_name = "planemo%d" % random.randint(0, 100000)

        # Value substitutions in Galaxy properties - for consistency with
        # non-Dockerized version.
        template_args = dict()
        env = _build_env_for_galaxy(properties, template_args)
        env["NONUSE"] = "nodejs,proftp,reports"
        if ctx.verbose:
            env["GALAXY_LOGGING"] = "full"

        # TODO: setup FTP upload dir and disable FTP server in container.
        _build_test_env(properties, env)

        docker_target_kwds = docker_host_args(**kwds)
        volumes = tool_volumes + [simple_docker_volume(config_directory)]
        export_directory = kwds.get("export_directory", None)
        if export_directory is not None:
            volumes.append(DockerVolume("%s:/export:rw" % export_directory))

        # TODO: Allow this to real Docker volumes and allow multiple.
        extra_volume = kwds.get("docker_extra_volume")
        if extra_volume:
            volumes.append(simple_docker_volume(extra_volume))
        yield DockerGalaxyConfig(
            ctx,
            config_directory,
            env,
            test_data_dir,
            port,
            server_name,
            master_api_key,
            runnables,
            docker_target_kwds=docker_target_kwds,
            volumes=volumes,
            export_directory=export_directory,
            kwds=kwds,
        )
コード例 #28
0
ファイル: config.py プロジェクト: lecorguille/planemo
def galaxy_config(ctx, tool_paths, for_tests=False, **kwds):
    """Set up a ``GalaxyConfig`` in an auto-cleaned context."""
    test_data_dir = _find_test_data(tool_paths, **kwds)
    tool_data_table = _find_tool_data_table(
        tool_paths,
        test_data_dir=test_data_dir,
        **kwds
    )
    galaxy_root = _check_galaxy(ctx, **kwds)
    install_galaxy = galaxy_root is None
    config_directory = kwds.get("config_directory", None)

    def config_join(*args):
        return os.path.join(config_directory, *args)

    created_config_directory = False
    if not config_directory:
        created_config_directory = True
        config_directory = mkdtemp()
    try:
        latest_galaxy = False
        install_env = {}
        if install_galaxy:
            _build_eggs_cache(ctx, install_env, kwds)
            _install_galaxy(ctx, config_directory, install_env, kwds)
            latest_galaxy = True
            galaxy_root = config_join("galaxy-dev")

        _handle_dependency_resolution(config_directory, kwds)
        _handle_job_metrics(config_directory, kwds)
        file_path = kwds.get("file_path") or config_join("files")
        _ensure_directory(file_path)

        tool_dependency_dir = kwds.get("tool_dependency_dir") or config_join("deps")
        _ensure_directory(tool_dependency_dir)

        shed_tool_conf = kwds.get("shed_tool_conf") or config_join("shed_tools_conf.xml")
        tool_definition = _tool_conf_entry_for(tool_paths)
        empty_tool_conf = config_join("empty_tool_conf.xml")

        tool_conf = config_join("tool_conf.xml")
        database_location = config_join("galaxy.sqlite")

        shed_tool_path = kwds.get("shed_tool_path") or config_join("shed_tools")
        _ensure_directory(shed_tool_path)

        sheds_config_path = _configure_sheds_config_file(
            ctx, config_directory, **kwds
        )
        master_api_key = kwds.get("master_api_key", DEFAULT_MASTER_API_KEY)
        dependency_dir = os.path.join(config_directory, "deps")
        preseeded_database = attempt_database_preseed(
            galaxy_root,
            database_location,
            latest_galaxy=latest_galaxy,
            **kwds
        )
        _ensure_directory(shed_tool_path)
        server_name = "planemo%d" % random.randint(0, 100000)
        port = int(kwds.get("port", 9090))
        template_args = dict(
            port=port,
            host=kwds.get("host", "127.0.0.1"),
            server_name=server_name,
            temp_directory=config_directory,
            shed_tool_path=shed_tool_path,
            database_location=database_location,
            tool_definition=tool_definition,
            tool_conf=tool_conf,
            debug=kwds.get("debug", "true"),
            master_api_key=master_api_key,
            id_secret=kwds.get("id_secret", "test_secret"),
            log_level=kwds.get("log_level", "DEBUG"),
        )
        tool_config_file = "%s,%s" % (tool_conf, shed_tool_conf)
        user_email = kwds.get("galaxy_email")
        properties = dict(
            single_user=user_email,
            admin_users=user_email,
            ftp_upload_dir_template="${ftp_upload_dir}",
            ftp_upload_purge="False",
            ftp_upload_dir=test_data_dir or os.path.abspath('.'),
            ftp_upload_site="Test Data",
            tool_dependency_dir=dependency_dir,
            file_path=file_path,
            new_file_path="${temp_directory}/tmp",
            tool_config_file=tool_config_file,
            tool_sheds_config_file=sheds_config_path,
            check_migrate_tools="False",
            manage_dependency_relationships="False",
            job_working_directory="${temp_directory}/job_working_directory",
            template_cache_path="${temp_directory}/compiled_templates",
            citation_cache_type="file",
            citation_cache_data_dir="${temp_directory}/citations/data",
            citation_cache_lock_dir="${temp_directory}/citations/lock",
            collect_outputs_from="job_working_directory",
            database_auto_migrate="True",
            cleanup_job="never",
            master_api_key="${master_api_key}",
            enable_beta_tool_formats="True",
            id_secret="${id_secret}",
            log_level="${log_level}",
            debug="${debug}",
            watch_tools="auto",
            default_job_shell="/bin/bash",  # For conda dependency resolution
            tool_data_table_config_path=tool_data_table,
            integrated_tool_panel_config=("${temp_directory}/"
                                          "integrated_tool_panel_conf.xml"),
            # Use in-memory database for kombu to avoid database contention
            # during tests.
            amqp_internal_connection="sqlalchemy+sqlite://",
            migrated_tools_config=empty_tool_conf,
            test_data_dir=test_data_dir,  # TODO: make gx respect this
        )
        if not for_tests:
            properties["database_connection"] = _database_connection(database_location, **kwds)

        _handle_kwd_overrides(properties, kwds)

        # TODO: consider following property
        # watch_tool = False
        # datatypes_config_file = config/datatypes_conf.xml
        # welcome_url = /static/welcome.html
        # logo_url = /
        # sanitize_all_html = True
        # serve_xss_vulnerable_mimetypes = False
        # track_jobs_in_database = None
        # outputs_to_working_directory = False
        # retry_job_output_collection = 0

        env = _build_env_for_galaxy(properties, template_args)
        env.update(install_env)
        _build_test_env(properties, env)
        env['GALAXY_TEST_SHED_TOOL_CONF'] = shed_tool_conf

        # No need to download twice - would GALAXY_TEST_DATABASE_CONNECTION
        # work?
        if preseeded_database:
            env["GALAXY_TEST_DB_TEMPLATE"] = os.path.abspath(database_location)
        env["GALAXY_TEST_UPLOAD_ASYNC"] = "false"
        env["GALAXY_DEVELOPMENT_ENVIRONMENT"] = "1"
        web_config = _sub(WEB_SERVER_CONFIG_TEMPLATE, template_args)
        write_file(config_join("galaxy.ini"), web_config)
        tool_conf_contents = _sub(TOOL_CONF_TEMPLATE, template_args)
        write_file(tool_conf, tool_conf_contents)
        write_file(empty_tool_conf, EMPTY_TOOL_CONF_TEMPLATE)

        shed_tool_conf_contents = _sub(SHED_TOOL_CONF_TEMPLATE, template_args)
        # Write a new shed_tool_conf.xml if needed.
        write_file(shed_tool_conf, shed_tool_conf_contents, force=False)

        pid_file = kwds.get("pid_file") or config_join("galaxy.pid")

        yield GalaxyConfig(
            galaxy_root,
            pid_file,
            config_directory,
            env,
            test_data_dir,
            port,
            server_name,
            master_api_key,
        )
    finally:
        cleanup = not kwds.get("no_cleanup", False)
        if created_config_directory and cleanup:
            shutil.rmtree(config_directory)
コード例 #29
0
ファイル: config.py プロジェクト: pvanheus/planemo
def docker_galaxy_config(ctx, runnables, for_tests=False, **kwds):
    """Set up a ``GalaxyConfig`` for Docker container."""
    tool_paths = [r.path for r in runnables if r.has_tools]
    test_data_dir = _find_test_data(tool_paths, **kwds)

    with _config_directory(ctx, **kwds) as config_directory:
        def config_join(*args):
            return os.path.join(config_directory, *args)

        ensure_dependency_resolvers_conf_configured(ctx, kwds, os.path.join(config_directory, "resolvers_conf.xml"))
        _handle_job_metrics(config_directory, kwds)

        shed_tool_conf = "config/shed_tool_conf.xml"
        all_tool_paths = _all_tool_paths(runnables, **kwds)

        tool_directories = set([])  # Things to mount...
        for tool_path in all_tool_paths:
            directory = os.path.dirname(os.path.normpath(tool_path))
            if os.path.exists(directory):
                tool_directories.add(directory)

        # TODO: remap these.
        tool_volumes = []
        for tool_directory in tool_directories:
            volume = simple_docker_volume(tool_directory)
            tool_volumes.append(volume)

        empty_tool_conf = config_join("empty_tool_conf.xml")

        tool_conf = config_join("tool_conf.xml")

        shed_tool_path = kwds.get("shed_tool_path") or config_join("shed_tools")
        _ensure_directory(shed_tool_path)

        sheds_config_path = _configure_sheds_config_file(
            ctx, config_directory, **kwds
        )
        port = _get_port(kwds)
        properties = _shared_galaxy_properties(config_directory, kwds, for_tests=for_tests)
        _handle_container_resolution(ctx, kwds, properties)
        master_api_key = _get_master_api_key(kwds)

        template_args = dict(
            shed_tool_path=shed_tool_path,
            tool_conf=tool_conf,
        )
        tool_config_file = "%s,%s" % (tool_conf, shed_tool_conf)

        _write_tool_conf(ctx, all_tool_paths, tool_conf)
        write_file(empty_tool_conf, EMPTY_TOOL_CONF_TEMPLATE)

        properties.update(dict(
            tool_config_file=tool_config_file,
            tool_sheds_config_file=sheds_config_path,
            migrated_tools_config=empty_tool_conf,
        ))

        server_name = "planemo%d" % random.randint(0, 100000)

        # Value substitutions in Galaxy properties - for consistency with
        # non-Dockerized version.
        template_args = dict(
        )
        env = _build_env_for_galaxy(properties, template_args)
        env["NONUSE"] = "nodejs,proftp,reports"
        if ctx.verbose:
            env["GALAXY_LOGGING"] = "full"

        # TODO: setup FTP upload dir and disable FTP server in container.
        _build_test_env(properties, env)

        docker_target_kwds = docker_host_args(**kwds)
        volumes = tool_volumes + [simple_docker_volume(config_directory)]
        export_directory = kwds.get("export_directory", None)
        if export_directory is not None:
            volumes.append(DockerVolume("%s:/export:rw" % export_directory))

        # TODO: Allow this to real Docker volumes and allow multiple.
        extra_volume = kwds.get("docker_extra_volume")
        if extra_volume:
            volumes.append(simple_docker_volume(extra_volume))
        yield DockerGalaxyConfig(
            ctx,
            config_directory,
            env,
            test_data_dir,
            port,
            server_name,
            master_api_key,
            runnables,
            docker_target_kwds=docker_target_kwds,
            volumes=volumes,
            export_directory=export_directory,
            kwds=kwds,
        )
コード例 #30
0
ファイル: config.py プロジェクト: pvanheus/planemo
def local_galaxy_config(ctx, runnables, for_tests=False, **kwds):
    """Set up a ``GalaxyConfig`` in an auto-cleaned context."""
    tool_paths = [r.path for r in runnables if r.has_tools]
    test_data_dir = _find_test_data(tool_paths, **kwds)
    tool_data_table = _find_tool_data_table(
        tool_paths,
        test_data_dir=test_data_dir,
        **kwds
    )
    galaxy_root = _find_galaxy_root(ctx, **kwds)
    install_galaxy = kwds.get("install_galaxy", False)
    if galaxy_root is not None:
        if os.path.isdir(galaxy_root) and not os.listdir(galaxy_root):
            os.rmdir(galaxy_root)
        if os.path.isdir(galaxy_root) and install_galaxy:
            raise Exception("%s is an existing non-empty directory, cannot install Galaxy again" % galaxy_root)

    # Duplicate block in docker variant above.
    if kwds.get("mulled_containers", False) and not kwds.get("docker", False):
        if ctx.get_option_source("docker") != OptionSource.cli:
            kwds["docker"] = True
        else:
            raise Exception("Specified no docker and mulled containers together.")

    with _config_directory(ctx, **kwds) as config_directory:
        def config_join(*args):
            return os.path.join(config_directory, *args)

        install_env = {}
        if kwds.get('galaxy_skip_client_build', True):
            install_env['GALAXY_SKIP_CLIENT_BUILD'] = '1'
        if galaxy_root is None:
            galaxy_root = config_join("galaxy-dev")
        if not os.path.isdir(galaxy_root):
            _build_eggs_cache(ctx, install_env, kwds)
            _install_galaxy(ctx, galaxy_root, install_env, kwds)

        if parse_version(kwds.get('galaxy_python_version') or DEFAULT_PYTHON_VERSION) >= parse_version('3'):
            # on python 3 we use gunicorn,
            # which requires 'main' as server name
            server_name = 'main'
        else:
            server_name = "planemo%d" % random.randint(0, 100000)
        # Once we don't have to support earlier than 18.01 - try putting these files
        # somewhere better than with Galaxy.
        log_file = "%s.log" % server_name
        pid_file = "%s.pid" % server_name
        ensure_dependency_resolvers_conf_configured(ctx, kwds, os.path.join(config_directory, "resolvers_conf.xml"))
        _handle_job_config_file(config_directory, server_name, kwds)
        _handle_job_metrics(config_directory, kwds)
        file_path = kwds.get("file_path") or config_join("files")
        _ensure_directory(file_path)

        tool_dependency_dir = kwds.get("tool_dependency_dir") or config_join("deps")
        _ensure_directory(tool_dependency_dir)

        shed_tool_conf = kwds.get("shed_tool_conf") or config_join("shed_tools_conf.xml")
        all_tool_paths = _all_tool_paths(runnables, **kwds)
        empty_tool_conf = config_join("empty_tool_conf.xml")

        tool_conf = config_join("tool_conf.xml")

        shed_data_manager_config_file = config_join("shed_data_manager_conf.xml")

        shed_tool_path = kwds.get("shed_tool_path") or config_join("shed_tools")
        _ensure_directory(shed_tool_path)

        sheds_config_path = _configure_sheds_config_file(
            ctx, config_directory, **kwds
        )

        database_location = config_join("galaxy.sqlite")
        master_api_key = _get_master_api_key(kwds)
        dependency_dir = os.path.join(config_directory, "deps")
        preseeded_database = attempt_database_preseed(
            ctx,
            galaxy_root,
            database_location,
            **kwds
        )
        _ensure_directory(shed_tool_path)
        port = _get_port(kwds)
        template_args = dict(
            port=port,
            host=kwds.get("host", "127.0.0.1"),
            server_name=server_name,
            temp_directory=config_directory,
            shed_tool_path=shed_tool_path,
            database_location=database_location,
            tool_conf=tool_conf,
            debug=kwds.get("debug", "true"),
            id_secret=kwds.get("id_secret", "test_secret"),
            log_level="DEBUG" if ctx.verbose else "INFO",
        )
        tool_config_file = "%s,%s" % (tool_conf, shed_tool_conf)
        # Setup both galaxy_email and older test user [email protected]
        # as admins for command_line, etc...
        properties = _shared_galaxy_properties(config_directory, kwds, for_tests=for_tests)
        properties.update(dict(
            server_name="main",
            ftp_upload_dir_template="${ftp_upload_dir}",
            ftp_upload_purge="False",
            ftp_upload_dir=test_data_dir or os.path.abspath('.'),
            ftp_upload_site="Test Data",
            check_upload_content="False",
            tool_dependency_dir=dependency_dir,
            file_path=file_path,
            new_file_path="${temp_directory}/tmp",
            tool_config_file=tool_config_file,
            tool_sheds_config_file=sheds_config_path,
            manage_dependency_relationships="False",
            job_working_directory="${temp_directory}/job_working_directory",
            template_cache_path="${temp_directory}/compiled_templates",
            citation_cache_type="file",
            citation_cache_data_dir="${temp_directory}/citations/data",
            citation_cache_lock_dir="${temp_directory}/citations/lock",
            database_auto_migrate="True",
            enable_beta_tool_formats="True",
            id_secret="${id_secret}",
            log_level="${log_level}",
            debug="${debug}",
            watch_tools="auto",
            default_job_shell="/bin/bash",  # For conda dependency resolution
            tool_data_table_config_path=tool_data_table,
            integrated_tool_panel_config=("${temp_directory}/"
                                          "integrated_tool_panel_conf.xml"),
            # Use in-memory database for kombu to avoid database contention
            # during tests.
            amqp_internal_connection="sqlalchemy+sqlite://",
            migrated_tools_config=empty_tool_conf,
            test_data_dir=test_data_dir,  # TODO: make gx respect this
            shed_data_manager_config_file=shed_data_manager_config_file,
        ))
        _handle_container_resolution(ctx, kwds, properties)
        write_file(config_join("logging.ini"), _sub(LOGGING_TEMPLATE, template_args))
        if not for_tests:
            properties["database_connection"] = _database_connection(database_location, **kwds)

        _handle_kwd_overrides(properties, kwds)

        # TODO: consider following property
        # watch_tool = False
        # datatypes_config_file = config/datatypes_conf.xml
        # welcome_url = /static/welcome.html
        # logo_url = /
        # sanitize_all_html = True
        # serve_xss_vulnerable_mimetypes = False
        # track_jobs_in_database = None
        # outputs_to_working_directory = False
        # retry_job_output_collection = 0

        env = _build_env_for_galaxy(properties, template_args)
        env.update(install_env)
        _build_test_env(properties, env)
        env['GALAXY_TEST_SHED_TOOL_CONF'] = shed_tool_conf

        # No need to download twice - would GALAXY_TEST_DATABASE_CONNECTION
        # work?
        if preseeded_database:
            env["GALAXY_TEST_DB_TEMPLATE"] = os.path.abspath(database_location)
        env["GALAXY_TEST_UPLOAD_ASYNC"] = "false"
        env["GALAXY_TEST_LOGGING_CONFIG"] = config_join("logging.ini")
        env["GALAXY_DEVELOPMENT_ENVIRONMENT"] = "1"
        # Following are needed in 18.01 to prevent Galaxy from changing log and pid.
        # https://github.com/galaxyproject/planemo/issues/788
        env["GALAXY_LOG"] = log_file
        env["GALAXY_PID"] = pid_file
        web_config = _sub(WEB_SERVER_CONFIG_TEMPLATE, template_args)
        write_file(config_join("galaxy.ini"), web_config)
        _write_tool_conf(ctx, all_tool_paths, tool_conf)
        write_file(empty_tool_conf, EMPTY_TOOL_CONF_TEMPLATE)

        shed_tool_conf_contents = _sub(SHED_TOOL_CONF_TEMPLATE, template_args)
        # Write a new shed_tool_conf.xml if needed.
        write_file(shed_tool_conf, shed_tool_conf_contents, force=False)

        write_file(shed_data_manager_config_file, SHED_DATA_MANAGER_CONF_TEMPLATE)

        yield LocalGalaxyConfig(
            ctx,
            config_directory,
            env,
            test_data_dir,
            port,
            server_name,
            master_api_key,
            runnables,
            galaxy_root,
            kwds,
        )
コード例 #31
0
def local_galaxy_config(ctx, runnables, for_tests=False, **kwds):
    """Set up a ``GalaxyConfig`` in an auto-cleaned context."""
    tool_paths = [r.path for r in runnables if r.has_tools]
    test_data_dir = _find_test_data(tool_paths, **kwds)
    tool_data_table = _find_tool_data_table(tool_paths,
                                            test_data_dir=test_data_dir,
                                            **kwds)
    galaxy_root = _check_galaxy(ctx, **kwds)
    install_galaxy = galaxy_root is None

    # Duplicate block in docker variant above.
    if kwds.get("mulled_containers", False) and not kwds.get("docker", False):
        if ctx.get_option_source("docker") != OptionSource.cli:
            kwds["docker"] = True
        else:
            raise Exception(
                "Specified no docker and mulled containers together.")

    with _config_directory(ctx, **kwds) as config_directory:

        def config_join(*args):
            return os.path.join(config_directory, *args)

        latest_galaxy = False
        install_env = {}
        if install_galaxy:
            _build_eggs_cache(ctx, install_env, kwds)
            _install_galaxy(ctx, config_directory, install_env, kwds)
            latest_galaxy = True
            galaxy_root = config_join("galaxy-dev")

        server_name = "planemo%d" % random.randint(0, 100000)
        _handle_dependency_resolution(ctx, config_directory, kwds)
        _handle_job_config_file(config_directory, server_name, kwds)
        _handle_job_metrics(config_directory, kwds)
        file_path = kwds.get("file_path") or config_join("files")
        _ensure_directory(file_path)

        tool_dependency_dir = kwds.get("tool_dependency_dir") or config_join(
            "deps")
        _ensure_directory(tool_dependency_dir)

        shed_tool_conf = kwds.get("shed_tool_conf") or config_join(
            "shed_tools_conf.xml")
        all_tool_paths = list(tool_paths) + list(kwds.get("extra_tools", []))
        tool_definition = _tool_conf_entry_for(all_tool_paths)
        empty_tool_conf = config_join("empty_tool_conf.xml")

        tool_conf = config_join("tool_conf.xml")

        shed_data_manager_config_file = config_join(
            "shed_data_manager_conf.xml")

        shed_tool_path = kwds.get("shed_tool_path") or config_join(
            "shed_tools")
        _ensure_directory(shed_tool_path)

        sheds_config_path = _configure_sheds_config_file(
            ctx, config_directory, **kwds)

        database_location = config_join("galaxy.sqlite")
        master_api_key = _get_master_api_key(kwds)
        dependency_dir = os.path.join(config_directory, "deps")
        preseeded_database = attempt_database_preseed(
            galaxy_root,
            database_location,
            latest_galaxy=latest_galaxy,
            **kwds)
        _ensure_directory(shed_tool_path)
        port = _get_port(kwds)
        template_args = dict(
            port=port,
            host=kwds.get("host", "127.0.0.1"),
            server_name=server_name,
            temp_directory=config_directory,
            shed_tool_path=shed_tool_path,
            database_location=database_location,
            tool_definition=tool_definition,
            tool_conf=tool_conf,
            debug=kwds.get("debug", "true"),
            id_secret=kwds.get("id_secret", "test_secret"),
            log_level="DEBUG" if ctx.verbose else "INFO",
        )
        tool_config_file = "%s,%s" % (tool_conf, shed_tool_conf)
        # Setup both galaxy_email and older test user [email protected]
        # as admins for command_line, etc...
        properties = _shared_galaxy_properties(config_directory,
                                               kwds,
                                               for_tests=for_tests)
        properties.update(
            dict(
                server_name="main",
                ftp_upload_dir_template="${ftp_upload_dir}",
                ftp_upload_purge="False",
                ftp_upload_dir=test_data_dir or os.path.abspath('.'),
                ftp_upload_site="Test Data",
                tool_dependency_dir=dependency_dir,
                file_path=file_path,
                new_file_path="${temp_directory}/tmp",
                tool_config_file=tool_config_file,
                tool_sheds_config_file=sheds_config_path,
                manage_dependency_relationships="False",
                job_working_directory="${temp_directory}/job_working_directory",
                template_cache_path="${temp_directory}/compiled_templates",
                citation_cache_type="file",
                citation_cache_data_dir="${temp_directory}/citations/data",
                citation_cache_lock_dir="${temp_directory}/citations/lock",
                database_auto_migrate="True",
                enable_beta_tool_formats="True",
                id_secret="${id_secret}",
                log_level="${log_level}",
                debug="${debug}",
                watch_tools="auto",
                default_job_shell="/bin/bash",  # For conda dependency resolution
                tool_data_table_config_path=tool_data_table,
                integrated_tool_panel_config=(
                    "${temp_directory}/"
                    "integrated_tool_panel_conf.xml"),
                # Use in-memory database for kombu to avoid database contention
                # during tests.
                amqp_internal_connection="sqlalchemy+sqlite://",
                migrated_tools_config=empty_tool_conf,
                test_data_dir=test_data_dir,  # TODO: make gx respect this
                shed_data_manager_config_file=shed_data_manager_config_file,
            ))
        _handle_container_resolution(ctx, kwds, properties)
        write_file(config_join("logging.ini"),
                   _sub(LOGGING_TEMPLATE, template_args))
        if not for_tests:
            properties["database_connection"] = _database_connection(
                database_location, **kwds)

        _handle_kwd_overrides(properties, kwds)

        # TODO: consider following property
        # watch_tool = False
        # datatypes_config_file = config/datatypes_conf.xml
        # welcome_url = /static/welcome.html
        # logo_url = /
        # sanitize_all_html = True
        # serve_xss_vulnerable_mimetypes = False
        # track_jobs_in_database = None
        # outputs_to_working_directory = False
        # retry_job_output_collection = 0

        env = _build_env_for_galaxy(properties, template_args)
        env.update(install_env)
        _build_test_env(properties, env)
        env['GALAXY_TEST_SHED_TOOL_CONF'] = shed_tool_conf

        # No need to download twice - would GALAXY_TEST_DATABASE_CONNECTION
        # work?
        if preseeded_database:
            env["GALAXY_TEST_DB_TEMPLATE"] = os.path.abspath(database_location)
        env["GALAXY_TEST_UPLOAD_ASYNC"] = "false"
        env["GALAXY_TEST_LOGGING_CONFIG"] = config_join("logging.ini")
        env["GALAXY_DEVELOPMENT_ENVIRONMENT"] = "1"
        web_config = _sub(WEB_SERVER_CONFIG_TEMPLATE, template_args)
        write_file(config_join("galaxy.ini"), web_config)
        tool_conf_contents = _sub(TOOL_CONF_TEMPLATE, template_args)
        write_file(tool_conf, tool_conf_contents)
        write_file(empty_tool_conf, EMPTY_TOOL_CONF_TEMPLATE)

        shed_tool_conf_contents = _sub(SHED_TOOL_CONF_TEMPLATE, template_args)
        # Write a new shed_tool_conf.xml if needed.
        write_file(shed_tool_conf, shed_tool_conf_contents, force=False)

        write_file(shed_data_manager_config_file,
                   SHED_DATA_MANAGER_CONF_TEMPLATE)

        pid_file = kwds.get("pid_file") or config_join("galaxy.pid")

        yield LocalGalaxyConfig(
            config_directory,
            env,
            test_data_dir,
            port,
            server_name,
            master_api_key,
            runnables,
            galaxy_root,
            pid_file,
        )
コード例 #32
0
ファイル: config.py プロジェクト: nturaga/planemo
def local_galaxy_config(ctx, runnables, for_tests=False, **kwds):
    """Set up a ``GalaxyConfig`` in an auto-cleaned context."""
    tool_paths = [r.path for r in runnables if r.has_tools]
    test_data_dir = _find_test_data(tool_paths, **kwds)
    tool_data_table = _find_tool_data_table(
        tool_paths,
        test_data_dir=test_data_dir,
        **kwds
    )
    galaxy_root = _check_galaxy(ctx, **kwds)
    install_galaxy = galaxy_root is None

    # Duplicate block in docker variant above.
    if kwds.get("mulled_containers", False) and not kwds.get("docker", False):
        if ctx.get_option_source("docker") != OptionSource.cli:
            kwds["docker"] = True
        else:
            raise Exception("Specified no docker and mulled containers together.")

    with _config_directory(ctx, **kwds) as config_directory:
        def config_join(*args):
            return os.path.join(config_directory, *args)

        latest_galaxy = False
        install_env = {}
        if install_galaxy:
            _build_eggs_cache(ctx, install_env, kwds)
            _install_galaxy(ctx, config_directory, install_env, kwds)
            latest_galaxy = True
            galaxy_root = config_join("galaxy-dev")

        server_name = "planemo%d" % random.randint(0, 100000)
        _handle_dependency_resolution(ctx, config_directory, kwds)
        _handle_job_config_file(config_directory, server_name, kwds)
        _handle_job_metrics(config_directory, kwds)
        file_path = kwds.get("file_path") or config_join("files")
        _ensure_directory(file_path)

        tool_dependency_dir = kwds.get("tool_dependency_dir") or config_join("deps")
        _ensure_directory(tool_dependency_dir)

        shed_tool_conf = kwds.get("shed_tool_conf") or config_join("shed_tools_conf.xml")
        all_tool_paths = list(tool_paths) + list(kwds.get("extra_tools", []))
        tool_definition = _tool_conf_entry_for(all_tool_paths)
        empty_tool_conf = config_join("empty_tool_conf.xml")

        tool_conf = config_join("tool_conf.xml")

        tool_conf = config_join("tool_conf.xml")
        shed_tool_path = kwds.get("shed_tool_path") or config_join("shed_tools")
        _ensure_directory(shed_tool_path)

        sheds_config_path = _configure_sheds_config_file(
            ctx, config_directory, **kwds
        )

        database_location = config_join("galaxy.sqlite")
        master_api_key = _get_master_api_key(kwds)
        dependency_dir = os.path.join(config_directory, "deps")
        preseeded_database = attempt_database_preseed(
            galaxy_root,
            database_location,
            latest_galaxy=latest_galaxy,
            **kwds
        )
        _ensure_directory(shed_tool_path)
        port = _get_port(kwds)
        template_args = dict(
            port=port,
            host=kwds.get("host", "127.0.0.1"),
            server_name=server_name,
            temp_directory=config_directory,
            shed_tool_path=shed_tool_path,
            database_location=database_location,
            tool_definition=tool_definition,
            tool_conf=tool_conf,
            debug=kwds.get("debug", "true"),
            id_secret=kwds.get("id_secret", "test_secret"),
            log_level=kwds.get("log_level", "DEBUG"),
        )
        tool_config_file = "%s,%s" % (tool_conf, shed_tool_conf)
        # Setup both galaxy_email and older test user [email protected]
        # as admins for command_line, etc...
        properties = _shared_galaxy_properties(kwds)
        properties.update(dict(
            server_name="main",
            ftp_upload_dir_template="${ftp_upload_dir}",
            ftp_upload_purge="False",
            ftp_upload_dir=test_data_dir or os.path.abspath('.'),
            ftp_upload_site="Test Data",
            tool_dependency_dir=dependency_dir,
            file_path=file_path,
            new_file_path="${temp_directory}/tmp",
            tool_config_file=tool_config_file,
            tool_sheds_config_file=sheds_config_path,
            manage_dependency_relationships="False",
            job_working_directory="${temp_directory}/job_working_directory",
            template_cache_path="${temp_directory}/compiled_templates",
            citation_cache_type="file",
            citation_cache_data_dir="${temp_directory}/citations/data",
            citation_cache_lock_dir="${temp_directory}/citations/lock",
            database_auto_migrate="True",
            enable_beta_tool_formats="True",
            id_secret="${id_secret}",
            log_level="${log_level}",
            debug="${debug}",
            watch_tools="auto",
            default_job_shell="/bin/bash",  # For conda dependency resolution
            tool_data_table_config_path=tool_data_table,
            integrated_tool_panel_config=("${temp_directory}/"
                                          "integrated_tool_panel_conf.xml"),
            # Use in-memory database for kombu to avoid database contention
            # during tests.
            amqp_internal_connection="sqlalchemy+sqlite://",
            migrated_tools_config=empty_tool_conf,
            test_data_dir=test_data_dir,  # TODO: make gx respect this
        ))
        _handle_container_resolution(ctx, kwds, properties)
        if not for_tests:
            properties["database_connection"] = _database_connection(database_location, **kwds)

        _handle_kwd_overrides(properties, kwds)

        # TODO: consider following property
        # watch_tool = False
        # datatypes_config_file = config/datatypes_conf.xml
        # welcome_url = /static/welcome.html
        # logo_url = /
        # sanitize_all_html = True
        # serve_xss_vulnerable_mimetypes = False
        # track_jobs_in_database = None
        # outputs_to_working_directory = False
        # retry_job_output_collection = 0

        env = _build_env_for_galaxy(properties, template_args)
        env.update(install_env)
        _build_test_env(properties, env)
        env['GALAXY_TEST_SHED_TOOL_CONF'] = shed_tool_conf

        # No need to download twice - would GALAXY_TEST_DATABASE_CONNECTION
        # work?
        if preseeded_database:
            env["GALAXY_TEST_DB_TEMPLATE"] = os.path.abspath(database_location)
        env["GALAXY_TEST_UPLOAD_ASYNC"] = "false"
        env["GALAXY_DEVELOPMENT_ENVIRONMENT"] = "1"
        web_config = _sub(WEB_SERVER_CONFIG_TEMPLATE, template_args)
        write_file(config_join("galaxy.ini"), web_config)
        tool_conf_contents = _sub(TOOL_CONF_TEMPLATE, template_args)
        write_file(tool_conf, tool_conf_contents)
        write_file(empty_tool_conf, EMPTY_TOOL_CONF_TEMPLATE)

        shed_tool_conf_contents = _sub(SHED_TOOL_CONF_TEMPLATE, template_args)
        # Write a new shed_tool_conf.xml if needed.
        write_file(shed_tool_conf, shed_tool_conf_contents, force=False)

        pid_file = kwds.get("pid_file") or config_join("galaxy.pid")

        yield LocalGalaxyConfig(
            config_directory,
            env,
            test_data_dir,
            port,
            server_name,
            master_api_key,
            runnables,
            galaxy_root,
            pid_file,
        )
コード例 #33
0
ファイル: config.py プロジェクト: lecorguille/planemo
def galaxy_config(ctx, tool_paths, for_tests=False, **kwds):
    """Set up a ``GalaxyConfig`` in an auto-cleaned context."""
    test_data_dir = _find_test_data(tool_paths, **kwds)
    tool_data_table = _find_tool_data_table(tool_paths,
                                            test_data_dir=test_data_dir,
                                            **kwds)
    galaxy_root = _check_galaxy(ctx, **kwds)
    install_galaxy = galaxy_root is None
    config_directory = kwds.get("config_directory", None)

    def config_join(*args):
        return os.path.join(config_directory, *args)

    created_config_directory = False
    if not config_directory:
        created_config_directory = True
        config_directory = mkdtemp()
    try:
        latest_galaxy = False
        install_env = {}
        if install_galaxy:
            _build_eggs_cache(ctx, install_env, kwds)
            _install_galaxy(ctx, config_directory, install_env, kwds)
            latest_galaxy = True
            galaxy_root = config_join("galaxy-dev")

        _handle_dependency_resolution(config_directory, kwds)
        _handle_job_metrics(config_directory, kwds)
        file_path = kwds.get("file_path") or config_join("files")
        _ensure_directory(file_path)

        tool_dependency_dir = kwds.get("tool_dependency_dir") or config_join(
            "deps")
        _ensure_directory(tool_dependency_dir)

        shed_tool_conf = kwds.get("shed_tool_conf") or config_join(
            "shed_tools_conf.xml")
        tool_definition = _tool_conf_entry_for(tool_paths)
        empty_tool_conf = config_join("empty_tool_conf.xml")

        tool_conf = config_join("tool_conf.xml")
        database_location = config_join("galaxy.sqlite")

        shed_tool_path = kwds.get("shed_tool_path") or config_join(
            "shed_tools")
        _ensure_directory(shed_tool_path)

        sheds_config_path = _configure_sheds_config_file(
            ctx, config_directory, **kwds)
        master_api_key = kwds.get("master_api_key", DEFAULT_MASTER_API_KEY)
        dependency_dir = os.path.join(config_directory, "deps")
        preseeded_database = attempt_database_preseed(
            galaxy_root,
            database_location,
            latest_galaxy=latest_galaxy,
            **kwds)
        _ensure_directory(shed_tool_path)
        server_name = "planemo%d" % random.randint(0, 100000)
        port = int(kwds.get("port", 9090))
        template_args = dict(
            port=port,
            host=kwds.get("host", "127.0.0.1"),
            server_name=server_name,
            temp_directory=config_directory,
            shed_tool_path=shed_tool_path,
            database_location=database_location,
            tool_definition=tool_definition,
            tool_conf=tool_conf,
            debug=kwds.get("debug", "true"),
            master_api_key=master_api_key,
            id_secret=kwds.get("id_secret", "test_secret"),
            log_level=kwds.get("log_level", "DEBUG"),
        )
        tool_config_file = "%s,%s" % (tool_conf, shed_tool_conf)
        user_email = kwds.get("galaxy_email")
        properties = dict(
            single_user=user_email,
            admin_users=user_email,
            ftp_upload_dir_template="${ftp_upload_dir}",
            ftp_upload_purge="False",
            ftp_upload_dir=test_data_dir or os.path.abspath('.'),
            ftp_upload_site="Test Data",
            tool_dependency_dir=dependency_dir,
            file_path=file_path,
            new_file_path="${temp_directory}/tmp",
            tool_config_file=tool_config_file,
            tool_sheds_config_file=sheds_config_path,
            check_migrate_tools="False",
            manage_dependency_relationships="False",
            job_working_directory="${temp_directory}/job_working_directory",
            template_cache_path="${temp_directory}/compiled_templates",
            citation_cache_type="file",
            citation_cache_data_dir="${temp_directory}/citations/data",
            citation_cache_lock_dir="${temp_directory}/citations/lock",
            collect_outputs_from="job_working_directory",
            database_auto_migrate="True",
            cleanup_job="never",
            master_api_key="${master_api_key}",
            enable_beta_tool_formats="True",
            id_secret="${id_secret}",
            log_level="${log_level}",
            debug="${debug}",
            watch_tools="auto",
            default_job_shell="/bin/bash",  # For conda dependency resolution
            tool_data_table_config_path=tool_data_table,
            integrated_tool_panel_config=("${temp_directory}/"
                                          "integrated_tool_panel_conf.xml"),
            # Use in-memory database for kombu to avoid database contention
            # during tests.
            amqp_internal_connection="sqlalchemy+sqlite://",
            migrated_tools_config=empty_tool_conf,
            test_data_dir=test_data_dir,  # TODO: make gx respect this
        )
        if not for_tests:
            properties["database_connection"] = _database_connection(
                database_location, **kwds)

        _handle_kwd_overrides(properties, kwds)

        # TODO: consider following property
        # watch_tool = False
        # datatypes_config_file = config/datatypes_conf.xml
        # welcome_url = /static/welcome.html
        # logo_url = /
        # sanitize_all_html = True
        # serve_xss_vulnerable_mimetypes = False
        # track_jobs_in_database = None
        # outputs_to_working_directory = False
        # retry_job_output_collection = 0

        env = _build_env_for_galaxy(properties, template_args)
        env.update(install_env)
        _build_test_env(properties, env)
        env['GALAXY_TEST_SHED_TOOL_CONF'] = shed_tool_conf

        # No need to download twice - would GALAXY_TEST_DATABASE_CONNECTION
        # work?
        if preseeded_database:
            env["GALAXY_TEST_DB_TEMPLATE"] = os.path.abspath(database_location)
        env["GALAXY_TEST_UPLOAD_ASYNC"] = "false"
        env["GALAXY_DEVELOPMENT_ENVIRONMENT"] = "1"
        web_config = _sub(WEB_SERVER_CONFIG_TEMPLATE, template_args)
        write_file(config_join("galaxy.ini"), web_config)
        tool_conf_contents = _sub(TOOL_CONF_TEMPLATE, template_args)
        write_file(tool_conf, tool_conf_contents)
        write_file(empty_tool_conf, EMPTY_TOOL_CONF_TEMPLATE)

        shed_tool_conf_contents = _sub(SHED_TOOL_CONF_TEMPLATE, template_args)
        # Write a new shed_tool_conf.xml if needed.
        write_file(shed_tool_conf, shed_tool_conf_contents, force=False)

        pid_file = kwds.get("pid_file") or config_join("galaxy.pid")

        yield GalaxyConfig(
            galaxy_root,
            pid_file,
            config_directory,
            env,
            test_data_dir,
            port,
            server_name,
            master_api_key,
        )
    finally:
        cleanup = not kwds.get("no_cleanup", False)
        if created_config_directory and cleanup:
            shutil.rmtree(config_directory)
コード例 #34
0
def local_galaxy_config(ctx, runnables, for_tests=False, **kwds):
    """Set up a ``GalaxyConfig`` in an auto-cleaned context."""
    tool_paths = [r.path for r in runnables if r.has_tools]
    test_data_dir = _find_test_data(tool_paths, **kwds)
    tool_data_table = _find_tool_data_table(tool_paths,
                                            test_data_dir=test_data_dir,
                                            **kwds)
    galaxy_root = _find_galaxy_root(ctx, **kwds)
    install_galaxy = kwds.get("install_galaxy", False)
    if galaxy_root is not None:
        if os.path.isdir(galaxy_root) and not os.listdir(galaxy_root):
            os.rmdir(galaxy_root)
        if os.path.isdir(galaxy_root) and install_galaxy:
            raise Exception(
                "%s is an existing non-empty directory, cannot install Galaxy again"
                % galaxy_root)

    # Duplicate block in docker variant above.
    if kwds.get("mulled_containers", False) and not kwds.get("docker", False):
        if ctx.get_option_source("docker") != OptionSource.cli:
            kwds["docker"] = True
        else:
            raise Exception(
                "Specified no docker and mulled containers together.")

    with _config_directory(ctx, **kwds) as config_directory:

        def config_join(*args):
            return os.path.join(config_directory, *args)

        install_env = {}
        if kwds.get('galaxy_skip_client_build', True):
            install_env['GALAXY_SKIP_CLIENT_BUILD'] = '1'
        if galaxy_root is None:
            galaxy_root = config_join("galaxy-dev")
        if not os.path.isdir(galaxy_root):
            _build_eggs_cache(ctx, install_env, kwds)
            _install_galaxy(ctx, galaxy_root, install_env, kwds)

        if parse_version(
                kwds.get('galaxy_python_version')
                or DEFAULT_PYTHON_VERSION) >= parse_version('3'):
            # on python 3 we use gunicorn,
            # which requires 'main' as server name
            server_name = 'main'
        else:
            server_name = "planemo%d" % random.randint(0, 100000)
        # Once we don't have to support earlier than 18.01 - try putting these files
        # somewhere better than with Galaxy.
        log_file = "%s.log" % server_name
        pid_file = "%s.pid" % server_name
        ensure_dependency_resolvers_conf_configured(
            ctx, kwds, os.path.join(config_directory, "resolvers_conf.xml"))
        _handle_job_config_file(config_directory, server_name, kwds)
        _handle_job_metrics(config_directory, kwds)
        file_path = kwds.get("file_path") or config_join("files")
        _ensure_directory(file_path)

        tool_dependency_dir = kwds.get("tool_dependency_dir") or config_join(
            "deps")
        _ensure_directory(tool_dependency_dir)

        shed_tool_conf = kwds.get("shed_tool_conf") or config_join(
            "shed_tools_conf.xml")
        all_tool_paths = _all_tool_paths(runnables, **kwds)
        empty_tool_conf = config_join("empty_tool_conf.xml")

        tool_conf = config_join("tool_conf.xml")

        shed_data_manager_config_file = config_join(
            "shed_data_manager_conf.xml")

        shed_tool_path = kwds.get("shed_tool_path") or config_join(
            "shed_tools")
        _ensure_directory(shed_tool_path)

        sheds_config_path = _configure_sheds_config_file(
            ctx, config_directory, **kwds)

        database_location = config_join("galaxy.sqlite")
        master_api_key = _get_master_api_key(kwds)
        dependency_dir = os.path.join(config_directory, "deps")
        preseeded_database = attempt_database_preseed(ctx, galaxy_root,
                                                      database_location,
                                                      **kwds)
        _ensure_directory(shed_tool_path)
        port = _get_port(kwds)
        template_args = dict(
            port=port,
            host=kwds.get("host", "127.0.0.1"),
            server_name=server_name,
            temp_directory=config_directory,
            shed_tool_path=shed_tool_path,
            database_location=database_location,
            tool_conf=tool_conf,
            debug=kwds.get("debug", "true"),
            id_secret=kwds.get("id_secret", "test_secret"),
            log_level="DEBUG" if ctx.verbose else "INFO",
        )
        tool_config_file = "%s,%s" % (tool_conf, shed_tool_conf)
        # Setup both galaxy_email and older test user [email protected]
        # as admins for command_line, etc...
        properties = _shared_galaxy_properties(config_directory,
                                               kwds,
                                               for_tests=for_tests)
        properties.update(
            dict(
                server_name="main",
                ftp_upload_dir_template="${ftp_upload_dir}",
                ftp_upload_purge="False",
                ftp_upload_dir=test_data_dir or os.path.abspath('.'),
                ftp_upload_site="Test Data",
                check_upload_content="False",
                tool_dependency_dir=dependency_dir,
                file_path=file_path,
                new_file_path="${temp_directory}/tmp",
                tool_config_file=tool_config_file,
                tool_sheds_config_file=sheds_config_path,
                manage_dependency_relationships="False",
                job_working_directory="${temp_directory}/job_working_directory",
                template_cache_path="${temp_directory}/compiled_templates",
                citation_cache_type="file",
                citation_cache_data_dir="${temp_directory}/citations/data",
                citation_cache_lock_dir="${temp_directory}/citations/lock",
                database_auto_migrate="True",
                enable_beta_tool_formats="True",
                id_secret="${id_secret}",
                log_level="${log_level}",
                debug="${debug}",
                watch_tools="auto",
                default_job_shell="/bin/bash",  # For conda dependency resolution
                tool_data_table_config_path=tool_data_table,
                integrated_tool_panel_config=(
                    "${temp_directory}/"
                    "integrated_tool_panel_conf.xml"),
                migrated_tools_config=empty_tool_conf,
                test_data_dir=test_data_dir,  # TODO: make gx respect this
                shed_data_manager_config_file=shed_data_manager_config_file,
            ))
        _handle_container_resolution(ctx, kwds, properties)
        write_file(config_join("logging.ini"),
                   _sub(LOGGING_TEMPLATE, template_args))
        if not for_tests:
            properties["database_connection"] = _database_connection(
                database_location, **kwds)

        _handle_kwd_overrides(properties, kwds)

        # TODO: consider following property
        # watch_tool = False
        # datatypes_config_file = config/datatypes_conf.xml
        # welcome_url = /static/welcome.html
        # logo_url = /
        # sanitize_all_html = True
        # serve_xss_vulnerable_mimetypes = False
        # track_jobs_in_database = None
        # outputs_to_working_directory = False
        # retry_job_output_collection = 0

        env = _build_env_for_galaxy(properties, template_args)
        env.update(install_env)
        _build_test_env(properties, env)
        env['GALAXY_TEST_SHED_TOOL_CONF'] = shed_tool_conf

        # No need to download twice - would GALAXY_TEST_DATABASE_CONNECTION
        # work?
        if preseeded_database:
            env["GALAXY_TEST_DB_TEMPLATE"] = os.path.abspath(database_location)
        env["GALAXY_TEST_UPLOAD_ASYNC"] = "false"
        env["GALAXY_TEST_LOGGING_CONFIG"] = config_join("logging.ini")
        env["GALAXY_DEVELOPMENT_ENVIRONMENT"] = "1"
        # Following are needed in 18.01 to prevent Galaxy from changing log and pid.
        # https://github.com/galaxyproject/planemo/issues/788
        env["GALAXY_LOG"] = log_file
        env["GALAXY_PID"] = pid_file
        web_config = _sub(WEB_SERVER_CONFIG_TEMPLATE, template_args)
        write_file(config_join("galaxy.ini"), web_config)
        _write_tool_conf(ctx, all_tool_paths, tool_conf)
        write_file(empty_tool_conf, EMPTY_TOOL_CONF_TEMPLATE)

        shed_tool_conf_contents = _sub(SHED_TOOL_CONF_TEMPLATE, template_args)
        # Write a new shed_tool_conf.xml if needed.
        write_file(shed_tool_conf, shed_tool_conf_contents, force=False)

        write_file(shed_data_manager_config_file,
                   SHED_DATA_MANAGER_CONF_TEMPLATE)

        yield LocalGalaxyConfig(
            ctx,
            config_directory,
            env,
            test_data_dir,
            port,
            server_name,
            master_api_key,
            runnables,
            galaxy_root,
            kwds,
        )
コード例 #35
0
ファイル: galaxy_config.py プロジェクト: chambm/planemo
def galaxy_config(ctx, tool_paths, for_tests=False, **kwds):
    test_data_dir = _find_test_data(tool_paths, **kwds)
    tool_data_table = _find_tool_data_table(tool_paths,
                                            test_data_dir=test_data_dir,
                                            **kwds)
    galaxy_root = _check_galaxy(ctx, **kwds)
    install_galaxy = galaxy_root is None
    config_directory = kwds.get("config_directory", None)

    def config_join(*args):
        return os.path.join(config_directory, *args)

    created_config_directory = False
    if not config_directory:
        created_config_directory = True
        config_directory = mkdtemp()
    try:
        latest_galaxy = False
        if install_galaxy:
            _install_galaxy(ctx, config_directory, kwds)
            latest_galaxy = True
            galaxy_root = config_join("galaxy-dev")

        _handle_dependency_resolution(config_directory, kwds)
        _handle_job_metrics(config_directory, kwds)
        tool_definition = _tool_conf_entry_for(tool_paths)
        empty_tool_conf = config_join("empty_tool_conf.xml")
        shed_tool_conf = _shed_tool_conf(install_galaxy, config_directory)
        tool_conf = config_join("tool_conf.xml")
        database_location = config_join("galaxy.sqlite")
        shed_tools_path = config_join("shed_tools")
        sheds_config_path = _configure_sheds_config_file(
            config_directory, **kwds)
        preseeded_database = True
        master_api_key = kwds.get("master_api_key", "test_key")
        dependency_dir = os.path.join(config_directory, "deps")

        try:
            _download_database_template(galaxy_root,
                                        database_location,
                                        latest=latest_galaxy)
        except Exception as e:
            print(e)
            # No network access - just roll forward from null.
            preseeded_database = False

        os.makedirs(shed_tools_path)
        server_name = "planemo%d" % random.randint(0, 100000)
        port = kwds.get("port", 9090)
        template_args = dict(
            port=port,
            host="127.0.0.1",
            server_name=server_name,
            temp_directory=config_directory,
            shed_tools_path=shed_tools_path,
            database_location=database_location,
            tool_definition=tool_definition,
            tool_conf=tool_conf,
            debug=kwds.get("debug", "true"),
            master_api_key=master_api_key,
            id_secret=kwds.get("id_secret", "test_secret"),
            log_level=kwds.get("log_level", "DEBUG"),
        )
        tool_config_file = "%s,%s" % (tool_conf, shed_tool_conf)
        properties = dict(
            tool_dependency_dir=dependency_dir,
            file_path="${temp_directory}/files",
            new_file_path="${temp_directory}/tmp",
            tool_config_file=tool_config_file,
            tool_sheds_config_file=sheds_config_path,
            check_migrate_tools="False",
            manage_dependency_relationships="False",
            job_working_directory="${temp_directory}/job_working_directory",
            template_cache_path="${temp_directory}/compiled_templates",
            citation_cache_type="file",
            citation_cache_data_dir="${temp_directory}/citations/data",
            citation_cache_lock_dir="${temp_directory}/citations/lock",
            collect_outputs_from="job_working_directory",
            database_auto_migrate="True",
            cleanup_job="never",
            master_api_key="${master_api_key}",
            id_secret="${id_secret}",
            log_level="${log_level}",
            debug="${debug}",
            watch_tools="auto",
            tool_data_table_config_path=tool_data_table,
            integrated_tool_panel_config=("${temp_directory}/"
                                          "integrated_tool_panel_conf.xml"),
            # Use in-memory database for kombu to avoid database contention
            # during tests.
            amqp_internal_connection="sqlalchemy+sqlite://",
            migrated_tools_config=empty_tool_conf,
            test_data_dir=test_data_dir,  # TODO: make gx respect this
        )
        if not for_tests:
            properties["database_connection"] = \
                "sqlite:///${database_location}?isolation_level=IMMEDIATE"

        _handle_kwd_overrides(properties, kwds)

        # TODO: consider following property
        # watch_tool = False
        # datatypes_config_file = config/datatypes_conf.xml
        # welcome_url = /static/welcome.html
        # logo_url = /
        # sanitize_all_html = True
        # serve_xss_vulnerable_mimetypes = False
        # track_jobs_in_database = None
        # outputs_to_working_directory = False
        # retry_job_output_collection = 0

        env = _build_env_for_galaxy(properties, template_args)
        if install_galaxy:
            _build_eggs_cache(ctx, env, kwds)
        _build_test_env(properties, env)
        env['GALAXY_TEST_SHED_TOOL_CONF'] = shed_tool_conf

        # No need to download twice - would GALAXY_TEST_DATABASE_CONNECTION
        # work?
        if preseeded_database:
            env["GALAXY_TEST_DB_TEMPLATE"] = os.path.abspath(database_location)
        env["GALAXY_TEST_UPLOAD_ASYNC"] = "false"
        env["GALAXY_DEVELOPMENT_ENVIRONMENT"] = "1"
        web_config = _sub(WEB_SERVER_CONFIG_TEMPLATE, template_args)
        write_file(config_join("galaxy.ini"), web_config)
        tool_conf_contents = _sub(TOOL_CONF_TEMPLATE, template_args)
        write_file(tool_conf, tool_conf_contents)
        write_file(empty_tool_conf, EMPTY_TOOL_CONF_TEMPLATE)

        shed_tool_conf_contents = _sub(SHED_TOOL_CONF_TEMPLATE, template_args)
        write_file(shed_tool_conf, shed_tool_conf_contents)

        yield GalaxyConfig(
            galaxy_root,
            config_directory,
            env,
            test_data_dir,
            port,
            server_name,
            master_api_key,
        )
    finally:
        cleanup = not kwds.get("no_cleanup", False)
        if created_config_directory and cleanup:
            shutil.rmtree(config_directory)