Exemplo n.º 1
0
 def on_or_after_5_0_0_beta1(self, version):
     major, minor, patch, suffix = versions.components(version)
     if major < 5:
         return False
     elif major == 5 and minor == 0 and patch == 0 and suffix and suffix.startswith("alpha"):
         return False
     return True
Exemplo n.º 2
0
    def _provisioner_variables(self):
        plugin_variables = {}
        mandatory_plugins = []
        for installer in self.plugin_installers:
            # For Elasticsearch < 6.3 more specific plugin names are required for mandatory plugin check
            # Details in: https://github.com/elastic/elasticsearch/pull/28710
            # TODO: Remove this section with Elasticsearch <6.3 becomes EOL.
            try:
                major, minor, _, _ = versions.components(
                    self.distribution_version)
                if (major == 6 and minor < 3) or major < 6:
                    mandatory_plugins.append(installer.sub_plugin_name)
                else:
                    mandatory_plugins.append(installer.plugin_name)
            except (TypeError, exceptions.InvalidSyntax):
                mandatory_plugins.append(installer.plugin_name)
            plugin_variables.update(installer.variables)

        cluster_settings = {}
        # Merge cluster config from the track. These may not be dynamically updateable so we need to define them in the config file.
        cluster_settings.update(self._cluster_settings)
        if mandatory_plugins:
            # as a safety measure, prevent the cluster to startup if something went wrong during plugin installation which
            # we did not detect already here. This ensures we fail fast.
            #
            # https://www.elastic.co/guide/en/elasticsearch/plugins/current/_plugins_directory.html#_mandatory_plugins
            cluster_settings["plugin.mandatory"] = mandatory_plugins

        provisioner_vars = {}
        provisioner_vars.update(self.es_installer.variables)
        provisioner_vars.update(plugin_variables)
        provisioner_vars["cluster_settings"] = cluster_settings

        return provisioner_vars
Exemplo n.º 3
0
    def download_url(self, version):
        major_version = int(versions.components(version)["major"])
        if major_version > 1:
            download_url = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/%s/" \
                           "elasticsearch-%s.tar.gz" % (version, version)
        else:
            download_url = "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-%s.tar.gz" % version

        return download_url
Exemplo n.º 4
0
    def download_url(self, version):
        major_version = int(versions.components(version)["major"])
        if major_version > 1:
            download_url = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/%s/" \
                           "elasticsearch-%s.tar.gz" % (version, version)
        else:
            download_url = "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-%s.tar.gz" % version

        return download_url
Exemplo n.º 5
0
 def download_url(self, version):
     major_version, _, _, _ = versions.components(version)
     if major_version > 1 and not self.on_or_after_5_0_0_beta1(version):
         download_url = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/%s/" \
                        "elasticsearch-%s.tar.gz" % (version, version)
     elif self.on_or_after_5_0_0_beta1(version):
         download_url = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-%s.tar.gz" % version
     else:
         download_url = "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-%s.tar.gz" % version
     return download_url
Exemplo n.º 6
0
 def download_url(self, version):
     major_version, _, _, _ = versions.components(version)
     if major_version > 1 and not self.on_or_after_5_0_0_beta1(version):
         download_url = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/%s/" \
                        "elasticsearch-%s.tar.gz" % (version, version)
     elif self.on_or_after_5_0_0_beta1(version):
         download_url = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-%s.tar.gz" % version
     else:
         download_url = "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-%s.tar.gz" % version
     return download_url
Exemplo n.º 7
0
 def number_of_nodes(self, car):
     distribution_version = self._config.opts("source", "distribution.version", mandatory=False)
     configure = False
     if versions.is_version_identifier(distribution_version):
         major, _, _, _ = versions.components(distribution_version)
         if major >= 2:
             configure = True
     else:
         # we're very likely benchmarking from sources which is ES 5+
         configure = True
     return "\nnode.max_local_storage_nodes: %d" % car.nodes if configure else ""
Exemplo n.º 8
0
 def number_of_nodes(self, car):
     distribution_version = self._config.opts("source", "distribution.version", mandatory=False)
     configure = False
     if versions.is_version_identifier(distribution_version):
         major, _, _, _ = versions.components(distribution_version)
         if major >= 2:
             configure = True
     else:
         # we're very likely benchmarking from sources which is ES 5+
         configure = True
     return "\nnode.max_local_storage_nodes: %d" % car.nodes if configure else ""
Exemplo n.º 9
0
def _do_wait(es, es_version, expected_cluster_status):
    reached_cluster_status = None
    relocating_shards = -1
    major, minor, patch, suffix = versions.components(es_version)
    if major < 5:
        use_wait_for_relocating_shards = True
    elif major == 5 and minor == 0 and patch == 0 and suffix and suffix.startswith(
            "alpha"):
        use_wait_for_relocating_shards = True
    else:
        use_wait_for_relocating_shards = False

    for attempt in range(10):
        try:
            if use_wait_for_relocating_shards:
                result = es.cluster.health(
                    wait_for_status=expected_cluster_status,
                    wait_for_relocating_shards=0,
                    timeout="3s")
            else:
                result = es.cluster.health(
                    wait_for_status=expected_cluster_status,
                    timeout="3s",
                    params={"wait_for_no_relocating_shards": True})
        except (socket.timeout, elasticsearch.exceptions.ConnectionError):
            pass
        except elasticsearch.exceptions.TransportError as e:
            if e.status_code == 408:
                logger.info(
                    "Timed out waiting for cluster health status. Retrying shortly..."
                )
                time.sleep(0.5)
            else:
                raise e
        else:
            reached_cluster_status = result["status"]
            relocating_shards = result["relocating_shards"]
            logger.info("GOT: %s" % str(result))
            logger.info("ALLOC:\n%s" % es.cat.allocation(v=True))
            logger.info("RECOVERY:\n%s" % es.cat.recovery(v=True))
            logger.info("SHARDS:\n%s" % es.cat.shards(v=True))
            if reached_cluster_status == expected_cluster_status and relocating_shards == 0:
                return reached_cluster_status, relocating_shards
            else:
                time.sleep(0.5)
    if reached_cluster_status != expected_cluster_status:
        msg = "Cluster did not reach status [%s]. Last reached status: [%s]" % (
            expected_cluster_status, reached_cluster_status)
    else:
        msg = "Cluster reached expected status [%s] but there were [%d] relocating shards and we require zero relocating shards " \
              "(Use the /_cat/shards API to check which shards are relocating.)" % (reached_cluster_status, relocating_shards)
    logger.error(msg)
    raise exceptions.RallyAssertionError(msg)
Exemplo n.º 10
0
def _do_wait(es, es_version, expected_cluster_status):
    reached_cluster_status = None
    relocating_shards = -1
    major, minor, patch, suffix = versions.components(es_version)
    if major < 5:
        use_wait_for_relocating_shards = True
    elif major == 5 and minor == 0 and patch == 0 and suffix and suffix.startswith("alpha"):
        use_wait_for_relocating_shards = True
    else:
        use_wait_for_relocating_shards = False

    for attempt in range(10):
        try:
            if use_wait_for_relocating_shards:
                result = es.cluster.health(wait_for_status=expected_cluster_status, wait_for_relocating_shards=0, timeout="3s")
            else:
                result = es.cluster.health(wait_for_status=expected_cluster_status, timeout="3s",
                                           params={"wait_for_no_relocating_shards": True})
        except (socket.timeout, elasticsearch.exceptions.ConnectionError):
            pass
        except elasticsearch.exceptions.TransportError as e:
            if e.status_code == 408:
                logger.info("Timed out waiting for cluster health status. Retrying shortly...")
                time.sleep(0.5)
            else:
                raise e
        else:
            reached_cluster_status = result["status"]
            relocating_shards = result["relocating_shards"]
            logger.info("GOT: %s" % str(result))
            logger.info("ALLOC:\n%s" % es.cat.allocation(v=True))
            logger.info("RECOVERY:\n%s" % es.cat.recovery(v=True))
            logger.info("SHARDS:\n%s" % es.cat.shards(v=True))
            if reached_cluster_status == expected_cluster_status and relocating_shards == 0:
                return reached_cluster_status, relocating_shards
            else:
                time.sleep(0.5)
    if reached_cluster_status != expected_cluster_status:
        msg = "Cluster did not reach status [%s]. Last reached status: [%s]" % (expected_cluster_status, reached_cluster_status)
    else:
        msg = "Cluster reached expected status [%s] but there were [%d] relocating shards and we require zero relocating shards " \
              "(Use the /_cat/shards API to check which shards are relocating.)" % (reached_cluster_status, relocating_shards)
    logger.error(msg)
    raise exceptions.RallyAssertionError(msg)
Exemplo n.º 11
0
    def _es_log_config(self):
        logging_yml_path = "%s/config/logging.yml" % self.binary_path
        log4j2_properties_path = "%s/config/log4j2.properties" % self.binary_path

        if os.path.isfile(logging_yml_path):
            return "logging.yml", logging_yml_path
        elif os.path.isfile(log4j2_properties_path):
            distribution_version = self._config.opts("mechanic",
                                                     "distribution.version",
                                                     mandatory=False)
            if versions.is_version_identifier(distribution_version):
                major, _, _, _ = versions.components(distribution_version)
                if major == 5:
                    return "log4j2.properties.5", log4j2_properties_path
            else:
                return "log4j2.properties", log4j2_properties_path
        else:
            raise exceptions.SystemSetupError(
                "Unrecognized Elasticsearch log config file format")
Exemplo n.º 12
0
 def test_components_ignores_invalid_versions(self):
     with self.assertRaises(exceptions.InvalidSyntax) as ctx:
         versions.components("5.0.0a")
     self.assertEqual(
         "version string '5.0.0a' does not conform to pattern '^(\d+)\.(\d+)\.(\d+)(?:-(.+))?$'",
         ctx.exception.args[0])
Exemplo n.º 13
0
 def test_finds_components_for_valid_version(self):
     self.assertEqual({"major": "5", "minor": "0", "patch": "3"}, versions.components("5.0.3"))
     self.assertEqual({"major": "5", "minor": "0", "patch": "3", "suffix": "SNAPSHOT"}, versions.components("5.0.3-SNAPSHOT"))
Exemplo n.º 14
0
 def test_components_ignores_invalid_versions(self):
     with self.assertRaises(exceptions.InvalidSyntax) as ctx:
         versions.components("5.0.0a")
     self.assertEqual("version string '5.0.0a' does not conform to pattern '^(\d+)\.(\d+)\.(\d+)(?:-(.+))?$'", ctx.exception.args[0])
Exemplo n.º 15
0
    def test_finds_components_for_valid_version(self):
        self.assertEqual((5, 0, 3, None), versions.components("5.0.3"))
        self.assertEqual((5, 0, 3, "SNAPSHOT"), versions.components("5.0.3-SNAPSHOT"))

        self.assertEqual((25, None, None, None), versions.components("25", strict=False))
        self.assertEqual((5, 1, None, None), versions.components("5.1", strict=False))
Exemplo n.º 16
0
 def test_components_ignores_invalid_versions(self):
     with pytest.raises(
         exceptions.InvalidSyntax,
         match=re.escape(r"version string '5.0.0a' does not conform to pattern " r"'^(\d+)\.(\d+)\.(\d+)(?:-(.+))?$'"),
     ):
         versions.components("5.0.0a")
Exemplo n.º 17
0
    def test_finds_components_for_valid_version(self):
        assert versions.components("5.0.3") == (5, 0, 3, None)
        assert versions.components("7.12.1-SNAPSHOT") == (7, 12, 1, "SNAPSHOT")

        assert versions.components("25", strict=False) == (25, None, None, None)
        assert versions.components("5.1", strict=False) == (5, 1, None, None)
Exemplo n.º 18
0
    def test_finds_components_for_valid_version(self):
        self.assertEqual((5, 0, 3, None), versions.components("5.0.3"))
        self.assertEqual((5, 0, 3, "SNAPSHOT"), versions.components("5.0.3-SNAPSHOT"))

        self.assertEqual((25, None, None, None), versions.components("25", strict=False))
        self.assertEqual((5, 1, None, None), versions.components("5.1", strict=False))
Exemplo n.º 19
0
def _do_wait(es, expected_cluster_status, sleep=time.sleep):
    import elasticsearch
    from enum import Enum
    from functools import total_ordering

    @total_ordering
    class ClusterHealthStatus(Enum):
        UNKNOWN = 0
        RED = 1
        YELLOW = 2
        GREEN = 3

        def __lt__(self, other):
            if self.__class__ is other.__class__:
                return self.value < other.value
            return NotImplemented

    def status(v):
        try:
            return ClusterHealthStatus[v.upper()]
        except (KeyError, AttributeError):
            return ClusterHealthStatus.UNKNOWN

    reached_cluster_status = None
    relocating_shards = -1
    major, minor, patch, suffix = versions.components(
        es.info()["version"]["number"])
    if major < 5:
        use_wait_for_relocating_shards = True
    elif major == 5 and minor == 0 and patch == 0 and suffix and suffix.startswith(
            "alpha"):
        use_wait_for_relocating_shards = True
    else:
        use_wait_for_relocating_shards = False

    max_attempts = 10
    for attempt in range(max_attempts):
        try:
            # Is this the last attempt? Then just retrieve the status
            if attempt + 1 == max_attempts:
                result = es.cluster.health()
            elif use_wait_for_relocating_shards:
                result = es.cluster.health(
                    wait_for_status=expected_cluster_status,
                    timeout="3s",
                    params={"wait_for_relocating_shards": 0})
            else:
                result = es.cluster.health(
                    wait_for_status=expected_cluster_status,
                    timeout="3s",
                    wait_for_no_relocating_shards=True)
        except (socket.timeout, elasticsearch.exceptions.ConnectionError):
            pass
        except elasticsearch.exceptions.TransportError as e:
            if e.status_code == 408:
                logger.info(
                    "Timed out waiting for cluster health status. Retrying shortly..."
                )
                sleep(0.5)
            else:
                raise e
        else:
            reached_cluster_status = result["status"]
            relocating_shards = result["relocating_shards"]
            logger.info("GOT: %s" % str(result))
            logger.info("ALLOC:\n%s" % es.cat.allocation(v=True))
            logger.info("RECOVERY:\n%s" % es.cat.recovery(v=True))
            logger.info("SHARDS:\n%s" % es.cat.shards(v=True))
            if status(reached_cluster_status) >= status(
                    expected_cluster_status) and relocating_shards == 0:
                return reached_cluster_status, relocating_shards
            else:
                sleep(0.5)
    if status(reached_cluster_status) < status(expected_cluster_status):
        msg = "Cluster did not reach status [%s]. Last reached status: [%s]" % (
            expected_cluster_status, reached_cluster_status)
    else:
        msg = "Cluster reached status [%s] which is equal or better than the expected status [%s] but there were [%d] relocating shards " \
              "and we require zero relocating shards (Use the /_cat/shards API to check which shards are relocating.)" % \
              (reached_cluster_status, expected_cluster_status, relocating_shards)
    logger.error(msg)
    raise exceptions.RallyAssertionError(msg)