def test_endpoint_strategy_path(self, monkeypatch, engine):
monkeypatch.setattr(config, "OPENSEARCH_ENDPOINT_STRATEGY", "path")
engine_type = engine[0]
engine_path_prefix = engine[1]
endpoint = build_cluster_endpoint(DomainKey("my-domain", "us-east-1",
TEST_AWS_ACCOUNT_ID),
engine_type=engine_type)
assert endpoint == f"localhost:4566/{engine_path_prefix}/us-east-1/my-domain"
endpoint = build_cluster_endpoint(DomainKey("my-domain-1",
"eu-central-1",
TEST_AWS_ACCOUNT_ID),
engine_type=engine_type)
assert endpoint == f"localhost:4566/{engine_path_prefix}/eu-central-1/my-domain-1"
def test_from_arn(self):
domain_key = DomainKey.from_arn(
"arn:aws:es:us-east-1:012345678901:domain/my-es-domain")
assert domain_key.domain_name == "my-es-domain"
assert domain_key.region == "us-east-1"
assert domain_key.account == "012345678901"
def create_domain(
self,
context: RequestContext,
domain_name: DomainName,
engine_version: VersionString = None,
cluster_config: ClusterConfig = None,
ebs_options: EBSOptions = None,
access_policies: PolicyDocument = None,
snapshot_options: SnapshotOptions = None,
vpc_options: VPCOptions = None,
cognito_options: CognitoOptions = None,
encryption_at_rest_options: EncryptionAtRestOptions = None,
node_to_node_encryption_options: NodeToNodeEncryptionOptions = None,
advanced_options: AdvancedOptions = None,
log_publishing_options: LogPublishingOptions = None,
domain_endpoint_options: DomainEndpointOptions = None,
advanced_security_options: AdvancedSecurityOptionsInput = None,
tag_list: TagList = None,
auto_tune_options: AutoTuneOptionsInput = None,
) -> CreateDomainResponse:
region = OpenSearchServiceBackend.get()
if not is_valid_domain_name(domain_name):
# TODO: this should use the server-side validation framework at some point.
raise ValidationException(
"Member must satisfy regular expression pattern: [a-z][a-z0-9\\-]+"
)
with _domain_mutex:
if domain_name in region.opensearch_domains:
raise ResourceAlreadyExistsException(
f"domain {domain_name} already exists in region {region.name}"
)
domain_key = DomainKey(
domain_name=domain_name,
region=context.region,
account=context.account_id,
)
# "create" domain data
region.opensearch_domains[domain_name] = get_domain_status(domain_key)
# lazy-init the cluster (sets the Endpoint and Processing flag of the domain status)
# TODO handle additional parameters (cluster config,...)
create_cluster(domain_key, engine_version, domain_endpoint_options)
# set the tags
self.add_tags(context, domain_key.arn, tag_list)
# get the (updated) status
status = get_domain_status(domain_key)
# record event
event_publisher.fire_event(
event_publisher.EVENT_OPENSEARCH_CREATE_DOMAIN,
payload={"n": event_publisher.get_hash(domain_name)},
)
return CreateDomainResponse(DomainStatus=status)
def test_endpoint_strategy_port(self, monkeypatch):
monkeypatch.setattr(config, "OPENSEARCH_ENDPOINT_STRATEGY", "port")
endpoint = build_cluster_endpoint(
DomainKey("my-domain", "us-east-1", TEST_AWS_ACCOUNT_ID))
parts = endpoint.split(":")
assert parts[0] == "localhost"
assert int(parts[1]) in range(config.EXTERNAL_SERVICE_PORTS_START,
config.EXTERNAL_SERVICE_PORTS_END)
def test_arn(self):
domain_key = DomainKey(
domain_name="my-es-domain",
region="us-east-1",
account="012345678901",
)
assert domain_key.arn == "arn:aws:es:us-east-1:012345678901:domain/my-es-domain"
def test_multiplexing_cluster(self, monkeypatch):
monkeypatch.setattr(config, "OPENSEARCH_ENDPOINT_STRATEGY", "domain")
monkeypatch.setattr(config, "OPENSEARCH_MULTI_CLUSTER", False)
manager = MultiplexingClusterManager()
# create two opensearch domains
domain_key_0 = DomainKey(domain_name=f"domain-{short_uid()}",
region="us-east-1",
account=TEST_AWS_ACCOUNT_ID)
domain_key_1 = DomainKey(domain_name=f"domain-{short_uid()}",
region="us-east-1",
account=TEST_AWS_ACCOUNT_ID)
cluster_0 = manager.create(domain_key_0.arn,
OPENSEARCH_DEFAULT_VERSION)
cluster_1 = manager.create(domain_key_1.arn,
OPENSEARCH_DEFAULT_VERSION)
try:
# spawn the two clusters
assert cluster_0.wait_is_up(240)
assert cluster_1.wait_is_up(240)
retry(lambda: try_cluster_health(cluster_0.url),
retries=12,
sleep=10)
retry(lambda: try_cluster_health(cluster_1.url),
retries=12,
sleep=10)
# create an index in cluster_0, wait for it to appear, make sure it's in cluster_1, too
index_url_0 = cluster_0.url + "/my-index?pretty"
index_url_1 = cluster_1.url + "/my-index?pretty"
response = requests.put(index_url_0)
assert response.ok, f"failed to put index into cluster {cluster_0.url}: {response.text}"
assert poll_condition(lambda: requests.head(index_url_0).ok,
timeout=10), "gave up waiting for index"
assert requests.head(
index_url_1).ok, "index should appear in second cluster"
finally:
call_safe(cluster_0.shutdown)
call_safe(cluster_1.shutdown)
def _ensure_domain_exists(arn: ARN) -> None:
"""
Checks if the domain for the given ARN exists. Otherwise, a ValidationException is raised.
:param arn: ARN string to lookup the domain for
:return: None if the domain exists, otherwise raises an exception
:raises: ValidationException if the domain for the given ARN cannot be found
"""
domain_key = DomainKey.from_arn(arn)
region = OpenSearchServiceBackend.get(domain_key.region)
domain_status = region.opensearch_domains.get(domain_key.domain_name)
if domain_status is None:
raise ValidationException("Invalid ARN. Domain not found.")
def test_endpoint_strategy_port_singleton_cluster(self, monkeypatch):
monkeypatch.setattr(config, "OPENSEARCH_ENDPOINT_STRATEGY", "port")
monkeypatch.setattr(config, "OPENSEARCH_MULTI_CLUSTER", False)
manager = SingletonClusterManager()
# create two opensearch domains
domain_key_0 = DomainKey(domain_name=f"domain-{short_uid()}",
region="us-east-1",
account=TEST_AWS_ACCOUNT_ID)
domain_key_1 = DomainKey(domain_name=f"domain-{short_uid()}",
region="us-east-1",
account=TEST_AWS_ACCOUNT_ID)
cluster_0 = manager.create(domain_key_0.arn,
OPENSEARCH_DEFAULT_VERSION)
cluster_1 = manager.create(domain_key_1.arn,
OPENSEARCH_DEFAULT_VERSION)
# check if the first port url matches the port range
parts = cluster_0.url.split(":")
assert parts[0] == "http"
assert parts[1] == "//localhost"
assert int(parts[2]) in range(config.EXTERNAL_SERVICE_PORTS_START,
config.EXTERNAL_SERVICE_PORTS_END)
# check if the second url matches the first one
assert cluster_0.url == cluster_1.url
try:
# wait for the two clusters
assert cluster_0.wait_is_up(240)
# make sure cluster_0 (which is equal to cluster_1) is reachable
retry(lambda: try_cluster_health(cluster_0.url),
retries=3,
sleep=5)
finally:
call_safe(cluster_0.shutdown)
call_safe(cluster_1.shutdown)
def describe_domain_config(
self, context: RequestContext, domain_name: DomainName
) -> DescribeDomainConfigResponse:
domain_key = DomainKey(
domain_name=domain_name,
region=context.region,
account=context.account_id,
)
region = OpenSearchServiceBackend.get(domain_key.region)
with _domain_mutex:
if domain_name not in region.opensearch_domains:
raise ResourceNotFoundException(f"Domain not found: {domain_name}")
domain_config = get_domain_config(domain_key)
return DescribeDomainConfigResponse(DomainConfig=domain_config)
def test_endpoint_strategy_domain(self, monkeypatch, engine):
monkeypatch.setattr(config, "OPENSEARCH_ENDPOINT_STRATEGY", "domain")
engine_type = engine[0]
engine_path_prefix = engine[1]
endpoint = build_cluster_endpoint(
domain_key=DomainKey("my-domain", "us-east-1",
TEST_AWS_ACCOUNT_ID),
engine_type=engine_type,
)
assert (
endpoint ==
f"my-domain.us-east-1.{engine_path_prefix}.localhost.localstack.cloud:4566"
)
endpoint = build_cluster_endpoint(
domain_key=DomainKey("my-domain-1", "eu-central-1",
TEST_AWS_ACCOUNT_ID),
engine_type=engine_type,
)
assert (
endpoint ==
f"my-domain-1.eu-central-1.{engine_path_prefix}.localhost.localstack.cloud:4566"
)
def update_domain_config(
self, context: RequestContext, payload: UpdateDomainConfigRequest
) -> UpdateDomainConfigResponse:
domain_key = DomainKey(
domain_name=payload["DomainName"],
region=context.region,
account=context.account_id,
)
region = OpenSearchServiceBackend.get(domain_key.region)
with _domain_mutex:
domain_status = region.opensearch_domains.get(domain_key.domain_name, None)
if domain_status is None:
raise ResourceNotFoundException(f"Domain not found: {domain_key.domain_name}")
status_update: Dict = _update_domain_config_request_to_status(payload)
domain_status.update(status_update)
return UpdateDomainConfigResponse(DomainConfig=_status_to_config(domain_status))
def delete_domain(self, context: RequestContext,
domain_name: DomainName) -> DeleteDomainResponse:
domain_key = DomainKey(
domain_name=domain_name,
region=context.region,
account=context.account_id,
)
region = OpenSearchServiceBackend.get(domain_key.region)
with _domain_mutex:
if domain_name not in region.opensearch_domains:
raise ResourceNotFoundException(
f"Domain not found: {domain_name}")
status = get_domain_status(domain_key, deleted=True)
_remove_cluster(domain_key)
# record event
event_publisher.fire_event(
event_publisher.EVENT_OPENSEARCH_DELETE_DOMAIN,
payload={"n": event_publisher.get_hash(domain_name)},
)
return DeleteDomainResponse(DomainStatus=status)
def test_custom_backend(self, httpserver, monkeypatch):
monkeypatch.setattr(config, "OPENSEARCH_ENDPOINT_STRATEGY", "domain")
monkeypatch.setattr(config, "OPENSEARCH_CUSTOM_BACKEND", httpserver.url_for("/"))
# create fake elasticsearch cluster
httpserver.expect_request("/").respond_with_json(
{
"name": "om",
"cluster_name": "opensearch",
"cluster_uuid": "gREewvVZR0mIswR-8-6VRQ",
"version": {
"number": "7.10.0",
"build_flavor": "default",
"build_type": "tar",
"build_hash": "51e9d6f22758d0374a0f3f5c6e8f3a7997850f96",
"build_date": "2020-11-09T21:30:33.964949Z",
"build_snapshot": False,
"lucene_version": "8.7.0",
"minimum_wire_compatibility_version": "6.8.0",
"minimum_index_compatibility_version": "6.0.0-beta1",
},
"tagline": "You Know, for Search",
}
)
httpserver.expect_request("/_cluster/health").respond_with_json(
{
"cluster_name": "opensearch",
"status": "green",
"timed_out": False,
"number_of_nodes": 1,
"number_of_data_nodes": 1,
"active_primary_shards": 0,
"active_shards": 0,
"relocating_shards": 0,
"initializing_shards": 0,
"unassigned_shards": 0,
"delayed_unassigned_shards": 0,
"number_of_pending_tasks": 0,
"number_of_in_flight_fetch": 0,
"task_max_waiting_in_queue_millis": 0,
"active_shards_percent_as_number": 100,
}
)
manager = create_cluster_manager()
assert isinstance(manager, CustomBackendManager)
domain_key = DomainKey(
domain_name=f"domain-{short_uid()}", region="us-east-1", account=TEST_AWS_ACCOUNT_ID
)
cluster = manager.create(domain_key.arn, OPENSEARCH_DEFAULT_VERSION)
# check that we're using the domain endpoint strategy
assert f"{domain_key.domain_name}." in cluster.url
try:
assert cluster.wait_is_up(10)
retry(lambda: try_cluster_health(cluster.url), retries=3, sleep=5)
finally:
call_safe(cluster.shutdown)
httpserver.check()
def test_from_arn_wrong_service(self):
with pytest.raises(ValueError):
DomainKey.from_arn("arn:aws:sqs:us-east-1:012345678901:my-queue")