def test_create_and_delete_log_stream(self, logs_client, logs_log_group):
test_name = f"test-log-stream-{short_uid()}"
log_streams_before = logs_client.describe_log_streams(
logGroupName=logs_log_group).get("logStreams", [])
logs_client.create_log_stream(logGroupName=logs_log_group,
logStreamName=test_name)
log_streams_between = logs_client.describe_log_streams(
logGroupName=logs_log_group).get("logStreams", [])
assert poll_condition(
lambda: len(log_streams_between) == len(log_streams_before) + 1,
timeout=5.0,
interval=0.5,
)
logs_client.delete_log_stream(logGroupName=logs_log_group,
logStreamName=test_name)
log_streams_after = logs_client.describe_log_streams(
logGroupName=logs_log_group).get("logStreams", [])
assert poll_condition(
lambda: len(log_streams_between) - 1 == len(log_streams_after),
timeout=5.0,
interval=0.5,
)
assert len(log_streams_after) == len(log_streams_before)
def dynamodb_create_table(dynamodb_client):
tables = []
def factory(**kwargs):
kwargs["client"] = dynamodb_client
if "table_name" not in kwargs:
kwargs["table_name"] = "test-table-%s" % short_uid()
if "partition_key" not in kwargs:
kwargs["partition_key"] = "id"
kwargs["sleep_after"] = 0
tables.append(kwargs["table_name"])
return create_dynamodb_table(**kwargs)
yield factory
# cleanup
for table in tables:
try:
# table has to be in ACTIVE state before deletion
def wait_for_table_created():
return (
dynamodb_client.describe_table(TableName=table)["Table"]["TableStatus"]
== "ACTIVE"
)
poll_condition(wait_for_table_created, timeout=30)
dynamodb_client.delete_table(TableName=table)
except Exception as e:
LOG.debug("error cleaning up table %s: %s", table, e)
def _wait_for_consumer_ready(consumer_arn: str):
def is_consumer_ready():
describe_response = kinesis_client.describe_stream_consumer(
ConsumerARN=consumer_arn)
return describe_response["ConsumerDescription"][
"ConsumerStatus"] == "ACTIVE"
poll_condition(is_consumer_ready)
def _wait_for_stream_ready(stream_name: str):
def is_stream_ready():
describe_stream_response = kinesis_client.describe_stream(StreamName=stream_name)
return describe_stream_response["StreamDescription"]["StreamStatus"] in [
"ACTIVE",
"UPDATING",
]
poll_condition(is_stream_ready)
def test_start_and_stop(self):
proxy_port = get_free_tcp_port()
backend_port = get_free_tcp_port()
server = start_proxy_for_service(
"myservice",
proxy_port,
backend_port,
update_listener=None,
quiet=True,
params={"protocol_version": "HTTP/1.0"},
)
self.assertIsNotNone(server)
try:
self.assertTrue(
poll_condition(lambda: is_port_open(proxy_port), timeout=15),
"gave up waiting for port %d" % proxy_port,
)
finally:
print("stopping proxy server")
server.stop()
print("waiting max 15 seconds for server to terminate")
server.join(timeout=15)
self.assertFalse(is_port_open(proxy_port))
def test_route_through_edge(self):
cluster_id = f"domain-{short_uid()}"
cluster_url = f"http://localhost:{config.EDGE_PORT}/{cluster_id}"
cluster = EdgeProxiedOpensearchCluster(cluster_url)
try:
cluster.start()
assert cluster.wait_is_up(240), "gave up waiting for server"
response = requests.get(cluster_url)
assert response.ok, f"cluster endpoint returned an error: {response.text}"
assert response.json()["version"]["number"] == "1.1.0"
response = requests.get(f"{cluster_url}/_cluster/health")
assert response.ok, f"cluster health endpoint returned an error: {response.text}"
assert response.json()["status"] in [
"red",
"orange",
"yellow",
"green",
], "expected cluster state to be in a valid state"
finally:
cluster.shutdown()
assert poll_condition(
lambda: not cluster.is_up(), timeout=240
), "gave up waiting for cluster to shut down"
def wait_container_is_ready(timeout: Optional[float] = None):
"""Blocks until the localstack main container is running and the ready marker has been printed."""
container_name = config.MAIN_CONTAINER_NAME
def is_container_running():
return DOCKER_CLIENT.is_container_running(container_name)
if not poll_condition(is_container_running, timeout=timeout):
return False
logfile = LocalstackContainer(container_name).logfile
ready = threading.Event()
def set_ready_if_marker_found(_line: str):
if _line == constants.READY_MARKER_OUTPUT:
ready.set()
# start a tail on the logfile
listener = FileListener(logfile, set_ready_if_marker_found)
listener.start()
try:
# but also check the existing log in case the container has been running longer
with open(logfile, "r") as fd:
for line in fd:
if constants.READY_MARKER_OUTPUT == line.strip():
return True
# TODO: calculate remaining timeout
return ready.wait(timeout)
finally:
listener.close()
def test_callback_exception_ignored(self, tail_engine, tmp_path):
lines = []
def callback(line):
if "throw" in line:
raise ValueError("oh noes")
lines.append(line)
file = tmp_path / "log.txt"
file.touch()
fd = open(file, "a")
listener = FileListener(str(file), callback)
listener.use_tail_command = tail_engine != "tailer"
try:
listener.start()
assert listener.started.is_set()
fd.write("hello" + os.linesep)
fd.flush()
fd.write("throw" + os.linesep)
fd.write("pytest" + os.linesep)
fd.flush()
assert poll_condition(
lambda: len(lines) == 2,
timeout=3), ("expected two lines to appear. %s" % lines)
assert lines[0] == "hello"
assert lines[1] == "pytest"
finally:
fd.close()
listener.close()
def test_basic_usage(self, tail_engine, tmp_path):
lines = []
file = tmp_path / "log.txt"
file.touch()
fd = open(file, "a")
listener = FileListener(str(file), lines.append)
listener.use_tail_command = tail_engine != "tailer"
try:
listener.start()
assert listener.started.is_set()
fd.write("hello" + os.linesep)
fd.write("pytest" + os.linesep)
fd.flush()
assert poll_condition(
lambda: len(lines) == 2,
timeout=3), ("expected two lines to appear. %s" % lines)
assert lines[0] == "hello"
assert lines[1] == "pytest"
finally:
listener.close()
try:
fd.write("foobar" + os.linesep)
time.sleep(0.5)
assert len(
lines
) == 2, "expected listener.stop() to stop listening on new "
finally:
fd.close()
def _wait_for_cluster(domain_name: str):
def finished_processing():
status = opensearch_client.describe_domain(
DomainName=domain_name)["DomainStatus"]
return status["Processing"] is False
assert poll_condition(finished_processing, timeout=5 *
60), f"could not start domain: {domain_name}"
def require(self, name: str) -> Service:
"""
High level function that always returns a running service, or raises an error. If the service is in a state
that it could be transitioned into a running state, then invoking this function will attempt that transition,
e.g., by starting the service if it is available.
"""
container = self.get_service_container(name)
if not container:
raise ValueError("no such service %s" % name)
if container.state == ServiceState.STARTING:
if not poll_condition(
lambda: container.state != ServiceState.STARTING,
timeout=30):
raise TimeoutError("gave up waiting for service %s to start" %
name)
if container.state == ServiceState.STOPPING:
if not poll_condition(
lambda: container.state == ServiceState.STOPPED,
timeout=30):
raise TimeoutError("gave up waiting for service %s to stop" %
name)
with container.lock:
if container.state == ServiceState.DISABLED:
raise ServiceDisabled("service %s is disabled" % name)
if container.state == ServiceState.RUNNING:
return container.service
if container.state == ServiceState.ERROR:
# raise any capture error
raise container.errors[-1]
if container.state == ServiceState.AVAILABLE or container.state == ServiceState.STOPPED:
if container.start():
return container.service
else:
raise container.errors[-1]
raise ServiceStateException(
"service %s is not ready (%s) and could not be started" %
(name, container.state))
def wait_is_up(self, timeout: float = None) -> bool:
"""
Waits until the server is started and is_up returns true.
:param timeout: the time in seconds to wait before returning false. If timeout is None, then wait indefinitely.
:returns: true if the server is up, false if not or the timeout was reached while waiting.
"""
# first wait until the started event was called
self._started.wait(timeout=timeout)
# then poll the health check
return poll_condition(self.is_up, timeout=timeout)
def container_client():
client = docker_utils.SdkDockerClient()
yield client
try:
client.stop_container(config.MAIN_CONTAINER_NAME)
except Exception:
pass
# wait until container has been removed
assert poll_condition(
lambda: not container_exists(client, config.MAIN_CONTAINER_NAME),
timeout=20)
def test_create_and_delete_log_group(self, logs_client):
test_name = f"test-log-group-{short_uid()}"
log_groups_before = logs_client.describe_log_groups(
logGroupNamePrefix="test-log-group-").get("logGroups", [])
logs_client.create_log_group(logGroupName=test_name)
log_groups_between = logs_client.describe_log_groups(
logGroupNamePrefix="test-log-group-").get("logGroups", [])
assert poll_condition(
lambda: len(log_groups_between) == len(log_groups_before) + 1,
timeout=5.0,
interval=0.5)
logs_client.delete_log_group(logGroupName=test_name)
log_groups_after = logs_client.describe_log_groups(
logGroupNamePrefix="test-log-group-").get("logGroups", [])
assert poll_condition(
lambda: len(log_groups_after) == len(log_groups_between) - 1,
timeout=5.0,
interval=0.5)
assert len(log_groups_after) == len(log_groups_before)
def _run_cluster_startup_monitor(cluster):
region = ElasticsearchServiceBackend.get()
LOG.debug("running cluster startup monitor for cluster %s", cluster)
# wait until the cluster is started, or the timeout is reached
status = poll_condition(cluster.is_up, timeout=CLUSTER_STARTUP_TIMEOUT, interval=5)
LOG.debug("cluster state polling returned! status = %s", status)
with _domain_mutex:
LOG.debug("iterating over cluster domains %s", region.es_clusters.keys())
for domain, domain_cluster in region.es_clusters.items():
LOG.debug("checking cluster for domain %s", domain)
if cluster is domain_cluster:
if domain in region.es_domains:
region.es_domains[domain]["Created"] = status
def proxy_server(proxy_listener, host="127.0.0.1", port=None) -> str:
"""
Create a temporary proxy server on a random port (or the specified port) with the given proxy listener
for the duration of the context manager.
"""
from localstack.services.generic_proxy import start_proxy_server
host = host
port = port or get_free_tcp_port()
thread = start_proxy_server(port, bind_address=host, update_listener=proxy_listener)
url = f"http://{host}:{port}"
assert poll_condition(
lambda: is_port_open(port), timeout=5
), f"server on port {port} did not start"
yield url
thread.stop()
def _run_cluster_startup_monitor(cluster):
LOG.debug("running cluster startup monitor for cluster %s", cluster)
# wait until the cluster is started, or the timeout is reached
status = poll_condition(cluster.is_up,
timeout=CLUSTER_STARTUP_TIMEOUT,
interval=5)
LOG.debug("cluster state polling returned! status = %s", status)
with _domain_mutex:
LOG.debug("iterating over cluster domains %s", ES_CLUSTERS.keys())
for domain, domain_cluster in ES_CLUSTERS.items():
LOG.debug("checking cluster for domain %s", domain)
if cluster is domain_cluster:
if domain in ES_DOMAINS:
ES_DOMAINS[domain]["Created"] = status
def test_multiplexing_cluster(self, monkeypatch):
monkeypatch.setattr(config, "ES_ENDPOINT_STRATEGY", "domain")
monkeypatch.setattr(config, "ES_MULTI_CLUSTER", False)
manager = MultiplexingClusterManager()
# create two elasticsearch domains
domain0_name = f"domain-{short_uid()}"
domain1_name = f"domain-{short_uid()}"
domain0_arn = get_domain_arn(domain0_name, "us-east-1",
TEST_AWS_ACCOUNT_ID)
domain1_arn = get_domain_arn(domain1_name, "us-east-1",
TEST_AWS_ACCOUNT_ID)
cluster0 = manager.create(domain0_arn, dict(DomainName=domain0_name))
cluster1 = manager.create(domain1_arn, dict(DomainName=domain1_name))
try:
# spawn the two clusters
assert cluster0.wait_is_up(240)
assert cluster1.wait_is_up(240)
retry(lambda: try_cluster_health(cluster0.url),
retries=12,
sleep=10)
retry(lambda: try_cluster_health(cluster1.url),
retries=12,
sleep=10)
# create an index in cluster0, wait for it to appear, make sure it's not in cluster1
index0_url = cluster0.url + "/my-index?pretty"
index1_url = cluster1.url + "/my-index?pretty"
response = requests.put(index0_url)
assert response.ok, "failed to put index into cluster %s: %s" % (
cluster0.url,
response.text,
)
assert poll_condition(lambda: requests.head(index0_url).ok,
timeout=10), "gave up waiting for index"
assert requests.head(
index1_url).ok, "expected index to appear by multiplexing"
finally:
call_safe(cluster0.shutdown)
call_safe(cluster1.shutdown)
def test_multiplexing_cluster(self, monkeypatch):
monkeypatch.setattr(config, "OPENSEARCH_ENDPOINT_STRATEGY", "domain")
monkeypatch.setattr(config, "OPENSEARCH_MULTI_CLUSTER", False)
manager = MultiplexingClusterManager()
# create two opensearch domains
domain_key_0 = DomainKey(domain_name=f"domain-{short_uid()}",
region="us-east-1",
account=TEST_AWS_ACCOUNT_ID)
domain_key_1 = DomainKey(domain_name=f"domain-{short_uid()}",
region="us-east-1",
account=TEST_AWS_ACCOUNT_ID)
cluster_0 = manager.create(domain_key_0.arn,
OPENSEARCH_DEFAULT_VERSION)
cluster_1 = manager.create(domain_key_1.arn,
OPENSEARCH_DEFAULT_VERSION)
try:
# spawn the two clusters
assert cluster_0.wait_is_up(240)
assert cluster_1.wait_is_up(240)
retry(lambda: try_cluster_health(cluster_0.url),
retries=12,
sleep=10)
retry(lambda: try_cluster_health(cluster_1.url),
retries=12,
sleep=10)
# create an index in cluster_0, wait for it to appear, make sure it's in cluster_1, too
index_url_0 = cluster_0.url + "/my-index?pretty"
index_url_1 = cluster_1.url + "/my-index?pretty"
response = requests.put(index_url_0)
assert response.ok, f"failed to put index into cluster {cluster_0.url}: {response.text}"
assert poll_condition(lambda: requests.head(index_url_0).ok,
timeout=10), "gave up waiting for index"
assert requests.head(
index_url_1).ok, "index should appear in second cluster"
finally:
call_safe(cluster_0.shutdown)
call_safe(cluster_1.shutdown)
def test_run_and_stop_server(self):
port = get_free_tcp_port()
host = "127.0.0.1"
LOG.info("%.2f starting server on port %d", time.time(), port)
thread = run_server(port=port, bind_address=host, asynchronous=True)
try:
url = f"http://{host}:{port}"
assert poll_condition(
lambda: is_port_open(url, http_path="/"),
timeout=15), f"gave up waiting for port {port}"
finally:
LOG.info("%.2f stopping server on port %d", time.time(), port)
thread.stop()
LOG.info("%.2f waiting on server to shut down", time.time())
thread.join(timeout=15)
assert not is_port_open(port), "port is still open after stop"
LOG.info("%.2f port stopped %d", time.time(), port)
def start_runtime_components():
from localstack.services.edge import start_edge
from localstack.services.internal import LocalstackResourceHandler, get_internal_apis
# serve internal APIs through the generic proxy
ProxyListener.DEFAULT_LISTENERS.append(LocalstackResourceHandler(get_internal_apis()))
# TODO: we want a composable LocalStack runtime (edge proxy, service manager, dns, ...)
t = start_thread(start_edge, quiet=False)
# TODO: properly encapsulate starting/stopping of edge server in a class
if not poll_condition(
lambda: is_port_open(config.get_edge_port_http()), timeout=5, interval=0.1
):
raise TimeoutError(
f"gave up waiting for edge server on {config.EDGE_BIND_HOST}:{config.EDGE_PORT}"
)
return t
def test_run_and_stop_server_from_different_threads(self):
port = get_free_tcp_port()
host = "127.0.0.1"
LOG.info("%.2f starting server on port %d", time.time(), port)
thread = run_server(port=port, bind_address=host, asynchronous=True)
try:
url = f"http://{host}:{port}"
self.assertTrue(
poll_condition(lambda: is_port_open(url, http_path="/"),
timeout=15),
"gave up waiting for port %d " % port,
)
finally:
LOG.info("%.2f stopping server on port %d", time.time(), port)
threading.Thread(target=thread.stop).start()
LOG.info("%.2f waiting on server to shut down", time.time())
thread.join(timeout=15)
self.assertFalse(is_port_open(port), "port is still open after stop")
LOG.info("%.2f port stopped %d", time.time(), port)
def test_start_and_stop(self, monkeypatch):
monkeypatch.setattr(config, "FORWARD_EDGE_INMEM", False)
proxy_port = get_free_tcp_port()
backend_port = get_free_tcp_port()
server = start_proxy_for_service(
"myservice",
proxy_port,
backend_port,
update_listener=None,
quiet=True,
)
assert server
try:
assert poll_condition(lambda: is_port_open(proxy_port), timeout=15)
finally:
server.stop()
server.join(timeout=15)
assert not is_port_open(proxy_port)
def http_server(handler, host="127.0.0.1", port=None) -> str:
"""
Create a temporary http server on a random port (or the specified port) with the given handler
for the duration of the context manager.
Example usage:
def handler(request, data):
print(request.method, request.path, data)
with testutil.http_server(handler) as url:
requests.post(url, json={"message": "hello"})
"""
from localstack.utils.server.http2_server import run_server
host = host
port = port or get_free_tcp_port()
thread = run_server(port, host, handler=handler, asynchronous=True)
url = f"http://{host}:{port}"
assert poll_condition(lambda: is_port_open(port),
timeout=5), f"server on port {port} did not start"
yield url
thread.stop()
def test_domain_creation(self):
es_client = aws_stack.connect_to_service("es")
# make sure we cannot re-create same domain name
self.assertRaises(
ClientError,
es_client.create_elasticsearch_domain,
DomainName=self.domain_name,
)
# get domain status
status = es_client.describe_elasticsearch_domain(
DomainName=self.domain_name)
self.assertEqual(self.domain_name,
status["DomainStatus"]["DomainName"])
self.assertTrue(status["DomainStatus"]["Created"])
self.assertFalse(status["DomainStatus"]["Deleted"])
# wait for domain to appear
self.assertTrue(
poll_condition(
lambda: status["DomainStatus"].get("Processing") is False,
timeout=30))
self.assertEqual(
"localhost:%s" % config.PORT_ELASTICSEARCH,
status["DomainStatus"]["Endpoint"],
)
self.assertTrue(status["DomainStatus"]["EBSOptions"]["EBSEnabled"])
# make sure we can fake adding tags to a domain
response = es_client.add_tags(ARN="string",
TagList=[{
"Key": "SOME_TAG",
"Value": "SOME_VALUE"
}])
self.assertEqual(200, response["ResponseMetadata"]["HTTPStatusCode"])
def test_kinesis_firehose_opensearch_s3_backup(
self,
firehose_client,
kinesis_client,
opensearch_client,
s3_client,
s3_bucket,
kinesis_create_stream,
monkeypatch,
opensearch_endpoint_strategy,
):
domain_name = f"test-domain-{short_uid()}"
stream_name = f"test-stream-{short_uid()}"
role_arn = "arn:aws:iam::000000000000:role/Firehose-Role"
delivery_stream_name = f"test-delivery-stream-{short_uid()}"
monkeypatch.setattr(config, "OPENSEARCH_ENDPOINT_STRATEGY",
opensearch_endpoint_strategy)
try:
opensearch_create_response = opensearch_client.create_domain(
DomainName=domain_name)
opensearch_url = f"http://{opensearch_create_response['DomainStatus']['Endpoint']}"
opensearch_arn = opensearch_create_response["DomainStatus"]["ARN"]
# create s3 backup bucket arn
bucket_arn = aws_stack.s3_bucket_arn(s3_bucket)
# create kinesis stream
kinesis_create_stream(StreamName=stream_name, ShardCount=2)
stream_arn = kinesis_client.describe_stream(
StreamName=stream_name)["StreamDescription"]["StreamARN"]
kinesis_stream_source_def = {
"KinesisStreamARN": stream_arn,
"RoleARN": role_arn,
}
opensearch_destination_configuration = {
"RoleARN": role_arn,
"DomainARN": opensearch_arn,
"IndexName": "activity",
"TypeName": "activity",
"S3BackupMode": "AllDocuments",
"S3Configuration": {
"RoleARN": role_arn,
"BucketARN": bucket_arn,
},
}
firehose_client.create_delivery_stream(
DeliveryStreamName=delivery_stream_name,
DeliveryStreamType="KinesisStreamAsSource",
KinesisStreamSourceConfiguration=kinesis_stream_source_def,
AmazonopensearchserviceDestinationConfiguration=
opensearch_destination_configuration,
)
# wait for opensearch cluster to be ready
def check_domain_state():
result = opensearch_client.describe_domain(
DomainName=domain_name)["DomainStatus"]["Processing"]
return not result
assert poll_condition(check_domain_state, 30, 1)
# put kinesis stream record
kinesis_record = {"target": "hello"}
kinesis_client.put_record(StreamName=stream_name,
Data=to_bytes(
json.dumps(kinesis_record)),
PartitionKey="1")
firehose_record = {"target": "world"}
firehose_client.put_record(
DeliveryStreamName=delivery_stream_name,
Record={"Data": to_bytes(json.dumps(firehose_record))},
)
def assert_opensearch_contents():
response = requests.get(f"{opensearch_url}/activity/_search")
response_bod = response.json()
assert "hits" in response_bod
response_bod_hits = response_bod["hits"]
assert "hits" in response_bod_hits
result = response_bod_hits["hits"]
assert len(result) == 2
sources = [item["_source"] for item in result]
assert firehose_record in sources
assert kinesis_record in sources
retry(assert_opensearch_contents)
def assert_s3_contents():
result = s3_client.list_objects(Bucket=s3_bucket)
contents = []
for o in result.get("Contents"):
data = s3_client.get_object(Bucket=s3_bucket,
Key=o.get("Key"))
content = data["Body"].read()
contents.append(content)
assert len(contents) == 2
assert to_bytes(json.dumps(firehose_record)) in contents
assert to_bytes(json.dumps(kinesis_record)) in contents
retry(assert_s3_contents)
finally:
firehose_client.delete_delivery_stream(
DeliveryStreamName=delivery_stream_name)
opensearch_client.delete_domain(DomainName=domain_name)
def wait_is_container_running(self, timeout=None) -> bool:
return poll_condition(self.is_container_running, timeout)
