def uncached_fetch(instance_ids: list) -> dict:
logger.bind(instance_ids=instance_ids).debug(
"Fetch EC2 instances from AWS with describe_instances."
) if s.DEBUG else None
instances_list = []
ids_chunks = toolbox.chunk_list(instance_ids, 100)
for ids_chunk in ids_chunks:
start_time = default_timer()
response = self.ec2.describe_instances(InstanceIds=ids_chunk)
for reservation in response["Reservations"]:
instances_list += reservation["Instances"]
DURATION.labels("describe_instances").observe(
max(default_timer() - start_time, 0)
)
if self.should_throttle:
time.sleep(self.throttle_interval_seconds)
dct = toolbox.list_to_dict(instances_list, "InstanceId")
if s.PRINT_STRUCTS:
toolbox.pstruct(dct, "ec2.describe_instances")
return dct
def uncached_fetch(arns: list) -> dict:
logger.bind(arns=arns).debug(
"Fetch container instances from AWS with describe_container_instances."
) if s.DEBUG else None
lst = []
arns_chunks = toolbox.chunk_list(arns, 100)
for arns_chunk in arns_chunks:
start_time = default_timer()
lst += self.ecs.describe_container_instances(
cluster=cluster_arn, containerInstances=arns_chunk
)["containerInstances"]
DURATION.labels("describe_container_instances").observe(
max(default_timer() - start_time, 0)
)
if self.should_throttle:
time.sleep(self.throttle_interval_seconds)
dct = toolbox.list_to_dict(lst, "containerInstanceArn")
if s.PRINT_STRUCTS:
toolbox.pstruct(dct, "describe_container_instances")
return dct
def uncached_fetch(task_arns: list) -> dict:
logger.bind(cluster_arn=cluster_arn, task_arns=task_arns).debug(
"Fetch tasks from AWS with describe_tasks."
) if s.DEBUG else None
tasks = []
chunked_task_arns = toolbox.chunk_list(task_arns, 100)
for task_arns_chunk in chunked_task_arns:
start_time = default_timer()
_t = self.ecs.describe_tasks(cluster=cluster_arn,
tasks=task_arns_chunk)["tasks"]
tasks += list(
filter(lambda x: x.get("lastStatus", None) == "RUNNING",
_t))
DURATION.labels("describe_tasks").observe(
max(default_timer() - start_time, 0))
if self.should_throttle:
time.sleep(self.throttle_interval_seconds)
if s.PRINT_STRUCTS:
toolbox.pstruct(tasks, "describe_tasks")
return toolbox.list_to_dict(tasks, "taskArn")
def test_list_to_dict():
lst = [{"key1": "hallo", "key2": "my"}, {"key1": "old", "key2": "friend"}]
dct = toolbox.list_to_dict(lst, "key1")
assert dct == {
"hallo": {"key1": "hallo", "key2": "my"},
"old": {"key1": "old", "key2": "friend"},
}
def test_discovery_full():
os.environ["AWS_DEFAULT_REGION"] = "eu-central-1"
os.environ["AWS_ACCESS_KEY_ID"] = "testing"
os.environ["AWS_SECRET_ACCESS_KEY"] = "testing"
os.environ["AWS_SECURITY_TOKEN"] = "testing"
os.environ["AWS_SESSION_TOKEN"] = "testing"
ecs_client = boto3.client("ecs")
ec2_client = boto3.client("ec2")
ecs_stubber = Stubber(ecs_client)
ec2_stubber = Stubber(ec2_client)
# ----------------------------------------------------------------------
# Preload stubbers with everything necessary.
ecs_stubber.add_response("list_clusters", data.list_clusters_response, {})
ecs_stubber.add_response(
"list_tasks", data.list_tasks_response, data.list_tasks_parameters
)
ecs_stubber.add_response(
"list_container_instances",
data.list_container_instances_response,
data.list_container_instances_parameters,
)
ecs_stubber.activate()
ec2_stubber.activate()
# --------------------------------------------------------------------------
fetcher = fetching.CachedFetcher(ecs_client, ec2_client)
discoverer = discovery.PrometheusEcsDiscoverer(fetcher)
# Inject data into caches.
fetcher.task_cache.current = toolbox.list_to_dict(
data.describe_tasks_response["tasks"], "taskArn"
)
fetcher.task_definition_cache.current = {}
for task_definition in data.describe_task_definition_responses:
fetcher.task_definition_cache.current[
task_definition["taskDefinition"]["taskDefinitionArn"]
] = task_definition["taskDefinition"]
fetcher.container_instance_cache.current = toolbox.list_to_dict(
data.describe_container_instances_response["containerInstances"],
"containerInstanceArn",
)
fetcher.ec2_instance_cache.current = {}
for reservation in data.describe_instances_response["Reservations"]:
for instance in reservation["Instances"]:
fetcher.ec2_instance_cache.current[instance["InstanceId"]] = instance
targets = discoverer.discover()
assert len(targets) == 6
target = targets[0]
assert target.ip == "10.0.2.85"
assert target.port == "32794"
assert target.p_instance == "10.0.2.85:32794"
assert target.task_name == "webapp-test-3"
assert target.metrics_path == "/app-3/metrics"
assert target.cluster_name == "cluster-name"
assert target.task_version == "2"
assert target.task_id == "550b823c-288a-4f31-b3e1-69f9ea15060d"
assert target.container_id == "213d21d5-4718-4b7f-9f7d-a1b3a8e57ad8"
assert target.instance_id == "i-0b967c2479dd4f5af"
assert target.custom_labels == {}
toolbox.pstruct(targets, "targets")