def uncached_fetch(instance_ids: list) -> dict:
logger.bind(instance_ids=instance_ids).debug(
"Fetch EC2 instances from AWS with describe_instances."
) if s.DEBUG else None
instances_list = []
ids_chunks = toolbox.chunk_list(instance_ids, 100)
for ids_chunk in ids_chunks:
start_time = default_timer()
response = self.ec2.describe_instances(InstanceIds=ids_chunk)
for reservation in response["Reservations"]:
instances_list += reservation["Instances"]
DURATION.labels("describe_instances").observe(
max(default_timer() - start_time, 0)
)
if self.should_throttle:
time.sleep(self.throttle_interval_seconds)
dct = toolbox.list_to_dict(instances_list, "InstanceId")
if s.PRINT_STRUCTS:
toolbox.pstruct(dct, "ec2.describe_instances")
return dct
def uncached_fetch(arns: list) -> dict:
logger.bind(arns=arns).debug(
"Fetch container instances from AWS with describe_container_instances."
) if s.DEBUG else None
lst = []
arns_chunks = toolbox.chunk_list(arns, 100)
for arns_chunk in arns_chunks:
start_time = default_timer()
lst += self.ecs.describe_container_instances(
cluster=cluster_arn, containerInstances=arns_chunk
)["containerInstances"]
DURATION.labels("describe_container_instances").observe(
max(default_timer() - start_time, 0)
)
if self.should_throttle:
time.sleep(self.throttle_interval_seconds)
dct = toolbox.list_to_dict(lst, "containerInstanceArn")
if s.PRINT_STRUCTS:
toolbox.pstruct(dct, "describe_container_instances")
return dct
def uncached_fetch(task_arns: list) -> dict:
logger.bind(cluster_arn=cluster_arn, task_arns=task_arns).debug(
"Fetch tasks from AWS with describe_tasks."
) if s.DEBUG else None
tasks = []
chunked_task_arns = toolbox.chunk_list(task_arns, 100)
for task_arns_chunk in chunked_task_arns:
start_time = default_timer()
_t = self.ecs.describe_tasks(cluster=cluster_arn,
tasks=task_arns_chunk)["tasks"]
tasks += list(
filter(lambda x: x.get("lastStatus", None) == "RUNNING",
_t))
DURATION.labels("describe_tasks").observe(
max(default_timer() - start_time, 0))
if self.should_throttle:
time.sleep(self.throttle_interval_seconds)
if s.PRINT_STRUCTS:
toolbox.pstruct(tasks, "describe_tasks")
return toolbox.list_to_dict(tasks, "taskArn")
def test_chunk_list_with_uneven_size():
big_list = list(range(103))
chunks = toolbox.chunk_list(big_list, 10)
assert len(chunks) == 11
for chunk in chunks:
assert len(chunk) <= 10
assert len(chunks[-1]) == 3
def test_chunk_list_with_even_size():
big_list = list(range(100))
chunks = toolbox.chunk_list(big_list, 10)
assert len(chunks) == 10
for chunk in chunks:
assert len(chunk) <= 10
