def uncached_fetch(instance_ids: list) -> dict: logger.bind(instance_ids=instance_ids).debug( "Fetch EC2 instances from AWS with describe_instances." ) if s.DEBUG else None instances_list = [] ids_chunks = toolbox.chunk_list(instance_ids, 100) for ids_chunk in ids_chunks: start_time = default_timer() response = self.ec2.describe_instances(InstanceIds=ids_chunk) for reservation in response["Reservations"]: instances_list += reservation["Instances"] DURATION.labels("describe_instances").observe( max(default_timer() - start_time, 0) ) if self.should_throttle: time.sleep(self.throttle_interval_seconds) dct = toolbox.list_to_dict(instances_list, "InstanceId") if s.PRINT_STRUCTS: toolbox.pstruct(dct, "ec2.describe_instances") return dct
def uncached_fetch(arns: list) -> dict: logger.bind(arns=arns).debug( "Fetch container instances from AWS with describe_container_instances." ) if s.DEBUG else None lst = [] arns_chunks = toolbox.chunk_list(arns, 100) for arns_chunk in arns_chunks: start_time = default_timer() lst += self.ecs.describe_container_instances( cluster=cluster_arn, containerInstances=arns_chunk )["containerInstances"] DURATION.labels("describe_container_instances").observe( max(default_timer() - start_time, 0) ) if self.should_throttle: time.sleep(self.throttle_interval_seconds) dct = toolbox.list_to_dict(lst, "containerInstanceArn") if s.PRINT_STRUCTS: toolbox.pstruct(dct, "describe_container_instances") return dct
def uncached_fetch(task_arns: list) -> dict: logger.bind(cluster_arn=cluster_arn, task_arns=task_arns).debug( "Fetch tasks from AWS with describe_tasks." ) if s.DEBUG else None tasks = [] chunked_task_arns = toolbox.chunk_list(task_arns, 100) for task_arns_chunk in chunked_task_arns: start_time = default_timer() _t = self.ecs.describe_tasks(cluster=cluster_arn, tasks=task_arns_chunk)["tasks"] tasks += list( filter(lambda x: x.get("lastStatus", None) == "RUNNING", _t)) DURATION.labels("describe_tasks").observe( max(default_timer() - start_time, 0)) if self.should_throttle: time.sleep(self.throttle_interval_seconds) if s.PRINT_STRUCTS: toolbox.pstruct(tasks, "describe_tasks") return toolbox.list_to_dict(tasks, "taskArn")
def test_chunk_list_with_uneven_size(): big_list = list(range(103)) chunks = toolbox.chunk_list(big_list, 10) assert len(chunks) == 11 for chunk in chunks: assert len(chunk) <= 10 assert len(chunks[-1]) == 3
def test_chunk_list_with_even_size(): big_list = list(range(100)) chunks = toolbox.chunk_list(big_list, 10) assert len(chunks) == 10 for chunk in chunks: assert len(chunk) <= 10