def get_batch_definition_list_from_batch_request(
self,
batch_request: BatchRequest,
) -> List[BatchDefinition]:
self._validate_batch_request(batch_request=batch_request)
if self._data_references_cache is None:
self._refresh_data_references_cache()
batch_definition_list: List[BatchDefinition] = list(
filter(
lambda batch_definition:
batch_definition_matches_batch_request(
batch_definition=batch_definition,
batch_request=batch_request),
self._get_batch_definition_list_from_cache(),
))
if batch_request.partition_request is not None:
partition_query_obj: PartitionQuery = build_partition_query(
partition_request_dict=batch_request.partition_request)
batch_definition_list = partition_query_obj.select_from_partition_request(
batch_definition_list=batch_definition_list)
if len(self.sorters) > 0:
sorted_batch_definition_list = self._sort_batch_definition_list(
batch_definition_list=batch_definition_list)
return sorted_batch_definition_list
else:
return batch_definition_list
def _get_batch_definition_list_from_batch_request(
self,
batch_request: BatchRequestBase,
) -> List[BatchDefinition]:
"""
Retrieve batch_definitions that match batch_request.
First retrieves all batch_definitions that match batch_request
- if batch_request also has a partition_query, then select batch_definitions that match partition_query.
- if data_connector has sorters configured, then sort the batch_definition list before returning.
Args:
batch_request (BatchRequestBase): BatchRequestBase (BatchRequest without attribute validation) to process
Returns:
A list of BatchDefinition objects that match BatchRequest
"""
self._validate_batch_request(batch_request=batch_request)
if self._data_references_cache is None:
self._refresh_data_references_cache()
batch_definition_list: List[BatchDefinition] = list(
filter(
lambda batch_definition: batch_definition_matches_batch_request(
batch_definition=batch_definition, batch_request=batch_request
),
self._get_batch_definition_list_from_cache(),
)
)
if batch_request.partition_request is not None:
partition_query_obj: PartitionQuery = build_partition_query(
partition_request_dict=batch_request.partition_request
)
batch_definition_list = partition_query_obj.select_from_partition_request(
batch_definition_list=batch_definition_list
)
if len(self.sorters) > 0:
sorted_batch_definition_list = self._sort_batch_definition_list(
batch_definition_list=batch_definition_list
)
return sorted_batch_definition_list
else:
return batch_definition_list