def _get_china_env_var(env_var_name: str) -> str:
value = os.environ.get(env_var_name)
if value:
return value
get_logger().critical(
f"Failed to send customer logs because {env_var_name} env var is missing and it is needed in China"
)
raise ChinaMissingEnvVar()
def _ship_logs_to_lumigo(
shipper_outputs: List[AwsLogSubscriptionEvent],
programmatic_error_keyword: Optional[str] = None,
exclude_filters: Optional[List[str]] = None,
) -> int:
get_logger().debug(f"Number of logs before filter {len(shipper_outputs)}")
filter_keywords = FILTER_KEYWORDS.copy()
if programmatic_error_keyword:
filter_keywords.append(programmatic_error_keyword)
shipper_outputs = filter_logs(shipper_outputs, filter_keywords,
exclude_filters)
get_logger().debug(f"Number of logs after filter {len(shipper_outputs)}")
if len(shipper_outputs) > 0 and not LOG_STREAM_KIIL_SWITCH:
account_id = shipper_outputs[0].owner
firehose_records = list(map(asdict, shipper_outputs))
firehose_dal = FirehoseDal(stream_name=STREAM_NAME,
account_id=account_id)
return firehose_dal.put_record_batch(firehose_records)
return 0
def __init__(
self,
stream_name: str,
account_id: str,
max_retry_count: int = MAX_RETRY_COUNT,
batch_size: int = MAX_MESSAGES_TO_FIREHOSE,
):
"""
:param stream_name: Name of the firehose delivery stream.
"""
self.current_account_id = account_id
self._client = FirehoseDal.get_boto_client(account_id,
TARGET_ACCOUNT_ID)
self._stream_name = stream_name
self.max_retry_count = max_retry_count
self.failed_by_error_code: Dict[str, int] = defaultdict(int)
self.batch_size = min(batch_size, MAX_MESSAGES_TO_FIREHOSE)
get_logger().debug(
f"Init firehose stream: {stream_name} target account: {TARGET_ACCOUNT_ID}"
)
def ship_logs(
aws_event: dict,
programmatic_error_keyword: Optional[str] = None,
exclude_filters: Optional[List[str]] = None,
) -> int:
try:
shipper_output: AwsLogSubscriptionEvent = extract_aws_logs_data(
aws_event)
res = _ship_logs_to_lumigo(
shipper_outputs=[shipper_output],
programmatic_error_keyword=programmatic_error_keyword,
exclude_filters=exclude_filters,
)
get_logger().info(f"Successfully sent {res} logs")
return res
except ChinaMissingEnvVar:
pass
except Exception as e:
# lumigo_shipper will print out the exception but won't raises it
get_logger().critical("Failed to send customer logs", exc_info=e)
return 0
def put_record_batch(self, records: List[dict]) -> int:
"""
:param max_batch_size: max batch size
:param records: The records to put
:return: number of records inserted
"""
number_of_records = 0
firehose_records = self._convert_to_firehose_record(records)
chunks = split_to_chunks(firehose_records, self.batch_size)
batches: List[Batch] = self.create_batches_from_chunks(chunks)
while batches:
current_batch = batches.pop(0)
should_retry = current_batch.retry_count < self.max_retry_count
try:
response = self._client.put_record_batch(
DeliveryStreamName=self._stream_name,
Records=current_batch.records)["RequestResponses"]
failed_items = self.get_failed_items(current_batch, response)
self.update_failed_by_error_code(response)
success_items_len = len(
current_batch.records) - len(failed_items)
number_of_records += success_items_len
if any(failed_items) and should_retry:
batches.append(
self.create_next_batch(current_batch, failed_items))
except Exception as e:
get_logger().debug(
"Error while trying to send data to firehose",
exc_info=e,
)
self.failed_by_error_code[str(type(e).__name__)] += 1
if should_retry:
next_records = current_batch.records
batches.append(
self.create_next_batch(current_batch, next_records))
return number_of_records