async def _loop_single_period(self_monitoring: LogSelfMonitoring,
sfm_queue: Queue, context: LoggingContext,
instance_metadata: InstanceMetadata):
try:
sfm_list = _pull_sfm(sfm_queue)
if sfm_list:
async with init_gcp_client_session() as gcp_session:
context = await _create_sfm_logs_context(
sfm_queue, context, gcp_session, instance_metadata)
self_monitoring = aggregate_self_monitoring_metrics(
self_monitoring, sfm_list)
_log_self_monitoring_data(self_monitoring, context)
if context.self_monitoring_enabled:
if context.token is None:
context.log(
"Cannot proceed without authorization token, failed to send log self monitoring"
)
return
if not isinstance(context.token, str):
context.log(
f"Failed to fetch access token, got non string value: {context.token}"
)
return
time_series = create_self_monitoring_time_series(
self_monitoring, context)
await push_self_monitoring_time_series(
context, time_series)
for _ in sfm_list:
sfm_queue.task_done()
except Exception:
context.exception("Log SFM Loop Exception:")
def _load_configs(self):
context = LoggingContext("ME startup")
working_directory = os.path.dirname(os.path.realpath(__file__))
config_directory = os.path.join(working_directory, "../../config_logs")
config_files = [
file for file in listdir(config_directory)
if isfile(os.path.join(config_directory, file))
and _is_json_file(file)
]
for file in config_files:
config_file_path = os.path.join(config_directory, file)
try:
with open(config_file_path) as config_file:
config_json = json.load(config_file)
if config_json.get("name", "") == DEFAULT_RULE_NAME:
self.default_rule = _create_config_rules(
context, config_json)[0]
elif config_json.get("name", "") == COMMON_RULE_NAME:
self.common_rule = _create_config_rules(
context, config_json)[0]
elif config_json.get("name",
"").startswith(AUDIT_LOGS_RULE):
self.audit_logs_rules = _create_config_rules(
context, config_json)
else:
self.rules.extend(
_create_config_rules(context, config_json))
except Exception as e:
context.exception(
f"Failed to load configuration file: '{config_file_path}'")
def _apply_rule(context: LoggingContext, rule: ConfigRule, record: Dict, parsed_record: Dict):
for attribute in rule.attributes:
try:
value = jmespath.search(attribute.pattern, record, JMESPATH_OPTIONS)
if value:
parsed_record[attribute.key] = value
except Exception:
context.exception(f"Encountered exception when evaluating attribute {attribute} of rule for {rule.entity_type_name}")
def apply(self, context: LoggingContext, record: Dict, parsed_record: Dict):
try:
if self.common_rule:
_apply_rule(context, self.common_rule, record, parsed_record)
for rule in self.rules:
if _check_if_rule_applies(rule, record, parsed_record):
_apply_rule(context, rule, record, parsed_record)
return
# No matching rule has been found, applying the default rule
if self.default_rule:
_apply_rule(context, self.default_rule, record, parsed_record)
except Exception:
context.exception("Encountered exception when running Rule Engine")
async def create_sfm_worker_loop(sfm_queue: Queue,
logging_context: LoggingContext,
instance_metadata: InstanceMetadata):
while True:
try:
await asyncio.sleep(SFM_WORKER_EXECUTION_PERIOD_SECONDS)
self_monitoring = LogSelfMonitoring()
asyncio.get_event_loop().create_task(
_loop_single_period(self_monitoring, sfm_queue,
logging_context, instance_metadata))
except Exception:
logging_context.exception(
"Logs Self Monitoring Worker Loop Exception:")
def run_ack_logs(worker_name: str, sfm_queue: Queue):
logging_context = LoggingContext(worker_name)
subscriber_client = pubsub.SubscriberClient()
subscription_path = subscriber_client.subscription_path(
LOGS_SUBSCRIPTION_PROJECT, LOGS_SUBSCRIPTION_ID)
logging_context.log(f"Starting processing")
worker_state = WorkerState(worker_name)
while True:
try:
perform_pull(worker_state, sfm_queue, subscriber_client,
subscription_path)
except Exception as e:
logging_context.exception("Failed to pull messages")
def run_ack_logs(worker_name: str, sfm_queue: Queue):
logging_context = LoggingContext(worker_name)
subscriber_client = pubsub.SubscriberClient()
subscription_path = subscriber_client.subscription_path(
LOGS_SUBSCRIPTION_PROJECT, LOGS_SUBSCRIPTION_ID)
logging_context.log(f"Starting processing")
worker_state = WorkerState(worker_name)
while True:
try:
perform_pull(worker_state, sfm_queue, subscriber_client,
subscription_path)
except Exception as e:
if isinstance(e, Forbidden):
logging_context.error(
f"{e} Please check whether assigned service account has permission to fetch Pub/Sub messages."
)
else:
logging_context.exception("Failed to pull messages")
# Backoff for 1 minute to avoid spamming requests and logs
time.sleep(60)