def print_rule_stats(reset=False):
"""Print some additional rule stats
Args:
reset (bool): Optional flag to reset the tracking statistics after printing
"""
if not RULE_STATS:
LOGGER.error('No rule statistics to print')
return
max_rule_name_len = max([len(rule) for rule in RULE_STATS.keys()])
stat_lines = []
for rule, stat in sorted(RULE_STATS.iteritems(), key=lambda (k, v):
(v, k)):
stat_lines.append('{rule: <{pad}}{stat}'.format(rule=rule,
pad=max_rule_name_len +
4,
stat=stat))
LOGGER.info('Rule statistics:\n%s', '\n'.join(stat_lines))
# Clear the dictionary that is storing statistics
# This allows for resetting when cumulative stats are not wanted
if reset:
RULE_STATS.clear()
def log_metric(cls, lambda_function, metric_name, value):
"""Log a metric using the logger the list of metrics to be sent to CloudWatch
Args:
metric_name (str): Name of metric to publish to. Choices are in `Metrics.Name` above
value (num): Numeric information to post to metric. AWS expects
this to be of type 'float' but will accept any numeric value that
is not super small (negative) or super large.
"""
# Do not log any metrics if they have been disabled by the user
if not ENABLE_METRICS:
return
if lambda_function not in cls._available_metrics:
LOGGER.error('Function \'%s\' not defined in available metrics. Options are: %s',
lambda_function,
', '.join('\'{}\''.format(key) for key in cls._available_metrics
if cls._available_metrics[key]))
return
if metric_name not in cls._available_metrics[lambda_function]:
LOGGER.error('Metric name (\'%s\') not defined for \'%s\' function. Options are: %s',
metric_name,
lambda_function,
', '.join('\'{}\''.format(value)
for value in cls._available_metrics[lambda_function]))
return
# Use a default format for logging this metric that will get picked up by the filters
LOGGER.info('{"metric_name": "%s", "metric_value": %s}', metric_name, value)
def _wrapped(details):
message = '[Backoff]: Giving up calling \'{}\' after {:f} seconds and {:d} tries'.format(
details['target'].__name__, details['elapsed'], details['tries'])
if not debug_only:
LOGGER.info(message)
else:
LOGGER.debug(message)
def _wrapped(details):
message = '[Backoff]: Calling \'{}\' again in {:f} seconds with {:d} tries so far'.format(
details['target'].__name__, details['wait'], details['tries'])
if not debug_only:
LOGGER.info(message)
else:
LOGGER.debug(message)
def giveup_handler(details):
"""Backoff logging handler for when backoff gives up.
Args:
details (dict): Backoff context containing the number of tries,
target function currently executing, kwargs, args, value,
and wait time.
"""
LOGGER.info('[Backoff]: Exiting after %d tries calling %s',
details['tries'], details['target'].__name__)
def backoff_handler(details):
"""Backoff logging handler for when polling occurs.
Args:
details (dict): Backoff context containing the number of tries,
target function currently executing, kwargs, args, value,
and wait time.
"""
LOGGER.info(
'[Backoff]: Trying again in %f seconds after %d tries calling %s',
details['wait'], details['tries'], details['target'].__name__)
def _wrapped(details):
message = '[Backoff]: Successfully called \'{}\' after {:f} seconds and {:d} tries'.format(
details['target'].__name__,
details['elapsed'],
details['tries'],
)
# We will only want to log backoff on_success when tries more than 1.
if not debug_only and int(details['tries']) > 1:
LOGGER.info(message)
else:
LOGGER.debug(message)
def drop_table(self, table_name):
"""Drop a specific table in the database
Args:
table_name (str): Table name which should be dropped from the database
Returns:
bool: True if the table was successfully dropped, False otherwise
"""
success = self.run_query('DROP TABLE {}'.format(table_name))
if not success:
LOGGER.error('Unable to drop table: %s', table_name)
return False
LOGGER.info('Successfully dropped table: %s', table_name)
return True
def download_s3_objects(self):
"""Download S3 files (json format) from S3 buckets into memory.
Returns:
dict: A dictionary contains information loaded from S3. The file name
will be the key, and value is file content in json format.
"""
_lookup_tables = {}
for bucket, files in self._buckets_info.iteritems():
for json_file in files:
try:
start_time = time.time()
s3_object = self._s3_client.Object(bucket, json_file).get()
size_kb = round(s3_object.get('ContentLength') / 1024.0, 2)
size_mb = round(size_kb / 1024.0, 2)
display_size = '{}MB'.format(size_mb) if size_mb else '{}KB'.format(size_kb)
LOGGER.info('Downloaded S3 file size %s and updated lookup table %s',
display_size, json_file)
data = s3_object.get('Body').read()
except ClientError as err:
LOGGER.error('Encounterred error while downloading %s from %s, %s',
json_file, bucket, err.response['Error']['Message'])
return _lookup_tables
except(Timeout, TimeoutError):
# Catching TimeoutError will catch both `ReadTimeoutError` and
# `ConnectionTimeoutError`.
LOGGER.error('Reading %s from S3 is timed out.', json_file)
return _lookup_tables
# The lookup data can optionally be compressed, so try to decompress
# This will fall back and use the original data if decompression fails
try:
data = zlib.decompress(data, 47)
except zlib.error:
LOGGER.debug('Data in \'%s\' is not compressed', json_file)
table_name = os.path.splitext(json_file)[0]
_lookup_tables[table_name] = json.loads(data)
total_time = time.time() - start_time
LOGGER.info('Downloaded S3 file %s seconds', round(total_time, 2))
return _lookup_tables
def toggle_staged_state(self, rule_name, stage):
"""Mark the specified rule as staged=True or staged=False
Args:
rule_name (string): The name of the rule being staged
stage (bool): True if this rule should be staged and False if
this rule should be promoted out of staging.
"""
if rule_name not in self.remote_rule_info:
LOGGER.error(
'Staging status for rule \'%s\' cannot be set to %s; rule does not exist',
rule_name, stage)
return
if self.remote_rule_info[rule_name]['Staged'] and stage:
LOGGER.info(
'Rule \'%s\' is already staged and will have its staging window updated',
rule_name)
LOGGER.debug('Toggling staged state for rule \'%s\' to: %s', rule_name,
stage)
update_expressions = ['set Staged = :staged']
expression_attributes = [':staged']
expression_values = [stage]
# If staging, add some additonal staging context to the expression
if stage:
update_expressions.extend(
['StagedAt = :staged_at', 'StagedUntil = :staged_until'])
expression_attributes.extend([':staged_at', ':staged_until'])
expression_values.extend(self._staged_window())
args = {
'UpdateExpression':
','.join(update_expressions),
'ExpressionAttributeValues':
dict(zip(expression_attributes, expression_values))
}
args.update(self._default_dynamo_kwargs(rule_name))
self._table.update_item(**args)
def load_lookup_tables(cls, config):
"""Load arbitrary json files to memory from S3 buckets when lookup table enabled
The lookup tables will also be refreshed based on "cache_refresh_minutes" setting
in the config.
Args:
config (dict): Loaded configuration from 'conf/' directory
Returns:
Return False if lookup table enabled or missing config. Otherwise, it
will return an instance of LookupTables class.
"""
lookup_tables = config['global']['infrastructure'].get('lookup_tables')
if not (lookup_tables and lookup_tables.get('enabled', False)):
return False
buckets_info = lookup_tables.get('buckets')
if not buckets_info:
LOGGER.error('Buckets not defined')
return False
lookup_refresh_interval = lookup_tables.get('cache_refresh_minutes',
10)
now = datetime.utcnow()
refresh_delta = timedelta(minutes=lookup_refresh_interval)
needs_refresh = cls._LOOKUP_TABLES_LAST_REFRESH + refresh_delta < now
if not needs_refresh:
LOGGER.debug(
'lookup tables do not need refresh (last refresh time: %s; '
'current time: %s)', cls._LOOKUP_TABLES_LAST_REFRESH, now)
return False
LOGGER.info(
'Refreshing lookup tables (last refresh time: %s; current time: %s)',
cls._LOOKUP_TABLES_LAST_REFRESH, now)
cls._LOOKUP_TABLES_LAST_REFRESH = now
return cls(buckets_info)
def download_s3_objects(self):
"""Download S3 files (json format) from S3 buckets into memory.
Returns:
dict: A dictionary contains information loaded from S3. The file name
will be the key, and value is file content in json format.
"""
_lookup_tables = {}
for bucket, files in self._buckets_info.iteritems():
for json_file in files:
table_name = os.path.splitext(json_file)[0]
try:
start_time = time.time()
s3_object = self._s3_client.Object(bucket, json_file).get()
size_kb = round(s3_object.get('ContentLength') / 1024.0, 2)
size_mb = round(size_kb / 1024.0, 2)
display_size = '{}MB'.format(
size_mb) if size_mb else '{}KB'.format(size_kb)
LOGGER.info(
'Downloaded S3 file size %s and updated lookup table %s',
display_size, table_name)
_lookup_tables[table_name] = json.loads(
s3_object.get('Body').read())
except ClientError as err:
LOGGER.error(
'Encounterred error while downloading %s from %s, %s',
json_file, bucket, err.response['Error']['Message'])
return _lookup_tables
total_time = time.time() - start_time
LOGGER.info('Downloaded S3 file %s seconds',
round(total_time, 2))
return _lookup_tables