def send_anonymous_usage_data(self, log, remove_ip_list, name):
"""
Send anonymous solution metrics
"""
if 'SEND_ANONYMOUS_USAGE_DATA' not in environ or environ.get('SEND_ANONYMOUS_USAGE_DATA').lower() != 'yes':
return
log.info("[remove_expired_ip: send_anonymous_usage_data] Start")
# Get ip set category
if 'Whitelist' in name:
ip_set = 'Allowlist'
elif 'Blacklist' in name:
ip_set = 'Denylist'
else:
ip_set = 'AllowOrDenylist'
usage_data = {
"data_type": "remove_expired_ip_lambda",
"number_of_removed_ips": len(remove_ip_list),
"ip_set": ip_set,
"lambda_invocation_count": 1,
"sns_email_notification": environ.get('SNS_EMAIL'),
}
log.info("[remove_expired_ip: send_anonymous_usage_data] Send Data")
response = send_metrics(data=usage_data)
response_code = response.status_code
log.info('[remove_expired_ip: send_anonymous_usage_data] Response Code: {}'.format(response_code))
log.info("[remove_expired_ip: send_anonymous_usage_data] End")
def send_anonymous_usage_data(log, action_type, resource_properties):
try:
if 'SendAnonymousUsageData' not in resource_properties or resource_properties[
'SendAnonymousUsageData'].lower() != 'yes':
return
log.info("[send_anonymous_usage_data] Start")
usage_data = {
"version":
resource_properties['Version'],
"data_type":
"custom_resource",
"region":
resource_properties['Region'],
"action":
action_type,
"sql_injection_protection":
resource_properties['ActivateSqlInjectionProtectionParam'],
"xss_scripting_protection":
resource_properties['ActivateCrossSiteScriptingProtectionParam'],
"http_flood_protection":
resource_properties['ActivateHttpFloodProtectionParam'],
"scanners_probes_protection":
resource_properties['ActivateScannersProbesProtectionParam'],
"reputation_lists_protection":
resource_properties['ActivateReputationListsProtectionParam'],
"bad_bot_protection":
resource_properties['ActivateBadBotProtectionParam'],
"request_threshold":
resource_properties['RequestThreshold'],
"error_threshold":
resource_properties['ErrorThreshold'],
"waf_block_period":
resource_properties['WAFBlockPeriod'],
"aws_managed_rules":
resource_properties['ActivateAWSManagedRulesParam'],
"keep_original_s3_data":
resource_properties['KeepDataInOriginalS3Location'],
"allowed_ip_retention_period_minute":
resource_properties['IPRetentionPeriodAllowedParam'],
"denied_ip_retention_period_minute":
resource_properties['IPRetentionPeriodDeniedParam'],
"sns_email_notification":
resource_properties['SNSEmailParam']
}
# --------------------------------------------------------------------------------------------------------------
log.info("[send_anonymous_usage_data] Send Data")
# --------------------------------------------------------------------------------------------------------------
response = send_metrics(data=usage_data)
response_code = response.status_code
log.info('[send_anonymous_usage_data] Response Code: {}'.format(
response_code))
log.info("[send_anonymous_usage_data] End")
except Exception as error:
log.debug("[send_anonymous_usage_data] Failed to Send Data")
log.debug(str(error))
def test_send_solution_metrics():
uuid = "waf3.0_test_00001"
solution_id = "waf_test"
data = {"test_string1": "waf3.0_test", "test_string2": "test_1"}
url = "https://oszclq8tyh.execute-api.us-east-1.amazonaws.com/prod/generic"
# url = 'https://metrics.awssolutionsbuilder.com/generic'
response = send_metrics(data, uuid, solution_id, url)
status_code = response.status_code
assert status_code == 200
def send_anonymous_usage_data(log, scope, ipset_name_v4, ipset_arn_v4,
ipset_name_v6, ipset_arn_v6):
try:
if 'SEND_ANONYMOUS_USAGE_DATA' not in environ or os.getenv(
'SEND_ANONYMOUS_USAGE_DATA').lower() != 'yes':
return
log.info("[send_anonymous_usage_data] Start")
metric_prefix = os.getenv('METRIC_NAME_PREFIX')
cw = boto3.client('cloudwatch')
usage_data = {
"data_type": "bad_bot",
"bad_bot_ip_set_size": 0,
"allowed_requests": 0,
"blocked_requests_all": 0,
"blocked_requests_bad_bot": 0,
"waf_type": os.getenv('LOG_TYPE')
}
# --------------------------------------------------------------------------------------------------------------
log.info("[send_anonymous_usage_data] Get num allowed requests")
# --------------------------------------------------------------------------------------------------------------
try:
response = cw.get_metric_statistics(
MetricName='AllowedRequests',
Namespace='AWS/WAFV2',
Statistics=['Sum'],
Period=12 * 3600,
StartTime=datetime.datetime.utcnow() -
datetime.timedelta(seconds=12 * 3600),
EndTime=datetime.datetime.utcnow(),
Dimensions=[{
"Name": "Rule",
"Value": "ALL"
}, {
"Name": "WebACL",
"Value": os.getenv('STACK_NAME')
}, {
"Name": "Region",
"Value": os.getenv('AWS_REGION')
}])
if len(response['Datapoints']) > 0:
usage_data['allowed_requests'] = response['Datapoints'][0][
'Sum']
except Exception as error:
log.info(
"[send_anonymous_usage_data] Failed to get Num Allowed Requests"
)
log.error(str(error))
# --------------------------------------------------------------------------------------------------------------
log.info(
"[send_anonymous_usage_data] Get num blocked requests - all rules")
# --------------------------------------------------------------------------------------------------------------
try:
response = cw.get_metric_statistics(
MetricName='BlockedRequests',
Namespace='AWS/WAFV2',
Statistics=['Sum'],
Period=12 * 3600,
StartTime=datetime.datetime.utcnow() -
datetime.timedelta(seconds=12 * 3600),
EndTime=datetime.datetime.utcnow(),
Dimensions=[{
"Name": "Rule",
"Value": "ALL"
}, {
"Name": "WebACL",
"Value": os.getenv('STACK_NAME')
}, {
"Name": "Region",
"Value": os.getenv('AWS_REGION')
}])
if len(response['Datapoints']) > 0:
usage_data['blocked_requests_all'] = response['Datapoints'][0][
'Sum']
except Exception as error:
log.info(
"[send_anonymous_usage_data] Failed to get num blocked requests - all rules"
)
log.error(str(error))
# --------------------------------------------------------------------------------------------------------------
log.info("[send_anonymous_usage_data] Get bad bot data")
# --------------------------------------------------------------------------------------------------------------
if 'IP_SET_ID_BAD_BOTV4' in environ or 'IP_SET_ID_BAD_BOTV6' in environ:
try:
countv4 = 0
response = waflib.get_ip_set(log, scope, ipset_name_v4,
ipset_arn_v4)
log.info(response)
if response is not None:
countv4 = len(response['IPSet']['Addresses'])
log.info("Bad Bot CountV4 %s", countv4)
countv6 = 0
response = waflib.get_ip_set(log, scope, ipset_name_v6,
ipset_arn_v6)
log.info(response)
if response is not None:
countv6 = len(response['IPSet']['Addresses'])
log.info("Bad Bot CountV6 %s", countv6)
usage_data['bad_bot_ip_set_size'] = str(countv4 + countv6)
response = cw.get_metric_statistics(
MetricName='BlockedRequests',
Namespace='AWS/WAFV2',
Statistics=['Sum'],
Period=12 * 3600,
StartTime=datetime.datetime.utcnow() -
datetime.timedelta(seconds=12 * 3600),
EndTime=datetime.datetime.utcnow(),
Dimensions=[{
"Name": "Rule",
"Value": metric_prefix + 'BadBotRule'
}, {
"Name": "WebACL",
"Value": os.getenv('STACK_NAME')
}, {
"Name": "Region",
"Value": os.getenv('AWS_REGION')
}])
if len(response['Datapoints']) > 0:
usage_data['blocked_requests_bad_bot'] = response[
'Datapoints'][0]['Sum']
except Exception as error:
log.info(
"[send_anonymous_usage_data] Failed to get bad bot data")
log.error(str(error))
# --------------------------------------------------------------------------------------------------------------
log.info("[send_anonymous_usage_data] Send Data")
# --------------------------------------------------------------------------------------------------------------
response = send_metrics(data=usage_data)
response_code = response.status_code
log.info('[send_anonymous_usage_data] Response Code: {}'.format(
response_code))
log.info("[send_anonymous_usage_data] End")
except Exception as error:
log.info("[send_anonymous_usage_data] Failed to Send Data")
log.error(str(error))
def send_anonymous_usage_data(log):
try:
if 'SEND_ANONYMOUS_USAGE_DATA' not in environ or os.getenv(
'SEND_ANONYMOUS_USAGE_DATA').lower() != 'yes':
return
log.info("[send_anonymous_usage_data] Start")
cw = boto3.client('cloudwatch')
usage_data = {
"data_type": "log_parser",
"scanners_probes_set_size": 0,
"http_flood_set_size": 0,
"allowed_requests": 0,
"blocked_requests_all": 0,
"blocked_requests_scanners_probes": 0,
"blocked_requests_http_flood": 0,
"allowed_requests_WAFWebACL": 0,
"blocked_requests_WAFWebACL": 0,
"waf_type": os.getenv('LOG_TYPE')
}
# --------------------------------------------------------------------------------------------------------------
log.info("[send_anonymous_usage_data] Get num allowed requests")
# --------------------------------------------------------------------------------------------------------------
try:
response = cw.get_metric_statistics(
MetricName='AllowedRequests',
Namespace='AWS/WAFV2',
Statistics=['Sum'],
Period=300,
StartTime=datetime.datetime.utcnow() -
datetime.timedelta(seconds=300),
EndTime=datetime.datetime.utcnow(),
Dimensions=[{
"Name": "Rule",
"Value": "ALL"
}, {
"Name": "WebACL",
"Value": os.getenv('STACK_NAME')
}, {
"Name": "Region",
"Value": os.getenv('AWS_REGION')
}])
if len(response['Datapoints']):
usage_data['allowed_requests'] = response['Datapoints'][0][
'Sum']
except Exception as error:
log.debug(
"[send_anonymous_usage_data] Failed to get Num Allowed Requests"
)
log.debug(str(error))
# --------------------------------------------------------------------------------------------------------------
log.info(
"[send_anonymous_usage_data] Get num blocked requests - all rules")
# --------------------------------------------------------------------------------------------------------------
try:
response = cw.get_metric_statistics(
MetricName='BlockedRequests',
Namespace='AWS/WAFV2',
Statistics=['Sum'],
Period=300,
StartTime=datetime.datetime.utcnow() -
datetime.timedelta(seconds=300),
EndTime=datetime.datetime.utcnow(),
Dimensions=[{
"Name": "Rule",
"Value": "ALL"
}, {
"Name": "WebACL",
"Value": os.getenv('STACK_NAME')
}, {
"Name": "Region",
"Value": os.getenv('AWS_REGION')
}])
if len(response['Datapoints']):
usage_data['blocked_requests_all'] = response['Datapoints'][0][
'Sum']
except Exception as error:
log.info(
"[send_anonymous_usage_data] Failed to get num blocked requests - all rules"
)
log.error(str(error))
# --------------------------------------------------------------------------------------------------------------
log.info("[send_anonymous_usage_data] Get scanners probes data")
# --------------------------------------------------------------------------------------------------------------
if 'IP_SET_ID_SCANNERS_PROBESV4' in environ or 'IP_SET_ID_SCANNERS_PROBESV6' in environ:
try:
countv4 = 0
response = waflib.get_ip_set(
log, scope, os.getenv('IP_SET_NAME_SCANNERS_PROBESV4'),
os.getenv('IP_SET_ID_SCANNERS_PROBESV4'))
log.info(response)
if response is not None:
countv4 = len(response['IPSet']['Addresses'])
log.info("Scanner Probes IPV4 address Count: %s", countv4)
countv6 = 0
response = waflib.get_ip_set(
log, scope, os.getenv('IP_SET_NAME_SCANNERS_PROBESV6'),
os.getenv('IP_SET_ID_SCANNERS_PROBESV6'))
log.info(response)
if response is not None:
countv6 = len(response['IPSet']['Addresses'])
log.info("Scanner Probes IPV6 address Count: %s", countv6)
usage_data['scanners_probes_set_size'] = str(countv4 + countv6)
response = cw.get_metric_statistics(
MetricName='BlockedRequests',
Namespace='AWS/WAFV2',
Statistics=['Sum'],
Period=300,
StartTime=datetime.datetime.utcnow() -
datetime.timedelta(seconds=300),
EndTime=datetime.datetime.utcnow(),
Dimensions=[{
"Name":
"Rule",
"Value":
os.getenv('METRIC_NAME_PREFIX') + 'ScannersProbesRule'
}, {
"Name": "WebACL",
"Value": os.getenv('STACK_NAME')
}, {
"Name": "Region",
"Value": os.getenv('AWS_REGION')
}])
if len(response['Datapoints']):
usage_data['blocked_requests_scanners_probes'] = response[
'Datapoints'][0]['Sum']
except Exception as error:
log.debug(
"[send_anonymous_usage_data] Failed to get scanners probes data"
)
log.debug(str(error))
# --------------------------------------------------------------------------------------------------------------
log.info("[send_anonymous_usage_data] Get HTTP flood data")
# --------------------------------------------------------------------------------------------------------------
if 'IP_SET_ID_HTTP_FLOODV4' in environ or 'IP_SET_ID_HTTP_FLOODV6' in environ:
try:
countv4 = 0
response = waflib.get_ip_set(
log, scope, os.getenv('IP_SET_NAME_HTTP_FLOODV4'),
os.getenv('IP_SET_ID_HTTP_FLOODV4'))
log.info(response)
if response is not None:
countv4 = len(response['IPSet']['Addresses'])
log.info("HTTP Flood IPV4 address Count: %s", countv4)
countv6 = 0
response = waflib.get_ip_set(
log, scope, os.getenv('IP_SET_NAME_HTTP_FLOODV6'),
os.getenv('IP_SET_ID_HTTP_FLOODV6'))
log.info(response)
if response is not None:
countv6 = len(response['IPSet']['Addresses'])
log.info("HTTP Flood IPV6 address Count: %s", countv6)
usage_data['http_flood_set_size'] = str(countv4 + countv6)
response = cw.get_metric_statistics(
MetricName='BlockedRequests',
Namespace='AWS/WAFV2',
Statistics=['Sum'],
Period=300,
StartTime=datetime.datetime.utcnow() -
datetime.timedelta(seconds=300),
EndTime=datetime.datetime.utcnow(),
Dimensions=[{
"Name":
"Rule",
"Value":
os.getenv('METRIC_NAME_PREFIX') +
'HttpFloodRegularRule'
}, {
"Name": "WebACL",
"Value": os.getenv('STACK_NAME')
}, {
"Name": "Region",
"Value": os.getenv('AWS_REGION')
}])
if len(response['Datapoints']):
usage_data['blocked_requests_http_flood'] = response[
'Datapoints'][0]['Sum']
except Exception as error:
log.info(
"[send_anonymous_usage_data] Failed to get HTTP flood data"
)
log.error(str(error))
# --------------------------------------------------------------------------------------------------------------
log.info(
"[send_anonymous_usage_data] Get num allowed requests - WAF Web ACL"
)
# --------------------------------------------------------------------------------------------------------------
try:
response = cw.get_metric_statistics(
MetricName='AllowedRequests',
Namespace='AWS/WAFV2',
Statistics=['Sum'],
Period=300,
StartTime=datetime.datetime.utcnow() -
datetime.timedelta(seconds=300),
EndTime=datetime.datetime.utcnow(),
Dimensions=[{
"Name":
"Rule",
"Value":
os.getenv('METRIC_NAME_PREFIX') + 'WAFWebACL'
}, {
"Name": "WebACL",
"Value": os.getenv('STACK_NAME')
}, {
"Name": "Region",
"Value": os.getenv('AWS_REGION')
}])
if len(response['Datapoints']):
usage_data['allowed_requests_WAFWebACL'] = response[
'Datapoints'][0]['Sum']
except Exception as error:
log.info(
"[send_anonymous_usage_data] Failed to get num blocked requests - all rules"
)
log.error(str(error))
# --------------------------------------------------------------------------------------------------------------
log.info(
"[send_anonymous_usage_data] Get num blocked requests - WAF Web ACL"
)
# --------------------------------------------------------------------------------------------------------------
try:
response = cw.get_metric_statistics(
MetricName='BlockedRequests',
Namespace='AWS/WAFV2',
Statistics=['Sum'],
Period=300,
StartTime=datetime.datetime.utcnow() -
datetime.timedelta(seconds=300),
EndTime=datetime.datetime.utcnow(),
Dimensions=[{
"Name":
"Rule",
"Value":
os.getenv('METRIC_NAME_PREFIX') + 'WAFWebACL'
}, {
"Name": "WebACL",
"Value": os.getenv('STACK_NAME')
}, {
"Name": "Region",
"Value": os.getenv('AWS_REGION')
}])
if len(response['Datapoints']):
usage_data['blocked_requests_WAFWebACL'] = response[
'Datapoints'][0]['Sum']
except Exception as error:
log.info(
"[send_anonymous_usage_data] Failed to get num blocked requests - all rules"
)
log.error(str(error))
# --------------------------------------------------------------------------------------------------------------
log.info("[send_anonymous_usage_data] Send Data")
# --------------------------------------------------------------------------------------------------------------
response = send_metrics(data=usage_data)
response_code = response.status_code
log.info('[send_anonymous_usage_data] Response Code: {}'.format(
response_code))
log.info("[send_anonymous_usage_data] End")
except Exception as error:
log.info("[send_anonymous_usage_data] Failed to send data")
log.error(str(error))
def send_anonymous_usage_data(log, scope):
try:
if 'SEND_ANONYMOUS_USAGE_DATA' not in os.environ or os.getenv(
'SEND_ANONYMOUS_USAGE_DATA').lower() != 'yes':
return
log.debug("[send_anonymous_usage_data] Start")
cw = create_client('cloudwatch')
usage_data = {
"data_type": "reputation_lists",
"ipv4_reputation_lists_size": 0,
"ipv4_reputation_lists": 0,
"ipv6_reputation_lists_size": 0,
"ipv6_reputation_lists": 0,
"allowed_requests": 0,
"blocked_requests": 0,
"blocked_requests_ip_reputation_lists": 0,
"waf_type": os.getenv('LOG_TYPE')
}
# --------------------------------------------------------------------------------------------------------------
log.debug(
"[send_anonymous_usage_data] Get size of the Reputation List IP set"
)
# --------------------------------------------------------------------------------------------------------------
try:
response = waflib.get_ip_set(log, scope,
os.getenv('IP_SET_NAME_REPUTATIONV4'),
os.getenv('IP_SET_ID_REPUTATIONV4'))
if response is not None:
usage_data['ipv4_reputation_lists_size'] = len(
response['IPSet']['Addresses'])
usage_data['ipv4_reputation_lists'] = response['IPSet'][
'Addresses']
except Exception as error:
log.debug(
"[send_anonymous_usage_data] Failed to get size of the Reputation List IPV4 set"
)
log.debug(str(error))
try:
response = waflib.get_ip_set(log, scope,
os.getenv('IP_SET_NAME_REPUTATIONV6'),
os.getenv('IP_SET_ID_REPUTATIONV6'))
if response is not None:
usage_data['ipv6_reputation_lists_size'] = len(
response['IPSet']['Addresses'])
usage_data['ipv6_reputation_lists'] = response['IPSet'][
'Addresses']
except Exception as error:
log.debug(
"[send_anonymous_usage_data] Failed to get size of the Reputation List IPV6 set"
)
log.debug(str(error))
# --------------------------------------------------------------------------------------------------------------
log.debug(
"[send_anonymous_usage_data] Get total number of allowed requests")
# --------------------------------------------------------------------------------------------------------------
try:
response = cw.get_metric_statistics(
MetricName='AllowedRequests',
Namespace='AWS/WAFV2',
Statistics=['Sum'],
Period=3600,
StartTime=datetime.datetime.utcnow() -
datetime.timedelta(seconds=3600),
EndTime=datetime.datetime.utcnow(),
Dimensions=[{
"Name": "Rule",
"Value": "ALL"
}, {
"Name": "WebACL",
"Value": os.getenv('STACK_NAME')
}, {
"Name": "Region",
"Value": os.getenv('AWS_REGION')
}])
if len(response['Datapoints']):
usage_data['allowed_requests'] = response['Datapoints'][0][
'Sum']
except Exception as error:
log.debug(
"[send_anonymous_usage_data] Failed to get Num Allowed Requests"
)
log.debug(str(error))
# --------------------------------------------------------------------------------------------------------------
log.debug(
"[send_anonymous_usage_data] Get total number of blocked requests")
# --------------------------------------------------------------------------------------------------------------
try:
response = cw.get_metric_statistics(
MetricName='BlockedRequests',
Namespace='AWS/WAFV2',
Statistics=['Sum'],
Period=3600,
StartTime=datetime.datetime.utcnow() -
datetime.timedelta(seconds=3600),
EndTime=datetime.datetime.utcnow(),
Dimensions=[{
"Name": "Rule",
"Value": "ALL"
}, {
"Name": "WebACL",
"Value": os.getenv('STACK_NAME')
}, {
"Name": "Region",
"Value": os.getenv('AWS_REGION')
}])
if len(response['Datapoints']):
usage_data['blocked_requests'] = response['Datapoints'][0][
'Sum']
except Exception as error:
log.debug(
"[send_anonymous_usage_data] Failed to get Num Allowed Requests"
)
log.debug(str(error))
# --------------------------------------------------------------------------------------------------------------
log.debug(
"[send_anonymous_usage_data] Get total number of blocked requests for Reputation Lists Rule"
)
# --------------------------------------------------------------------------------------------------------------
try:
response = cw.get_metric_statistics(
MetricName='BlockedRequests',
Namespace='AWS/WAFV2',
Statistics=['Sum'],
Period=3600,
StartTime=datetime.datetime.utcnow() -
datetime.timedelta(seconds=3600),
EndTime=datetime.datetime.utcnow(),
Dimensions=[{
"Name": "Rule",
"Value": os.getenv('IPREPUTATIONLIST_METRICNAME')
}, {
"Name": "WebACL",
"Value": os.getenv('STACK_NAME')
}, {
"Name": "Region",
"Value": os.getenv('AWS_REGION')
}])
if len(response['Datapoints']):
usage_data['blocked_requests_ip_reputation_lists'] = response[
'Datapoints'][0]['Sum']
except Exception as error:
log.debug(
"[send_anonymous_usage_data] Failed to get Num Allowed Requests"
)
log.debug(str(error))
# --------------------------------------------------------------------------------------------------------------
log.info("[send_anonymous_usage_data] Send Data")
# --------------------------------------------------------------------------------------------------------------
response = send_metrics(data=usage_data)
response_code = response.status_code
log.debug('[send_anonymous_usage_data] Response Code: {}'.format(
response_code))
log.debug("[send_anonymous_usage_data] End")
except Exception as error:
log.debug("[send_anonymous_usage_data] Failed to send data")