def handle_ec2_terminate_event(instance_id):
"""
Purpose: Handler for EC2 terminate event
Parameters: Instance Id
Returns:
Raises:
"""
utl.put_line_in_log('EC2 Terminate Handler', 'thin')
logger.info("Received EC2 termination notification for instance-id: " +
str(instance_id))
# SNS class initialization
sns = SimpleNotificationService()
if instance_id is not None: # Since Instance termination initiated, delete entries
logger.info("Instance termination has been initiated: " + instance_id)
logger.info("Initiating vm_delete function via SNS")
if not const.DISABLE_VM_DELETE_FUNC:
message_subject = 'Event: ' + e_var[
'AutoScaleGrpName'] + ' ' + 'instance delete' + ' ' + instance_id
msg_body = utl.sns_msg_body_configure_ftdv_topic(
'vm_delete', 'FIRST', instance_id)
sns.publish_to_topic(e_var['AutoScaleManagerTopic'],
message_subject, msg_body)
if e_var['USER_NOTIFY_TOPIC_ARN'] is not None:
# Email to user
details_of_the_device = None
message_subject = 'Event: ' + e_var['AutoScaleGrpName'] + ' ' + instance_id + ' ' + \
'instance is terminated'
msg_body = utl.sns_msg_body_user_notify_topic(
'VM Terminated', e_var['AutoScaleGrpName'], instance_id,
details_of_the_device)
sns.publish_to_topic(e_var['USER_NOTIFY_TOPIC_ARN'],
message_subject, msg_body)
# -------------
utl.put_line_in_log('EC2 Terminate Handler finished', 'thin')
return
def handle_ec2_terminate_event(instance_id):
"""
Purpose: Handler for EC2 terminate event
Parameters: Instance Id
Returns:
Raises:
"""
utl.put_line_in_log('EC2 Terminate Handler', 'thin')
logger.info("Received EC2 termination notification for instance-id: " +
str(instance_id))
# SNS class initialization
sns = SimpleNotificationService()
if instance_id is not None: # Since Instance termination initiated, delete entries
logger.info("Instance termination has been initiated: " + instance_id)
logger.info("Initiating vm_delete function via SNS")
message_subject = 'EVENT: ' + utl.e_var[
'AutoScaleGrpName'] + ' ' + 'instance delete' + ' ' + instance_id
sns.publish_to_topic(utl.e_var['AutoScaleManagerTopic'],
'VM not accessible', message_subject, 'vm_delete',
'FIRST', instance_id)
utl.put_line_in_log('EC2 Terminate Handler finished', 'thin')
return
def handle_ec2_launch_event(instance_id):
"""
Purpose: Handler for EC2 launch event
Parameters: Instance Id
Returns:
Raises:
"""
utl.put_line_in_log('EC2 Launch Handler', 'thin')
if instance_id is not None:
logger.info("Received EC2 launch notification for instance-id: " +
str(instance_id))
# SNS class initialization
sns = SimpleNotificationService()
# FTD class initialization
instance = NgfwInstance(instance_id)
instance_state = instance.get_instance_state()
interfaces_ip = instance.get_instance_interfaces_ip()
if interfaces_ip is None:
logger.warn("Unable to get IPs of the instance" + instance_id)
message_subject = 'EVENT: ' + utl.e_var[
'AutoScaleGrpName'] + ' ' + 'instance poll' + ' ' + instance_id
sns.publish_to_topic(utl.e_var['AutoScaleManagerTopic'],
'Check VM is ready', message_subject,
'vm_ready', 'FIRST', instance_id)
utl.put_line_in_log('EC2 Launch Handler finished', 'thin')
return
if instance_state == 'running' or instance_state == 'pending':
logger.info("Instance %s is in state: %s" %
(instance_id, instance_state))
message_subject = 'EVENT: ' + utl.e_var[
'AutoScaleGrpName'] + ' ' + 'instance poll' + ' ' + instance_id
sns.publish_to_topic(utl.e_var['AutoScaleManagerTopic'],
'Check VM is ready', message_subject,
'vm_ready', 'FIRST', instance_id)
else:
logger.warn("Instance %s is in state: %s" %
(instance_id, instance_state))
else:
logger.critical("Received instance_id None!")
utl.put_line_in_log('EC2 Launch Handler finished', 'thin')
return
def handle_sns_event(sns_data):
"""
Purpose: Handler for SNS event
Parameters: SNS data from Lambda handler
Returns:
Raises:
"""
utl.put_line_in_log('SNS Handler', 'thin')
logger.debug("SNS Message: " + json.dumps(sns_data, separators=(',', ':')))
# SNS class initialization
sns = SimpleNotificationService()
sns_msg_attr = json.loads(sns_data['Message'])
logger.info("SNS Message: " +
json.dumps(sns_msg_attr, separators=(',', ':')))
if sns_msg_attr is None:
logger.critical("Unable to get required attributes from SNS message!")
utl.put_line_in_log('SNS Handler Finished', 'thin')
return
try:
if sns_msg_attr['instance_id'] is None:
logger.critical("Received instance_id None!")
utl.put_line_in_log('SNS Handler Finished', 'thin')
return
if int(sns_msg_attr['counter']
) <= 0 and sns_msg_attr['to_function'] != 'vm_delete':
message_subject = 'EVENT: ' + utl.e_var['AutoScaleGrpName'] + ' ' + 'instance delete' + ' ' + \
sns_msg_attr['instance_id']
sns.publish_to_topic(utl.e_var['AutoScaleManagerTopic'],
'VM not accessible', message_subject,
'vm_delete', 'FIRST',
sns_msg_attr['instance_id'])
logger.critical("Has ran out of retries! calling vm_delete...")
utl.put_line_in_log('SNS Handler Finished', 'thin')
return
elif int(sns_msg_attr['counter']
) <= 0 and sns_msg_attr['to_function'] == 'vm_delete':
logger.critical("Unable to delete device %s" %
sns_msg_attr['instance_id'])
utl.put_line_in_log('SNS Handler Finished', 'thin')
return
except KeyError as e:
logger.error(
"Unable to get one of required parameter from SNS Message body: {}"
.format(repr(e)))
utl.put_line_in_log('SNS Handler Finished', 'thin')
return
# FMC class initialization
fmc = FirepowerManagementCenter()
fmc.get_auth_token()
# FTD class initialization
ftd = NgfwInstance(sns_msg_attr['instance_id'])
instance_state = ftd.get_instance_state()
logger.info("Instance %s " % sns_msg_attr['instance_id'] +
"is in %s state" % instance_state)
if sns_msg_attr['to_function'] == 'vm_delete':
pass
elif instance_state == 'running' or instance_state == 'pending':
pass
else:
logger.error("Device in %s state, can't be handled by %s function" %
(instance_state, sns_msg_attr['to_function']))
utl.put_line_in_log('SNS Handler Finished', 'thin')
return
logger.info("Continue to execute action of " + sns_msg_attr['to_function'])
if sns_msg_attr['to_function'] == 'vm_ready':
ftd.put_instance_name()
if sns_msg_attr['category'] == 'FIRST':
if execute_vm_ready_first(ftd) == 'SUCCESS':
logger.info(
"SSH to NGFWv with instance_id is successful, Next action: Registration"
)
message_subject = 'EVENT: ' + utl.e_var['AutoScaleGrpName'] + ' ' + 'instance register' + ' ' + \
sns_msg_attr['instance_id']
sns.publish_to_topic(utl.e_var['AutoScaleManagerTopic'],
'VM is ready', message_subject,
'vm_register', 'FIRST',
sns_msg_attr['instance_id'])
else:
logger.warn(
"SSH to NGFWv with instance_id: %s is un-successful, Retrying..."
% sns_msg_attr['instance_id'])
message_subject = 'EVENT: ' + utl.e_var['AutoScaleGrpName'] + ' ' + 'instance poll' + ' ' + \
sns_msg_attr['instance_id']
sns.publish_to_topic(utl.e_var['AutoScaleManagerTopic'],
'Check VM is ready', message_subject,
'vm_ready', 'FIRST',
sns_msg_attr['instance_id'],
str(int(sns_msg_attr['counter']) - 1))
elif sns_msg_attr['to_function'] == 'vm_register':
if sns_msg_attr['category'] == 'FIRST':
if execute_vm_register_first(ftd, fmc) == 'SUCCESS':
logger.info(
"Instance is registered to FMC, Next action: Configuration"
)
message_subject = 'EVENT: ' + utl.e_var['AutoScaleGrpName'] + ' ' + 'instance configure' + ' ' + \
sns_msg_attr['instance_id']
sns.publish_to_topic(utl.e_var['AutoScaleManagerTopic'],
'VM registered', message_subject,
'vm_configure', 'FIRST',
sns_msg_attr['instance_id'])
else:
logger.warn(
"Registration failed! trying again in next cycle...")
message_subject = 'EVENT: ' + utl.e_var['AutoScaleGrpName'] + ' ' + 'instance register' + ' ' + \
sns_msg_attr['instance_id']
sns.publish_to_topic(utl.e_var['AutoScaleManagerTopic'],
'VM not registered', message_subject,
'vm_register', 'FIRST',
sns_msg_attr['instance_id'],
str(int(sns_msg_attr['counter']) - 1))
elif sns_msg_attr['to_function'] == 'vm_configure':
if sns_msg_attr['category'] == 'FIRST':
if execute_vm_configure_first(ftd, fmc) == 'SUCCESS':
logger.info(
"Instance is configured in FMC, Next action: Deployment")
message_subject = 'EVENT: ' + utl.e_var['AutoScaleGrpName'] + ' ' + 'instance deploy' + ' ' + \
sns_msg_attr['instance_id']
sns.publish_to_topic(utl.e_var['AutoScaleManagerTopic'],
'VM is configured', message_subject,
'vm_deploy', 'FIRST',
sns_msg_attr['instance_id'])
else:
logger.warn(
"Configuration failed! trying again in next cycle...")
message_subject = 'EVENT: ' + utl.e_var['AutoScaleGrpName'] + ' ' + 'instance configure' + ' ' + \
sns_msg_attr['instance_id']
sns.publish_to_topic(utl.e_var['AutoScaleManagerTopic'],
'VM not configured', message_subject,
'vm_configure', 'FIRST',
sns_msg_attr['instance_id'],
str(int(sns_msg_attr['counter']) - 1))
elif sns_msg_attr['to_function'] == 'vm_deploy':
if sns_msg_attr['category'] == 'FIRST':
if execute_vm_deploy_first(ftd, fmc) == 'SUCCESS':
logger.info(
"Configuration is deployed, health status in TG needs to be checked"
)
else:
logger.warn("Deployment failed! trying again in next cycle...")
message_subject = 'EVENT: ' + utl.e_var['AutoScaleGrpName'] + ' ' + 'instance deploy' + ' ' + \
sns_msg_attr['instance_id']
sns.publish_to_topic(utl.e_var['AutoScaleManagerTopic'],
'VM not deployed', message_subject,
'vm_deploy', 'FIRST',
sns_msg_attr['instance_id'],
str(int(sns_msg_attr['counter']) - 1))
elif sns_msg_attr['to_function'] == 'vm_delete':
if sns_msg_attr['category'] == 'FIRST':
if execute_vm_delete_first(ftd, fmc) == 'SUCCESS':
logger.info("Instance has been deleted! ")
else:
logger.critical("Unable to delete instance!")
message_subject = 'EVENT: ' + utl.e_var['AutoScaleGrpName'] + ' ' + 'instance delete' + ' ' + \
sns_msg_attr['instance_id']
sns.publish_to_topic(utl.e_var['AutoScaleManagerTopic'],
'VM not deleted from ASG',
message_subject, 'vm_delete', 'FIRST',
sns_msg_attr['instance_id'],
str(int(sns_msg_attr['counter']) - 1))
utl.put_line_in_log('SNS Handler Finished', 'thin')
return
def lambda_handler(event, context):
"""
Purpose: Main Lambda functions of Autoscale Manager
Parameters: AWS Events (cloudwatch, SNS)
Returns:
Raises:
"""
utl.put_line_in_log('AutoScale Manager Lambda Handler started', 'thick')
logger.info("Received event: " + json.dumps(event, separators=(',', ':')))
# SNS Event
try:
if event["Records"][0]["EventSource"] == "aws:sns":
sns_data = event["Records"][0]["Sns"]
handle_sns_event(sns_data)
utl.put_line_in_log('AutoScale Manager Lambda Handler finished',
'thick')
return
except Exception as e:
logger.info("Received an event but not a SNS notification event")
logger.debug(str(e))
pass
# EC2 CloudWatch Event
try:
if event["detail-type"] == "EC2 Instance Launch Successful":
try:
instance_id = event['detail']['EC2InstanceId']
handle_ec2_launch_event(instance_id)
utl.put_line_in_log(
'AutoScale Manager Lambda Handler finished', 'thick')
return
except Exception as e:
logger.error("Unable to get instance ID from event!")
logger.error("Error occurred {}".format(repr(e)))
utl.put_line_in_log(
'AutoScale Manager Lambda Handler finished', 'thick')
return
elif event["detail-type"] == "EC2 Instance Terminate Successful":
try:
instance_id = event['detail']['EC2InstanceId']
handle_ec2_terminate_event(instance_id)
utl.put_line_in_log(
'AutoScale Manager Lambda Handler finished', 'thick')
return
except Exception as e:
logger.error("Unable to get instance ID from event!")
logger.error("Error occurred {}".format(repr(e)))
utl.put_line_in_log(
'AutoScale Manager Lambda Handler finished', 'thick')
return
except Exception as e:
logger.info("Received an event but not an EC2 CloudWatch event")
logger.debug(str(e))
pass
utl.put_line_in_log('AutoScale Manager Lambda Handler finished', 'thick')
return
def handle_cron_event(event):
"""
Purpose: To handle cron CloudWatch Event
Parameters: event
Returns:
Raises:
"""
# utl.put_line_in_log('Cron Handler Started', 'thin')
# AutoScale Group class initialization
asg = AutoScaleGroup(user_input['AutoScaleGrpName'])
instances_list = asg.get_instance_list()
if len(instances_list) <= 0:
logger.info("No instance found in AWS AutoScale Group")
logger.info(
"Will DISABLE Cron job for Custom Metric Collection, gets ENABLED if new instance launches"
)
# Initialize CloudWatchEvent class
cw_event = CloudWatchEvent(user_input['cron_event_name'])
cw_event.stop_cron_job()
utl.put_line_in_log('Cron Handler Finished', 'thin')
return
# Update aws instance list as vm_name list
append_str = user_input['AutoScaleGrpName'] + '-'
aws_instance_name_list = [append_str + suf for suf in instances_list]
# FMC class initialization
fmc = FirepowerManagementCenter(user_input['FmcServer'],
user_input['FmcMetUserName'],
user_input['FmcMetPassword'])
try:
fmc.get_auth_token()
device_grp_id = fmc.get_device_grp_id_by_name(
user_input['fmcDeviceGroupName'])
if device_grp_id is None:
raise ValueError("Unable to find Device Group in FMC: %s " %
user_input['fmcDeviceGroupName'])
else:
logger.debug("FMC Device group ID: %s " % device_grp_id)
except Exception as e:
logger.exception("Exception {}".format(e))
logger.info("Will DISABLE Cron job for Custom Metric Collection"
"check if FMC is accessible & has mentioned device group")
# Decided to not to disable Publisher if FMC is unreachable
# Initialize CloudWatchEvent class
# cw_event = CloudWatchEvent(user_input['cron_event_name'])
# # cw_event.stop_cron_job()
else:
fmc_devices_list, device_id_list = fmc.get_member_list_in_device_grp(
device_grp_id)
query_device_dict = dict(zip(fmc_devices_list, device_id_list))
intersection_list = utl.intersection(aws_instance_name_list,
fmc_devices_list)
pair_of_metric_name_value = []
metric_name_value = {
"unit": const.DEVICE_NO_UNIT,
"metric_name": const.NO_DEV_IN_FMC_NOT_IN_AWS,
"value": len(fmc_devices_list) - len(aws_instance_name_list)
}
pair_of_metric_name_value.append(metric_name_value)
metric_name_value = {
"unit": const.DEVICE_NO_UNIT,
"metric_name": const.NO_DEV_IN_AWS_NOT_IN_FMC,
"value": len(aws_instance_name_list) - len(fmc_devices_list)
}
pair_of_metric_name_value.append(metric_name_value)
metric_name_value = {
"unit": const.DEVICE_NO_UNIT,
"metric_name": const.NO_DEV_IN_BOTH_FMC_AWS,
"value": len(intersection_list)
}
pair_of_metric_name_value.append(metric_name_value)
# Update list with memory metrics
pair_of_metric_name_value, ftdv_memory_metric_dict = get_memory_metric_pair(
fmc, pair_of_metric_name_value, intersection_list,
query_device_dict)
# Publish Metrics to CloudWatch
update_cloudwatch_metric(pair_of_metric_name_value)
logger.info("List of instance name in AWS AutoScale Group: {}".format(
aws_instance_name_list))
logger.info(
"List of members in FMC device group: {}".format(fmc_devices_list))
logger.info("Memory Metric per FTDv: " +
json.dumps(ftdv_memory_metric_dict, separators=(',',
': ')))
logger.info(
"Metrics published: " +
json.dumps(pair_of_metric_name_value, separators=(',', ': ')))
utl.put_line_in_log('Cron Handler Finished', 'thin')
return
def execute_instance_tg_health_doctor():
"""
Purpose: To remove un-healthy instances from TG if satisfies conditions
Parameters:
Returns: SUCCESS, FAIL
Raises:
"""
# Initializing ElasticLoadBalancer
elb_client = ElasticLoadBalancer()
# Initialize EC2Instance with None
ec2_client = EC2Instance('', e_var['AutoScaleGrpName'])
utl.put_line_in_log('Instance Doctor', 'dot')
asg_name = ''
now = datetime.now(timezone.utc)
killable_ftd_instance = []
unhealthy_ip_targets = []
try:
_ip_targets = elb_client.get_unhealthy_ip_targets(
e_var['LB_ARN_OUTSIDE'])
for i in _ip_targets:
unhealthy_ip_targets.append(i)
except Exception as e:
logger.debug("Exception occurred: {}".format(repr(e)))
logger.info("Unable to get unhealthy IP targets!")
return killable_ftd_instance, unhealthy_ip_targets
try:
logger.info("IPs: " + str(unhealthy_ip_targets) + " found unhealthy!")
list_len = len(unhealthy_ip_targets)
if list_len > 0:
for i in range(0, list_len):
try:
unhealthy_instance = ec2_client.get_describe_instance_from_private_ip(
unhealthy_ip_targets[i])
instance = unhealthy_instance['Reservations'][0][
'Instances'][0]
unhealthy_instance_id = instance['InstanceId']
except IndexError as e:
logger.debug("{}".format(repr(e)))
logger.info("Removing IP: " +
str(unhealthy_ip_targets[i]) +
" as no associated Instance found!")
elb_client.deregister_ip_target_from_lb(
e_var['LB_ARN_OUTSIDE'], unhealthy_ip_targets[i])
utl.put_line_in_log('Instance Doctor finished', 'dot')
return killable_ftd_instance, unhealthy_ip_targets
for val in instance['Tags']:
if val['Key'] == "aws:autoscaling:groupName":
asg_name = str(val['Value'])
if asg_name == e_var['AutoScaleGrpName']:
days = (now - instance['LaunchTime']).days
hours = (now - instance['LaunchTime']).seconds / 60 / 60
logger.info(
'%s, %s, %d days %d hours alive' %
(unhealthy_instance_id,
instance['LaunchTime'].strftime('%Y-%m-%d %H:%M:%S'),
days, hours))
if days > const.UNHEALTHY_DAYS_THRESHOLD or hours > const.UNHEALTHY_HOURS_THRESHOLD:
killable_ftd_instance.append(unhealthy_instance_id)
else:
logger.info(unhealthy_instance_id + " is not part of " +
str(e_var['AutoScaleGrpName']))
logger.info("Removing IP: " +
str(unhealthy_ip_targets[i]) +
" as it is not of an NGFWv VM!")
elb_client.deregister_ip_target_from_lb(
e_var['LB_ARN_OUTSIDE'], unhealthy_ip_targets[i])
utl.put_line_in_log('Instance Doctor finished', 'dot')
return killable_ftd_instance, unhealthy_ip_targets
except Exception as e:
logger.debug("{}".format(repr(e)))
logger.info("Unable to get unhealthy Instances from IPs")
utl.put_line_in_log('Instance Doctor finished', 'dot')
return killable_ftd_instance, unhealthy_ip_targets
try:
logger.info("NGFWv instances: " + str(killable_ftd_instance) +
" found unhealthy for more than threshold!")
list_len = len(killable_ftd_instance)
if list_len > 0:
ec2_group = AutoScaleGroup(e_var['AutoScaleGrpName'])
for i in range(0, list_len):
response = ec2_group.remove_instance(
killable_ftd_instance[i],
const.DECREMENT_CAP_IF_VM_REMOVED_BY_DOCTOR)
if response is not None:
logger.info("Removing instance response: " + str(response))
else:
logger.info("Unable to kill instance: " +
str(killable_ftd_instance[i]))
except Exception as e:
logger.exception(e)
logger.info("Unable to terminate unhealthy Instances")
utl.put_line_in_log('Instance Doctor finished', 'dot')
return killable_ftd_instance, unhealthy_ip_targets
utl.put_line_in_log('Instance Doctor finished', 'dot')
return killable_ftd_instance, unhealthy_ip_targets
def lambda_handler(event, context):
"""
Purpose: Main Lambda functions of Autoscale Manager
Parameters: AWS Events (cloudwatch, SNS)
Returns:
Raises:
"""
utl.put_line_in_log('AutoScale Manager Lambda Handler started', 'thick')
logger.info("Received event: " + json.dumps(event, separators=(',', ':')))
if const.DISABLE_AUTOSCALE_MANAGER_LAMBDA is True:
logger.info(
"Autoscale manager Lambda running is disabled! Check constant.py")
utl.put_line_in_log('Autoscale manager Lambda Handler finished',
'thick')
return
# SNS Event
try:
if event["Records"][0]["EventSource"] == "aws:sns":
sns_data = event["Records"][0]["Sns"]
except Exception as e:
logger.debug(str(e))
pass
else:
try:
handle_sns_event(sns_data)
utl.put_line_in_log('AutoScale Manager Lambda Handler finished',
'thick')
return
except Exception as e:
logger.exception(e)
utl.put_line_in_log('AutoScale Manager Lambda Handler finished',
'thick')
return
logger.info("Received an event but not a SNS notification event")
# EC2 CloudWatch Event
try:
if event["detail-type"] == "Scheduled Event":
handle_cron_event(event)
utl.put_line_in_log('AutoScale Manager Lambda Handler finished',
'thick')
return
except Exception as e:
logger.debug(str(e))
pass
logger.info("Received an event but not a Scheduled Event")
try:
if event["detail-type"] == "EC2 Instance Launch Successful":
try:
instance_id = event['detail']['EC2InstanceId']
except Exception as e:
logger.error("Unable to get instance ID from event!")
logger.error("Error occurred {}".format(repr(e)))
utl.put_line_in_log(
'AutoScale Manager Lambda Handler finished', 'thick')
return
else:
try:
handle_ec2_launch_event(instance_id)
utl.put_line_in_log(
'AutoScale Manager Lambda Handler finished', 'thick')
return
except Exception as e:
logger.exception(e)
utl.put_line_in_log(
'AutoScale Manager Lambda Handler finished', 'thick')
return
elif event["detail-type"] == "EC2 Instance Terminate Successful":
try:
instance_id = event['detail']['EC2InstanceId']
except Exception as e:
logger.error("Unable to get instance ID from event!")
logger.error("Error occurred {}".format(repr(e)))
utl.put_line_in_log(
'AutoScale Manager Lambda Handler finished', 'thick')
return
else:
try:
handle_ec2_terminate_event(instance_id)
utl.put_line_in_log(
'AutoScale Manager Lambda Handler finished', 'thick')
return
except Exception as e:
logger.exception(e)
utl.put_line_in_log(
'AutoScale Manager Lambda Handler finished', 'thick')
return
except Exception as e:
logger.debug(str(e))
pass
logger.info("Received an event but not an EC2 CloudWatch event")
# When its not any expected event / run cron event
# Initialize DerivedFMC & AutoScaleGroup
handle_cron_event(event)
utl.put_line_in_log('AutoScale Manager Lambda Handler finished', 'thick')
return
def handle_ec2_launch_event(instance_id):
"""
Purpose: Handler for EC2 launch event
Parameters: Instance Id
Returns:
Raises:
"""
utl.put_line_in_log('EC2 Launch Handler', 'thin')
# SNS class initialization
sns = SimpleNotificationService()
if instance_id is not None:
logger.info("Received EC2 launch notification for instance-id: " +
str(instance_id))
# Enable Health Doctor
cw_event = CloudWatchEvent(e_var['A_CRON_JOB_NAME'])
status = cw_event.cron_job_status()
if status == 'DISABLED':
cw_event.start_cron_job()
logger.info("ENABLED CloudWatch Rule: " + cw_event.name)
else:
logger.info("CloudWatch Rule: " + cw_event.name +
" is already ENABLED")
# Initialize DerivedFMC & AutoScaleGroup
fmc = fmc_cls_init()
# FTD class initialization
instance = ftd_cls_init(instance_id, fmc)
instance_state = instance.get_instance_state()
interfaces_ip = instance.get_instance_interfaces_ip()
if interfaces_ip is None:
logger.warn("Unable to get IPs of the instance" + instance_id)
message_subject = 'Event: ' + e_var[
'AutoScaleGrpName'] + ' ' + 'instance poll' + ' ' + instance_id
msg_body = utl.sns_msg_body_configure_ftdv_topic(
'vm_ready', 'FIRST', instance_id)
sns.publish_to_topic(e_var['AutoScaleManagerTopic'],
message_subject, msg_body)
utl.put_line_in_log('EC2 Launch Handler finished', 'thin')
return
if instance_state == 'running' or instance_state == 'pending':
logger.info("Instance %s is in state: %s" %
(instance_id, instance_state))
message_subject = 'Event: ' + e_var[
'AutoScaleGrpName'] + ' ' + 'instance poll' + ' ' + instance_id
msg_body = utl.sns_msg_body_configure_ftdv_topic(
'vm_ready', 'FIRST', instance_id)
sns.publish_to_topic(e_var['AutoScaleManagerTopic'],
message_subject, msg_body)
else:
logger.warn("Instance %s is in state: %s" %
(instance_id, instance_state))
if e_var['USER_NOTIFY_TOPIC_ARN'] is not None:
# Email to user
details_of_the_device = None
message_subject = 'Event: ' + e_var['AutoScaleGrpName'] + ' ' + instance_id + ' ' + \
'instance is launched'
msg_body = utl.sns_msg_body_user_notify_topic(
'VM Launched', e_var['AutoScaleGrpName'], instance_id,
details_of_the_device)
sns.publish_to_topic(e_var['USER_NOTIFY_TOPIC_ARN'],
message_subject, msg_body)
# -------------
else:
logger.critical("Received instance_id None!")
utl.put_line_in_log('EC2 Launch Handler finished', 'thin')
return
def handle_sns_event(sns_data):
"""
Purpose: Handler for SNS event
Parameters: SNS data from Lambda handler
Returns:
Raises:
"""
utl.put_line_in_log('SNS Handler', 'thin')
logger.debug("SNS Message: " + json.dumps(sns_data, separators=(',', ':')))
# SNS class initialization
sns = SimpleNotificationService()
_m_attr = json.loads(sns_data['Message'])
logger.info("SNS Message: " + json.dumps(_m_attr, separators=(',', ':')))
if _m_attr is None:
logger.critical("Unable to get required attributes from SNS message!")
utl.put_line_in_log('SNS Handler Finished', 'thin')
return
if _m_attr['instance_id'] is None:
logger.critical("Received instance_id None!")
utl.put_line_in_log('SNS Handler Finished', 'thin')
return
# AutoScaleGroup
aws_grp = aws_asg_cls_init()
# Initialize DerivedFMC
fmc = fmc_cls_init()
# FTD class initialization
ftd = ftd_cls_init(_m_attr['instance_id'], fmc)
try:
if int(_m_attr['counter']
) <= 0 and _m_attr['to_function'] != 'vm_delete':
logger.critical("Lambda has ran out of retries, calling vm_delete")
# Email to user
details_of_the_device = json.dumps(ftd.get_instance_tags())
logger.info(details_of_the_device)
if e_var['USER_NOTIFY_TOPIC_ARN'] is not None:
message_subject = 'Event: ' + e_var['AutoScaleGrpName'] + ' ' + \
_m_attr['instance_id'] + ' ' + 'unable to complete ' + \
_m_attr['to_function']
msg_body = utl.sns_msg_body_user_notify_topic(
'VM not completing ' + _m_attr['to_function'],
e_var['AutoScaleGrpName'], _m_attr['instance_id'],
details_of_the_device)
sns.publish_to_topic(e_var['USER_NOTIFY_TOPIC_ARN'],
message_subject, msg_body)
# -------------
if not const.DISABLE_VM_DELETE_FUNC:
message_subject = 'Event: ' + e_var['AutoScaleGrpName'] + ' ' + 'instance delete' + ' ' + \
_m_attr['instance_id']
msg_body = utl.sns_msg_body_configure_ftdv_topic(
'vm_delete', 'FIRST', _m_attr['instance_id'])
sns.publish_to_topic(e_var['AutoScaleManagerTopic'],
message_subject, msg_body)
else:
logger.info(
" vm_delete function is disabled! Check constant.py")
utl.put_line_in_log('SNS Handler Finished', 'thin')
return
elif int(_m_attr['counter']
) <= 0 and _m_attr['to_function'] == 'vm_delete':
logger.critical("Unable to delete device %s" %
_m_attr['instance_id'])
# Email to user
details_of_the_device = json.dumps(ftd.get_instance_tags())
logger.info(details_of_the_device)
if e_var['USER_NOTIFY_TOPIC_ARN'] is not None:
message_subject = 'Event: ' + e_var['AutoScaleGrpName'] + ' ' + \
_m_attr['instance_id'] + ' ' + 'instance not deleted'
msg_body = utl.sns_msg_body_user_notify_topic(
'VM not getting deleted', e_var['AutoScaleGrpName'],
_m_attr['instance_id'], details_of_the_device)
sns.publish_to_topic(e_var['USER_NOTIFY_TOPIC_ARN'],
message_subject, msg_body)
# -------------
utl.put_line_in_log('SNS Handler Finished', 'thin')
return
except KeyError as e:
logger.error(
"Unable to get one of required parameter from SNS Message body: {}"
.format(repr(e)))
utl.put_line_in_log('SNS Handler Finished', 'thin')
return
if (_m_attr['to_function'] == 'vm_ready' and int(_m_attr['counter']) == const.TO_FUN_RETRY_COUNT[0]) or \
(_m_attr['to_function'] == 'vm_register' and int(_m_attr['counter']) == const.TO_FUN_RETRY_COUNT[1]) or \
(_m_attr['to_function'] == 'vm_configure' and int(_m_attr['counter']) == const.TO_FUN_RETRY_COUNT[2]) or \
(_m_attr['to_function'] == 'vm_deploy' and int(_m_attr['counter']) == const.TO_FUN_RETRY_COUNT[3]):
logger.info("Fmc validation: " +
fmc_configuration_validation(fmc, aws_grp))
if fmc.configuration_status == 'UN-CONFIGURED':
# Email to user
details_of_the_device = json.dumps(ftd.get_instance_tags())
logger.info(details_of_the_device)
if e_var['USER_NOTIFY_TOPIC_ARN'] is not None:
message_subject = 'Event: ' + e_var['AutoScaleGrpName'] + ' ' + \
_m_attr['instance_id'] + ' ' + 'unable to connect Fmc ' + \
_m_attr['to_function']
msg_body = utl.sns_msg_body_user_notify_topic(
"Unable to connect Fmc", e_var['AutoScaleGrpName'],
_m_attr['instance_id'], details_of_the_device)
sns.publish_to_topic(e_var['USER_NOTIFY_TOPIC_ARN'],
message_subject, msg_body)
# -------------
if not const.DISABLE_VM_DELETE_FUNC:
logger.info("Terminating instance")
message_subject = 'Event: ' + e_var['AutoScaleGrpName'] + ' ' + 'instance delete' + ' ' + \
_m_attr['instance_id']
msg_body = utl.sns_msg_body_configure_ftdv_topic(
'vm_delete', 'FIRST', _m_attr['instance_id'])
sns.publish_to_topic(e_var['AutoScaleManagerTopic'],
message_subject, msg_body)
else:
logger.info(
"Need to terminate the instance, but vm_delete function is disabled, check constant.py"
)
return
instance_state = ftd.get_instance_state()
logger.info("Instance %s " % _m_attr['instance_id'] +
"is in %s state" % instance_state)
if _m_attr['to_function'] == 'vm_delete':
pass
elif instance_state == 'running' or instance_state == 'pending':
pass
else:
logger.error("Device in %s state, can't be handled by %s function" %
(instance_state, _m_attr['to_function']))
utl.put_line_in_log('SNS Handler Finished', 'thin')
return
logger.info("Continue to execute action of " + _m_attr['to_function'])
if _m_attr['to_function'] == 'vm_ready':
if const.DISABLE_VM_READY_FUNC is True:
logger.info(_m_attr['to_function'] +
" function is disabled! Check constant.py")
utl.put_line_in_log('SNS Handler Finished', 'thin')
return
ftd.create_instance_tags('Name',
ftd.vm_name) # To put Name tag on instance
if _m_attr['category'] == 'FIRST':
if execute_vm_ready_first(ftd) == 'SUCCESS':
ftd.create_instance_tags('NGFWvConnectionStatus', 'AVAILABLE')
logger.info(
"SSH to NGFWv with instance_id is successful, Next action: Registration"
)
if not const.DISABLE_VM_REGISTER_FUNC:
message_subject = 'Event: ' + e_var['AutoScaleGrpName'] + ' ' + 'instance register' + ' ' + \
_m_attr['instance_id']
msg_body = utl.sns_msg_body_configure_ftdv_topic(
'vm_register', 'FIRST', _m_attr['instance_id'])
sns.publish_to_topic(e_var['AutoScaleManagerTopic'],
message_subject, msg_body)
else:
logger.info(
" vm_register function is disabled! Check constant.py")
else:
logger.warn(
"SSH to NGFWv with instance_id: %s is un-successful, Retrying..."
% _m_attr['instance_id'])
message_subject = 'Event: ' + e_var['AutoScaleGrpName'] + ' ' + 'instance poll' + ' ' + \
_m_attr['instance_id']
msg_body = utl.sns_msg_body_configure_ftdv_topic(
'vm_ready', 'FIRST', _m_attr['instance_id'],
str(int(_m_attr['counter']) - 1))
sns.publish_to_topic(e_var['AutoScaleManagerTopic'],
message_subject, msg_body)
elif _m_attr['to_function'] == 'vm_register':
if _m_attr['category'] == 'FIRST':
if execute_vm_register_first(ftd) == 'SUCCESS':
ftd.create_instance_tags('NGFWvRegistrationStatus', 'DONE')
logger.info(
"Instance is registered to FMC, Next action: Configuration"
)
if not const.DISABLE_VM_CONFIGURE_FUNC:
message_subject = 'Event: ' + e_var['AutoScaleGrpName'] + ' ' + 'instance configure' + ' ' + \
_m_attr['instance_id']
msg_body = utl.sns_msg_body_configure_ftdv_topic(
'vm_configure', 'FIRST', _m_attr['instance_id'])
sns.publish_to_topic(e_var['AutoScaleManagerTopic'],
message_subject, msg_body)
else:
logger.info(
" vm_configure function is disabled! Check constant.py"
)
else:
logger.warn(
"Registration failed! trying again in next cycle...")
message_subject = 'Event: ' + e_var['AutoScaleGrpName'] + ' ' + 'instance register' + ' ' + \
_m_attr['instance_id']
msg_body = utl.sns_msg_body_configure_ftdv_topic(
'vm_register', 'FIRST', _m_attr['instance_id'],
str(int(_m_attr['counter']) - 1))
sns.publish_to_topic(e_var['AutoScaleManagerTopic'],
message_subject, msg_body)
elif _m_attr['to_function'] == 'vm_configure':
if _m_attr['category'] == 'FIRST':
if execute_vm_configure_first(ftd) == 'SUCCESS':
ftd.create_instance_tags('NGFWvConfigurationStatus', 'DONE')
logger.info(
"Instance is configured in FMC, Next action: Deployment")
if not const.DISABLE_VM_DEPLOY_FUNC:
message_subject = 'Event: ' + e_var['AutoScaleGrpName'] + ' ' + 'instance deploy' + ' ' + \
_m_attr['instance_id']
msg_body = utl.sns_msg_body_configure_ftdv_topic(
'vm_deploy', 'FIRST', _m_attr['instance_id'])
sns.publish_to_topic(e_var['AutoScaleManagerTopic'],
message_subject, msg_body)
else:
logger.info(
" vm_deploy function is disabled! Check constant.py")
else:
logger.warn(
"Configuration failed! trying again in next cycle...")
message_subject = 'Event: ' + e_var['AutoScaleGrpName'] + ' ' + 'instance configure' + ' ' + \
_m_attr['instance_id']
msg_body = utl.sns_msg_body_configure_ftdv_topic(
'vm_configure', 'FIRST', _m_attr['instance_id'],
str(int(_m_attr['counter']) - 1))
sns.publish_to_topic(e_var['AutoScaleManagerTopic'],
message_subject, msg_body)
elif _m_attr['to_function'] == 'vm_deploy':
if _m_attr['category'] == 'FIRST':
if execute_vm_deploy_first(ftd, fmc) == 'SUCCESS':
ftd.create_instance_tags('NGFWvConfigDeployStatus', 'DONE')
logger.info(
"Configuration is deployed, health status in TG needs to be checked"
)
details_of_the_device = json.dumps(ftd.get_instance_tags())
logger.info(details_of_the_device)
if e_var['USER_NOTIFY_TOPIC_ARN'] is not None:
# Email to user
message_subject = 'Event: ' + e_var['AutoScaleGrpName'] + ' ' + \
_m_attr['instance_id'] + ' ' + 'instance deploy configuration successful'
msg_body = utl.sns_msg_body_user_notify_topic(
'VM Configuration Deployed', e_var['AutoScaleGrpName'],
_m_attr['instance_id'], details_of_the_device)
sns.publish_to_topic(e_var['USER_NOTIFY_TOPIC_ARN'],
message_subject, msg_body)
# -------------
else:
logger.warn("Deployment failed! trying again in next cycle...")
message_subject = 'Event: ' + e_var['AutoScaleGrpName'] + ' ' + 'instance deploy' + ' ' + \
_m_attr['instance_id']
msg_body = utl.sns_msg_body_configure_ftdv_topic(
'vm_deploy', 'FIRST', _m_attr['instance_id'],
str(int(_m_attr['counter']) - 1))
sns.publish_to_topic(e_var['AutoScaleManagerTopic'],
message_subject, msg_body)
elif _m_attr['to_function'] == 'vm_delete':
if _m_attr['category'] == 'FIRST':
if execute_vm_delete_first(ftd, fmc) == 'SUCCESS':
logger.info("Instance has been deleted")
else:
logger.critical("Unable to delete instance")
message_subject = 'Event: ' + e_var['AutoScaleGrpName'] + ' ' + 'instance delete' + ' ' + \
_m_attr['instance_id']
msg_body = utl.sns_msg_body_configure_ftdv_topic(
'vm_delete', 'FIRST', _m_attr['instance_id'],
str(int(_m_attr['counter']) - 1))
sns.publish_to_topic(e_var['AutoScaleManagerTopic'],
message_subject, msg_body)
utl.put_line_in_log('SNS Handler Finished', 'thin')
return