def call_me_back_send(delay=None):
delay = delay if delay is not None else max(next_call_delay() + 1, 2)
client = ctx["sqs.client"]
log.log(
log.NOTICE,
"Sending SQS 'CallMeBack' message (delay=%d) to Main lambda queue: %s"
% (delay, ctx["MainSQSQueue"]))
if (misc.is_sam_local()):
log.debug("Should have sent a call_me_back()")
return
caller_function = inspect.currentframe().f_back
while caller_function.f_code.co_name in ["record_subsegment", "__call__"
]: # Skip X-Ray wrappers
caller_function = caller_function.f_back
response = client.send_message(QueueUrl=ctx["MainSQSQueue"],
DelaySeconds=int(delay),
MessageBody=json.dumps({
"SQSHeartBeat": {
"Reason":
"NEED_UPDATE",
"CallerFunction":
caller_function.f_code.co_name,
"LambdaFunction":
ctx["FunctionName"]
}
}))
def is_called_too_early():
global ctx
delay = Cfg.get_duration_secs("app.run_period")
delta = sqs.seconds_since_last_call()
if delta != -1 and delta < delay:
if misc.is_sam_local():
log.warning("is_called_too_early disabled because running in SAM!")
return False
log.log(log.NOTICE, "Called too early (now=%s, delay=%s => delta_seconds=%s)..." %
(ctx["now"], delay, delta))
return True
return False
def init(context, with_kvtable=True, with_predefined_configuration=True):
global _init
_init = {}
_init["context"] = context
_init["all_configs"] = [{
"source": "Built-in defaults",
"config": {},
"metas" : {}
}]
if with_kvtable:
_init["configuration_table"] = kvtable.KVTable(context, context["ConfigurationTable"])
_init["configuration_table"].reread_table()
_init["with_kvtable"] = with_kvtable
_init["active_parameter_set"] = None
register({
"config.dump_configuration,Stable" : {
"DefaultValue": "0",
"Format" : "Bool",
"Description" : """Display all relevant configuration parameters in CloudWatch logs.
Used for debugging purpose.
"""
},
"config.loaded_files,Stable" : {
"DefaultValue" : "internal:predefined.config.yaml;internal:custom.config.yaml",
"Format" : "StringList",
"Description" : """A semi-column separated list of URL to load as configuration.
Upon startup, CloneSquad will load the listed files in sequence and stack them allowing override between layers.
The default contains a reference to the empty internal file 'custom.config.yaml'. Users that intend to embed
customization directly inside the Lambda delivery should override this file with their own configuration. See
[Customizing the Lambda package](#customizing-the-lambda-package).
This key is evaluated again after each URL parsing meaning that a layer can redefine the 'config.loaded_files' to load further
YAML files.
"""
},
"config.max_file_hierarchy_depth" : 10,
"config.active_parameter_set,Stable": {
"DefaultValue": "",
"Format" : "String",
"Description" : """Defines the parameter set to activate.
See [Parameter sets](#parameter-sets) documentation.
"""
},
"config.default_ttl": 0
})
# Load extra configuration from specified URLs
xray_recorder.begin_subsegment("config.init:load_files")
files_to_load = get_list("config.loaded_files", default=[]) if with_predefined_configuration else []
if "ConfigurationURL" in context:
files_to_load.extend(context["ConfigurationURL"].split(";"))
if misc.is_sam_local():
# For debugging purpose. Ability to override config when in SAM local
resource_file = "internal:sam.local.config.yaml"
log.info("Reading local resource file %s..." % resource_file)
files_to_load.append(resource_file)
loaded_files = []
i = 0
while i < len(files_to_load):
f = files_to_load[i]
if f == "":
i += 1
continue
try:
c = yaml.safe_load(misc.get_url(f))
if c is None: c = {}
loaded_files.append({
"source": f,
"config": c
})
if "config.loaded_files" in c:
files_to_load.extend(c["config.loaded_files"].split(";"))
if i > get_int("config.max_file_hierarchy_depth"):
log.warning("Too much config file loads (%s)!! Stopping here!" % loaded_files)
except Exception as e:
log.warning("Failed to load and/or parse config file '%s'! (Notice: It will be safely ignored!)" % f)
i += 1
_init["loaded_files"] = loaded_files
xray_recorder.end_subsegment()
builtin_config = _init["all_configs"][0]["config"]
for cfg in _get_config_layers(reverse=True):
c = cfg["config"]
if "config.active_parameter_set" in c:
if c == builtin_config and isinstance(c, dict):
_init["active_parameter_set"] = c["config.active_parameter_set"]["DefaultValue"]
else:
_init["active_parameter_set"] = c["config.active_parameter_set"]
break
_parameterset_sanity_check()
register({
"config.ignored_warning_keys,Stable" : {
"DefaultValue": "",
"Format" : "StringList",
"Description" : """A list of config keys that are generating a warning on usage, to disable them.
Typical usage is to avoid the 'WARNING' Cloudwatch Alarm to trigger when using a non-Stable configuration key.
Ex: ec2.schedule.key1;ec2.schedule.key2
Remember that using non-stable configuration keys, is creating risk as semantic and/or existence could change
from CloneSquad version to version!
"""
}
})
_init["ignored_warning_keys"] = get_list_of_dict("config.ignored_warning_keys")
def main_handler_entrypoint(event, context):
"""
Parameters
----------
event: dict, required
context: object, required
Lambda Context runtime methods and attributes
Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
Returns
------
"""
#print(Dbg.pprint(event))
ctx["now"] = misc.utc_now()
ctx["FunctionName"] = "Main"
init()
if Cfg.get_int("app.disable") != 0 and not misc.is_sam_local():
log.warning("Application disabled due to 'app.disable' key")
return
no_is_called_too_early = False
# Manage Spot interruption as fast as we can
if sqs.process_sqs_records(event, function=ec2_schedule.manage_spot_notification, function_arg=ctx):
log.info("Managed Spot Interruption SQS record!")
# Force to run now disregarding `app.run_period` as we have at least one Spot instance to
# remove from target groups immediatly
no_is_called_too_early = True
# Check that we are not called too early
# Note: We peform a direct read to the KVTable to spare initialization time when the
# Lambda is called too early
ctx["main.last_call_date"] = ctx["o_ec2"].get_state("main.last_call_date", direct=True)
if ctx["main.last_call_date"] is None or ctx["main.last_call_date"] == "":
ctx["main.last_call_date"] = str(misc.epoch())
if not no_is_called_too_early and is_called_too_early():
log.log(log.NOTICE, "Called too early by: %s" % event)
notify.do_not_notify = True
sqs.process_sqs_records(event)
sqs.call_me_back_send()
return
log.debug("Load prerequisites.")
load_prerequisites(["o_state", "o_notify", "o_ec2", "o_cloudwatch", "o_targetgroup", "o_ec2_schedule", "o_scheduler", "o_rds"])
# Remember 'now' as the last execution date
ctx["o_ec2"].set_state("main.last_call_date", value=ctx["now"], TTL=Cfg.get_duration_secs("app.default_ttl"))
Cfg.dump()
# Perform actions:
log.debug("Main processing.")
ctx["o_targetgroup"].manage_targetgroup()
ctx["o_ec2_schedule"].schedule_instances()
ctx["o_ec2_schedule"].stop_drained_instances()
ctx["o_cloudwatch"].configure_alarms()
ctx["o_rds"].manage_subfleet_rds()
ctx["o_ec2_schedule"].prepare_metrics()
ctx["o_cloudwatch"].send_metrics()
ctx["o_cloudwatch"].configure_dashboard()
# If we got woke up by SNS, acknowledge the message(s) now
sqs.process_sqs_records(event)
ctx["o_notify"].notify_user_arn_resources()
# Call me back if needed
sqs.call_me_back_send()
ctx["StateTable"] = "CloneSquad-%s%s-State" % (ctx["GroupName"], ctx["VariantNumber"])
ctx["EventTable"] = "CloneSquad-%s%s-EventLog" % (ctx["GroupName"], ctx["VariantNumber"])
ctx["LongTermEventTable"] = "CloneSquad-%s%s-EventLog-LongTerm" % (ctx["GroupName"], ctx["VariantNumber"])
ctx["SchedulerTable"] = "CloneSquad-%s%s-Scheduler" % (ctx["GroupName"], ctx["VariantNumber"])
ctx["MainSQSQueue"] = "https://sqs.%s.amazonaws.com/%s/CloneSquad-Main-%s" % (ctx["AWS_DEFAULT_REGION"], account_id, ctx["GroupName"])
ctx["InteractSQSUrl"] = "https://sqs.%s.amazonaws.com/%s/CloneSquad-Interact-%s" % (ctx["AWS_DEFAULT_REGION"], account_id, ctx["GroupName"])
ctx["CloudWatchEventRoleArn"] = "arn:aws:iam::%s:role/CloneSquad-%s-CWRole" % (account_id, ctx["GroupName"])
ctx["GenericInsufficientDataActions_SNSTopicArn"] = "arn:aws:sns:%s:%s:CloneSquad-CloudWatchAlarm-InsufficientData-%s" % (ctx["AWS_DEFAULT_REGION"], account_id, ctx["GroupName"])
ctx["GenericOkActions_SNSTopicArn"] = "arn:aws:sns:%s:%s:CloneSquad-CloudWatchAlarm-Ok-%s" % (ctx["AWS_DEFAULT_REGION"], account_id, ctx["GroupName"])
ctx["ScaleUp_SNSTopicArn"] = "arn:aws:sns:%s:%s:CloneSquad-CloudWatchAlarm-ScaleUp-%s" % (ctx["AWS_DEFAULT_REGION"], account_id, ctx["GroupName"])
ctx["InteractLambdaArn"] = "arn:aws:lambda:%s:%s:function:CloneSquad-Interact-%s" % (ctx["AWS_DEFAULT_REGION"], account_id, ctx["GroupName"])
ctx["AWS_LAMBDA_LOG_GROUP_NAME"] = "/aws/lambda/CloneSquad-Main-%s" % ctx["GroupName"]
# Special treatment while started from SMA invoke loval
if misc.is_sam_local() or __name__ == '__main__':
fix_sam_bugs()
print("SAM Local Environment:")
for env in os.environ:
print("%s=%s" % (env, os.environ[env]))
def initialize_clients(clients, context):
global ctx
log.debug("Initialize clients.")
ctx["cwd"] = os.getcwd()
config = Config(
retries = {
'max_attempts': 5,
'mode': 'standard'
})
for c in clients: