def handle(self, *args, **options):
func_name = self.canary_function_name
prod_alias = LAMBDA.get_alias(FunctionName=func_name, Name="PROD")
self.log("PROD Alias starting at version: %s" %
prod_alias["FunctionVersion"])
canary_alias = LAMBDA.get_alias(FunctionName=func_name, Name="CANARY")
self.log("CANARY Alias starting at version: %s" %
canary_alias["FunctionVersion"])
# Create new version
self.stdout.write("Publishing new version...")
new_version = LAMBDA.publish_version(FunctionName=func_name)
new_version_num = new_version["Version"]
self.log("New version is: %s" % new_version_num)
# This causes the new code to start getting used on canary upload events.
canary_period_start = now()
self.set_canary_version(new_version_num)
if options["bypass_canary"]:
# We promote all aliases and return immediately.
self.log("Bypassing canary stage and promoting immediately")
self.set_prod_version(new_version_num)
self.log("Finished.")
return
if is_processing_disabled():
self.log("Processing is disabled so we will not see any canaries")
self.log("Bypassing canary stage and promoting immediately")
self.set_prod_version(new_version_num)
self.log("Finished.")
return
# If we did not exit already, then we are doing a canary deployment.
max_wait_seconds = options["max_wait_seconds"]
max_wait_time = now() + timedelta(seconds=max_wait_seconds)
min_canary_uploads = options["min_canary_uploads"]
wait_for_more_canary_uploads = True
try:
while wait_for_more_canary_uploads:
canary_uploads = UploadEvent.objects.filter(
canary=True,
created__gte=canary_period_start,
).exclude(status__in=UploadEventStatus.processing_statuses())
if canary_uploads.count() >= min_canary_uploads:
wait_for_more_canary_uploads = False
elif len(self.get_failed_canaries(canary_uploads.all())):
# Exit early since some canaries have already failed
wait_for_more_canary_uploads = False
else:
if now() > max_wait_time:
msg = "Waited too long for canary events. Exiting."
self.log(msg)
raise RuntimeError(msg)
self.log("Found %i uploads... sleeping 5 seconds" %
(canary_uploads.count()))
time.sleep(5)
self.log("%s canary upload events have been found" %
(canary_uploads.count()))
canary_failures = self.get_canary_failures_since(
canary_period_start)
if canary_failures:
# We have canary failures, time to rollback.
self.log("The following canary uploads have failed:")
self.log(", ".join(u.shortid for u in canary_failures))
raise RuntimeError(
"Failed canary events detected. Rolling back.")
except Exception:
# Revert the canary alias back to what PROD still points to
prod_version = prod_alias["FunctionVersion"]
self.log("CANARY will be reverted to version: %s" % prod_version)
self.set_canary_version(prod_version)
self.log("Initiating reprocessing to fix canary uploads")
# Query again for all canary failures after reverting so we don't miss any
canary_failures = self.get_canary_failures_since(
canary_period_start)
queue_upload_events_for_reprocessing(canary_failures,
use_kinesis=True)
self.log("Finished queuing canaries for reprocessing.")
sys.exit(1)
# We didn't have any canary failures so it's time to promote PROD.
self.log("The canary version is a success! Promoting PROD.")
self.set_prod_version(new_version_num)
self.log("Finished.")
def handle(self, *args, **options):
func_name = self.canary_function_name
prod_alias = LAMBDA.get_alias(FunctionName=func_name, Name="PROD")
self.log("PROD Alias starting at version: %s" % prod_alias["FunctionVersion"])
canary_alias = LAMBDA.get_alias(FunctionName=func_name, Name="CANARY")
self.log("CANARY Alias starting at version: %s" % canary_alias["FunctionVersion"])
# Create new version
self.stdout.write("Publishing new version...")
new_version = LAMBDA.publish_version(FunctionName=func_name)
new_version_num = new_version["Version"]
self.log("New version is: %s" % new_version_num)
# This causes the new code to start getting used on canary upload events.
canary_period_start = now()
self.set_canary_version(new_version_num)
if options["bypass_canary"]:
# We promote all aliases and return immediately.
self.log("Bypassing canary stage and promoting immediately")
self.set_prod_version(new_version_num)
self.log("Finished.")
return
if is_processing_disabled():
self.log("Processing is disabled so we will not see any canaries")
self.log("Bypassing canary stage and promoting immediately")
self.set_prod_version(new_version_num)
self.log("Finished.")
return
# If we did not exit already, then we are doing a canary deployment.
max_wait_seconds = options["max_wait_seconds"]
max_wait_time = now() + timedelta(seconds=max_wait_seconds)
min_canary_uploads = options["min_canary_uploads"]
wait_for_more_canary_uploads = True
try:
while wait_for_more_canary_uploads:
canary_uploads = UploadEvent.objects.filter(
canary=True,
created__gte=canary_period_start,
).exclude(status__in=UploadEventStatus.processing_statuses())
if canary_uploads.count() >= min_canary_uploads:
wait_for_more_canary_uploads = False
elif len(self.get_failed_canaries(canary_uploads.all())):
# Exit early since some canaries have already failed
wait_for_more_canary_uploads = False
else:
if now() > max_wait_time:
msg = "Waited too long for canary events. Exiting."
self.log(msg)
raise RuntimeError(msg)
self.log(
"Found %i uploads... sleeping 5 seconds" % (canary_uploads.count(),)
)
time.sleep(5)
self.log(
"%s canary upload events have been found" % canary_uploads.count()
)
canary_failures = self.get_canary_failures_since(canary_period_start)
if canary_failures:
# We have canary failures, time to rollback.
self.log("The following canary uploads have failed:")
self.log(", ".join(u.shortid for u in canary_failures))
raise RuntimeError("Failed canary events detected. Rolling back.")
except Exception:
# Revert the canary alias back to what PROD still points to
prod_version = prod_alias["FunctionVersion"]
self.log("CANARY will be reverted to version: %s" % prod_version)
self.set_canary_version(prod_version)
self.log("Initiating reprocessing to fix canary uploads")
# Query again for all canary failures after reverting so we don't miss any
canary_failures = self.get_canary_failures_since(canary_period_start)
queue_upload_events_for_reprocessing(canary_failures, use_kinesis=True)
self.log("Finished queuing canaries for reprocessing.")
sys.exit(1)
# We didn't have any canary failures so it's time to promote PROD.
self.log("The canary version is a success! Promoting PROD.")
self.set_prod_version(new_version_num)
self.log("Finished.")
