def handle(self, *args, **options): enabled = True if options["enabled"].lower() == "true" else False event_source_list = LAMBDA.list_event_source_mappings(FunctionName=options["lambda"]) self.stdout.write("Stream processing for %r will be set to %r" % (options["lambda"], options["enabled"])) for mapping in event_source_list["EventSourceMappings"]: LAMBDA.update_event_source_mapping(UUID=mapping["UUID"], Enabled=enabled) self.stdout.write("Finished.")
def schedule_lambda_trigger(self, url, payload): from hsreplaynet.utils.aws.clients import LAMBDA final_payload = self._serialize_payload(payload) LAMBDA.invoke( FunctionName="trigger_webhook", InvocationType="Event", # Triggers asynchronous invocation Payload=final_payload, )
def lambda_invoker(payload, shortid): try: LAMBDA.invoke( FunctionName="process_single_replay_upload_stream_handler", InvocationType="RequestResponse", # Triggers synchronous invocation Payload=payload, ) finally: logger.debug("Lambda completed for %s Decrementing latch.", shortid) countdown_latch.count_down()
def lambda_invoker(payload, shortid): try: LAMBDA.invoke( FunctionName="process_single_replay_upload_stream_handler", InvocationType= "RequestResponse", # Triggers synchronous invocation Payload=payload, ) finally: logger.info("Lambda completed for %s Decrementing latch.", shortid) countdown_latch.count_down()
def handle(self, *args, **options): enabled = True if options["enabled"].lower() == "true" else False event_source_list = LAMBDA.list_event_source_mappings( FunctionName=options["lambda"] ) self.stdout.write("Stream processing for %r will be set to %r" % ( options["lambda"], options["enabled"] )) for mapping in event_source_list["EventSourceMappings"]: LAMBDA.update_event_source_mapping( UUID=mapping["UUID"], Enabled=enabled, ) self.stdout.write("Finished.")
def predict_archetype_id(self, deck): event = self._to_prediction_event(deck) if settings.USE_ARCHETYPE_PREDICTION_LAMBDA or settings.ENV_AWS: with influx_timer("callout_to_predict_deck_archetype"): response = LAMBDA.invoke( FunctionName="predict_deck_archetype", InvocationType="RequestResponse", # Synchronous invocation Payload=json.dumps(event), ) if response[ "StatusCode"] == 200 and "FunctionError" not in response: result = json.loads( response["Payload"].read().decode("utf8")) else: raise RuntimeError( response["Payload"].read().decode("utf8")) else: from keras_handler import handler result = handler(event, None) predicted_class = result["predicted_class"] id_encoding = self.one_hot_external_ids(inverse=True) predicted_archetype_id = id_encoding[predicted_class] if predicted_archetype_id == -1: return None else: return predicted_archetype_id
def set_prod_version(self, version): update_prod_result = LAMBDA.update_alias( FunctionName=self.canary_function_name, Name="PROD", FunctionVersion=version, ) self.stdout.write("PROD Alias updated to version: %s" % update_prod_result["FunctionVersion"]) return update_prod_result["FunctionVersion"]
def set_canary_version(self, version): update_canary_result = LAMBDA.update_alias( FunctionName=self.canary_function_name, Name="CANARY", FunctionVersion=version, ) self.log("CANARY Alias updated to version: %s" % update_canary_result["FunctionVersion"]) return update_canary_result["FunctionVersion"]
def schedule_delivery(self): """ Schedule the webhook for delivery. On ENV_AWS, this schedules a Lambda trigger. Otherwise, triggers immediately. """ self.status = WebhookStatus.PENDING self.save() if settings.WEBHOOKS["USE_LAMBDA"]: from hsreplaynet.utils.aws.clients import LAMBDA LAMBDA.invoke( FunctionName="trigger_webhook", InvocationType="Event", Payload=json.dumps({"webhook": str(self.pk)}), ) else: self.deliver()
def set_canary_version(self, version): update_canary_result = LAMBDA.update_alias( FunctionName=self.canary_function_name, Name="CANARY", FunctionVersion=version, ) self.log( "CANARY Alias updated to version: %s" % update_canary_result["FunctionVersion"] ) return update_canary_result["FunctionVersion"]
def set_prod_version(self, version): update_prod_result = LAMBDA.update_alias( FunctionName=self.canary_function_name, Name="PROD", FunctionVersion=version, ) self.stdout.write( "PROD Alias updated to version: %s" % update_prod_result["FunctionVersion"] ) return update_prod_result["FunctionVersion"]
def execute_query(parameterized_query, run_local=False): if run_local: _do_execute_query_work(parameterized_query) # It's safe to launch multiple attempts to execute for the same query # Because the dogpile lock will only allow one to execute # But we can save resources by not even launching the attempt # If we see that the lock already exists if not _lock_exists(parameterized_query.cache_key): log.info( "No lock already exists for query. Will attempt to execute async.") if settings.ENV_AWS and settings.PROCESS_REDSHIFT_QUERIES_VIA_LAMBDA: # In PROD use Lambdas so the web-servers don't get overloaded LAMBDA.invoke( FunctionName="execute_redshift_query", InvocationType="Event", # Triggers asynchronous invocation Payload=_to_lambda_payload(parameterized_query), ) else: _do_execute_query_work(parameterized_query) else: msg = "An async attempt to run this query is in-flight. Will not launch another." log.info(msg)
def handle(self, *args, **options): for module_name in options["module"].split(","): importlib.import_module(module_name) descriptors = get_lambda_descriptors() if not descriptors: self.output("No descriptors found. Exiting.") return all_lambdas = LAMBDA.list_functions() iam_default_role_response = IAM.get_role( RoleName=settings.LAMBDA_DEFAULT_EXECUTION_ROLE_NAME ) execution_role_arn = iam_default_role_response["Role"]["Arn"] iam_private_role_response = IAM.get_role( RoleName=settings.LAMBDA_PRIVATE_EXECUTION_ROLE_NAME ) private_execution_role_arn = iam_private_role_response["Role"]["Arn"] self.output("Execution Role Arn: %r" % (execution_role_arn)) self.output("Private Execution Role Arn: %r" % (private_execution_role_arn)) artifact_obj = options["artifact"] artifact_bucket = options["artifact_bucket"] self.output("Using code at S3 path: %r/%r" % (artifact_bucket, artifact_obj)) for descriptor in descriptors: self.output("About to deploy: %r" % (descriptor["name"])) existing_lambda = None for func in all_lambdas["Functions"]: if func["FunctionName"] == descriptor["name"]: existing_lambda = func lambda_args = { "Runtime": descriptor["runtime"], "FunctionName": descriptor["name"], "Handler": descriptor["handler"], "Timeout": descriptor["cpu_seconds"], "MemorySize": descriptor["memory"], } if descriptor["requires_vpc_access"]: lambda_args["Role"] = private_execution_role_arn lambda_args["VpcConfig"] = { "SubnetIds": [settings.AWS_PROD_PRIVATE_SUBNET], "SecurityGroupIds": [settings.AWS_PROD_SECURITY_GROUP_ID], } else: lambda_args["Role"] = execution_role_arn if existing_lambda: self.output("Updating lambda with arguments %r" % (lambda_args)) LAMBDA.update_function_configuration(**lambda_args) LAMBDA.update_function_code( FunctionName=descriptor["name"], S3Bucket=artifact_bucket, S3Key=artifact_obj, ) else: lambda_args["Code"] = { "S3Bucket": artifact_bucket, "S3Key": artifact_obj, } self.output("Creating lambda with arguments %r" % (lambda_args)) LAMBDA.create_function(**lambda_args) if descriptor["stream_name"]: # This lambda would like to be registered as a listener on a kinesis stream stream_name = descriptor["stream_name"] batch_size = descriptor["stream_batch_size"] self.output("Applying event source mapping for stream: %s" % (stream_name)) target_event_source = get_kinesis_stream_arn_from_name(stream_name) event_source_list = LAMBDA.list_event_source_mappings( FunctionName=descriptor["name"] ) if event_source_list["EventSourceMappings"]: # We need to update the existing mappings # First we need to remove any stale event source mappings. # Then we need to look for an existing match and update it. # Finally if we didn't update an existing match, we need to create a new mapping. # So long as we don't DELETE an in-use mapping # it will not loose its place in the stream. update_existing_mapping_success = False for mapping in event_source_list["EventSourceMappings"]: mapping_uuid = mapping["UUID"] mapping_event_source = mapping["EventSourceArn"] mapping_batch_size = mapping["BatchSize"] if mapping_event_source != target_event_source: # Delete this event source, it's stale. self.output("Deleting stale mapping: %r" % (mapping_event_source)) LAMBDA.delete_event_source_mapping(UUID=mapping_uuid) else: update_existing_mapping_success = True if mapping_batch_size != batch_size: # The batch size is the only thing that might have changed self.output("Updating existing stream batch size from %s to %s" % ( mapping_batch_size, batch_size )) LAMBDA.update_event_source_mapping( UUID=mapping_uuid, BatchSize=batch_size ) else: # Nothing has changed. self.output("No changes required.") if not update_existing_mapping_success: # We didn't find an existing mapping to update, so we still must create one self.output("Creating new mapping for event source: %s" % (target_event_source)) LAMBDA.create_event_source_mapping( EventSourceArn=target_event_source, FunctionName=descriptor["name"], BatchSize=batch_size, StartingPosition="TRIM_HORIZON" ) else: # No mappings currently exist, so we need to create a new mapping self.output("Creating new mapping for event source: %s" % (target_event_source)) LAMBDA.create_event_source_mapping( EventSourceArn=target_event_source, FunctionName=descriptor["name"], BatchSize=batch_size, StartingPosition="TRIM_HORIZON" ) if options["wait"]: self.wait_for_complete_deployment(timeout=30)
def handle(self, *args, **options): func_name = self.canary_function_name prod_alias = LAMBDA.get_alias(FunctionName=func_name, Name="PROD") self.log("PROD Alias starting at version: %s" % prod_alias["FunctionVersion"]) canary_alias = LAMBDA.get_alias(FunctionName=func_name, Name="CANARY") self.log("CANARY Alias starting at version: %s" % canary_alias["FunctionVersion"]) # Create new version self.stdout.write("Publishing new version...") new_version = LAMBDA.publish_version(FunctionName=func_name) new_version_num = new_version["Version"] self.log("New version is: %s" % new_version_num) # This causes the new code to start getting used on canary upload events. canary_period_start = now() self.set_canary_version(new_version_num) if options["bypass_canary"]: # We promote all aliases and return immediately. self.log("Bypassing canary stage and promoting immediately") self.set_prod_version(new_version_num) self.log("Finished.") return if is_processing_disabled(): self.log("Processing is disabled so we will not see any canaries") self.log("Bypassing canary stage and promoting immediately") self.set_prod_version(new_version_num) self.log("Finished.") return # If we did not exit already, then we are doing a canary deployment. max_wait_seconds = options["max_wait_seconds"] max_wait_time = now() + timedelta(seconds=max_wait_seconds) min_canary_uploads = options["min_canary_uploads"] wait_for_more_canary_uploads = True try: while wait_for_more_canary_uploads: canary_uploads = UploadEvent.objects.filter( canary=True, created__gte=canary_period_start, ).exclude(status__in=UploadEventStatus.processing_statuses()) if canary_uploads.count() >= min_canary_uploads: wait_for_more_canary_uploads = False elif len(self.get_failed_canaries(canary_uploads.all())): # Exit early since some canaries have already failed wait_for_more_canary_uploads = False else: if now() > max_wait_time: msg = "Waited too long for canary events. Exiting." self.log(msg) raise RuntimeError(msg) self.log("Found %i uploads... sleeping 5 seconds" % (canary_uploads.count())) time.sleep(5) self.log("%s canary upload events have been found" % (canary_uploads.count())) canary_failures = self.get_canary_failures_since( canary_period_start) if canary_failures: # We have canary failures, time to rollback. self.log("The following canary uploads have failed:") self.log(", ".join(u.shortid for u in canary_failures)) raise RuntimeError( "Failed canary events detected. Rolling back.") except Exception: # Revert the canary alias back to what PROD still points to prod_version = prod_alias["FunctionVersion"] self.log("CANARY will be reverted to version: %s" % prod_version) self.set_canary_version(prod_version) self.log("Initiating reprocessing to fix canary uploads") # Query again for all canary failures after reverting so we don't miss any canary_failures = self.get_canary_failures_since( canary_period_start) queue_upload_events_for_reprocessing(canary_failures, use_kinesis=True) self.log("Finished queuing canaries for reprocessing.") sys.exit(1) # We didn't have any canary failures so it's time to promote PROD. self.log("The canary version is a success! Promoting PROD.") self.set_prod_version(new_version_num) self.log("Finished.")
def handle(self, *args, **options): for module_name in options["module"].split(","): importlib.import_module(module_name) descriptors = get_lambda_descriptors() if not descriptors: self.stdout.write("No descriptors found. Exiting.") return all_lambdas = LAMBDA.list_functions() response = IAM.get_role(RoleName=settings.LAMBDA_DEFAULT_EXECUTION_ROLE_NAME) execution_role_arn = response["Role"]["Arn"] self.stdout.write("Execution Role Arn: %r" % (execution_role_arn)) artifact_obj = options["artifact"] artifact_bucket = settings.AWS_LAMBDA_ARTIFACTS_BUCKET self.stdout.write( "Using code at S3 path: %r/%r" % (artifact_bucket, artifact_obj) ) for descriptor in descriptors: self.stdout.write("About to deploy: %s" % (descriptor["name"])) existing_lambda = None for func in all_lambdas["Functions"]: if func["FunctionName"] == descriptor["name"]: existing_lambda = func if existing_lambda: self.stdout.write("Lambda exists - will update.") LAMBDA.update_function_configuration( FunctionName=descriptor["name"], Role=execution_role_arn, Handler=descriptor["handler"], Timeout=descriptor["cpu_seconds"], MemorySize=descriptor["memory"], ) LAMBDA.update_function_code( FunctionName=descriptor["name"], S3Bucket=artifact_bucket, S3Key=artifact_obj, ) else: self.stdout.write("New Lambda - will create.") LAMBDA.create_function( FunctionName=descriptor["name"], Runtime="python2.7", Role=execution_role_arn, Handler=descriptor["handler"], Code={ "S3Bucket": artifact_bucket, "S3Key": artifact_obj, }, Timeout=descriptor["cpu_seconds"], MemorySize=descriptor["memory"], ) if descriptor["stream_name"]: # This lambda would like to be registered as a listener on a kinesis stream stream_name = descriptor["stream_name"] batch_size = descriptor["stream_batch_size"] self.stdout.write("Applying event source mapping for stream: %s" % (stream_name)) target_event_source = get_kinesis_stream_arn_from_name(stream_name) event_source_list = LAMBDA.list_event_source_mappings( FunctionName=descriptor["name"] ) if event_source_list["EventSourceMappings"]: # We need to update the existing mappings # First we need to remove any stale event source mappings. # Then we need to look for an existing match and update it. # Finally if we didn't update an existing match, we need to create a new mapping. # So long as we don't DELETE an in-use mapping # it will not loose its place in the stream. update_existing_mapping_success = False for mapping in event_source_list["EventSourceMappings"]: mapping_uuid = mapping["UUID"] mapping_event_source = mapping["EventSourceArn"] mapping_batch_size = mapping["BatchSize"] if mapping_event_source != target_event_source: # Delete this event source, it's stale. self.stdout.write("Deleting stale mapping: %r" % (mapping_event_source)) LAMBDA.delete_event_source_mapping(UUID=mapping_uuid) else: update_existing_mapping_success = True if mapping_batch_size != batch_size: # The batch size is the only thing that might have changed self.stdout.write("Updating existing stream batch size from %s to %s" % ( mapping_batch_size, batch_size )) LAMBDA.update_event_source_mapping( UUID=mapping_uuid, BatchSize=batch_size ) else: # Nothing has changed. self.stdout.write("No changes required.") if not update_existing_mapping_success: # We didn't find an existing mapping to update, so we still must create one self.stdout.write("Creating new mapping for event source: %s" % target_event_source) LAMBDA.create_event_source_mapping( EventSourceArn=target_event_source, FunctionName=descriptor["name"], BatchSize=batch_size, StartingPosition="TRIM_HORIZON" ) else: # No mappings currently exist, so we need to create a new mapping self.stdout.write("Creating new mapping for event source: %s" % target_event_source) LAMBDA.create_event_source_mapping( EventSourceArn=target_event_source, FunctionName=descriptor["name"], BatchSize=batch_size, StartingPosition="TRIM_HORIZON" ) if options["wait"]: self.wait_for_complete_deployment(timeout=30)
def handle(self, *args, **options): for module_name in options["module"].split(","): importlib.import_module(module_name) descriptors = get_lambda_descriptors() if not descriptors: self.stdout.write("No descriptors found. Exiting.") return all_lambdas = LAMBDA.list_functions() response = IAM.get_role( RoleName=settings.LAMBDA_DEFAULT_EXECUTION_ROLE_NAME) execution_role_arn = response["Role"]["Arn"] self.stdout.write("Execution Role Arn: %r" % (execution_role_arn)) artifact_path = options["artifact"] self.stdout.write("Using code at path: %r" % (artifact_path)) with open(artifact_path, "rb") as artifact: code_payload_bytes = artifact.read() self.stdout.write("Code Payload Bytes: %s" % (len(code_payload_bytes))) for descriptor in descriptors: self.stdout.write("About to deploy: %s" % (descriptor["name"])) existing_lambda = None for func in all_lambdas["Functions"]: if func["FunctionName"] == descriptor["name"]: existing_lambda = func if existing_lambda: self.stdout.write("Lambda exists - will update.") LAMBDA.update_function_configuration( FunctionName=descriptor["name"], Role=execution_role_arn, Handler=descriptor["handler"], Timeout=descriptor["cpu_seconds"], MemorySize=descriptor["memory"], ) LAMBDA.update_function_code( FunctionName=descriptor["name"], ZipFile=code_payload_bytes, ) else: self.stdout.write("New Lambda - will create.") LAMBDA.create_function( FunctionName=descriptor["name"], Runtime="python2.7", Role=execution_role_arn, Handler=descriptor["handler"], Code={"ZipFile": code_payload_bytes}, Timeout=descriptor["cpu_seconds"], MemorySize=descriptor["memory"], ) if descriptor["stream_name"]: # This lambda would like to be registered as a listener on a kinesis stream stream_name = descriptor["stream_name"] batch_size = descriptor["stream_batch_size"] self.stdout.write( "Applying event source mapping for stream: %s" % (stream_name)) target_event_source = get_kinesis_stream_arn_from_name( stream_name) event_source_list = LAMBDA.list_event_source_mappings( FunctionName=descriptor["name"]) if event_source_list["EventSourceMappings"]: # We need to update the existing mappings # First we need to remove any stale event source mappings. # Then we need to look for an existing match and update it. # Finally if we didn't update an existing match, we need to create a new mapping. # So long as we don't DELETE an in-use mapping # it will not loose its place in the stream. update_existing_mapping_success = False for mapping in event_source_list["EventSourceMappings"]: mapping_uuid = mapping["UUID"] mapping_event_source = mapping["EventSourceArn"] mapping_batch_size = mapping["BatchSize"] if mapping_event_source != target_event_source: # Delete this event source, it's stale. self.stdout.write("Deleting stale mapping: %r" % (mapping_event_source)) LAMBDA.delete_event_source_mapping( UUID=mapping_uuid) else: update_existing_mapping_success = True if mapping_batch_size != batch_size: # The batch size is the only thing that might have changed self.stdout.write( "Updating existing stream batch size from %s to %s" % (mapping_batch_size, batch_size)) LAMBDA.update_event_source_mapping( UUID=mapping_uuid, BatchSize=batch_size) else: # Nothing has changed. self.stdout.write("No changes required.") if not update_existing_mapping_success: # We didn't find an existing mapping to update, so we still must create one self.stdout.write( "Creating new mapping for event source: %s" % target_event_source) LAMBDA.create_event_source_mapping( EventSourceArn=target_event_source, FunctionName=descriptor["name"], BatchSize=batch_size, StartingPosition="TRIM_HORIZON") else: # No mappings currently exist, so we need to create a new mapping self.stdout.write( "Creating new mapping for event source: %s" % target_event_source) LAMBDA.create_event_source_mapping( EventSourceArn=target_event_source, FunctionName=descriptor["name"], BatchSize=batch_size, StartingPosition="TRIM_HORIZON") if options["wait"]: self.wait_for_complete_deployment(timeout=30)
def get_meta_preview(num_items=10): from hsreplaynet.utils.aws.clients import LAMBDA from hsreplaynet.utils.aws.redshift import get_redshift_query LAMBDA.invoke(FunctionName="do_refresh_meta_preview", InvocationType="Event") query = get_redshift_query("archetype_popularity_distribution_stats") unique_archetypes = set() data = [] ranks = [x for x in range(0, 21)] for rank in ranks: regions = ["REGION_EU", "REGION_US", "REGION_KR", "REGION_CN"] for region in regions: parameterized_query = query.build_full_params( dict(TimeRange="LAST_1_DAY", GameType="RANKED_STANDARD", RankRange=RANK_MAP[rank], Region=region)) if not parameterized_query.result_available: continue response = parameterized_query.response_payload archetypes = [] for class_values in response["series"]["data"].values(): for value in class_values: if (value["archetype_id"] > 0 and value["pct_of_total"] > 0.5 and value["total_games"] > 30 and value["win_rate"] > 51): archetypes.append(value) if not len(archetypes): continue archetype = list( sorted(archetypes, key=lambda a: a["win_rate"], reverse=True))[0] unique_archetypes.add(archetype["archetype_id"]) data.append({ "rank": rank, "region": region, "data": archetype, "as_of": response["as_of"] }) results = [] for archetype_id in unique_archetypes: for datum in data: if datum["data"]["archetype_id"] == archetype_id: results.append(datum) break data.remove(datum) hour = datetime.utcnow().replace(minute=0, second=0, microsecond=0) random = seed(int(hour.timestamp())) if len(results) < num_items: shuffle(data, random=random) for i in range(0, num_items - len(results)): results.append(data[i]) shuffle(results, random=random) return results[:num_items]
def handle(self, *args, **options): func_name = self.canary_function_name prod_alias = LAMBDA.get_alias(FunctionName=func_name, Name="PROD") self.log("PROD Alias starting at version: %s" % prod_alias["FunctionVersion"]) canary_alias = LAMBDA.get_alias(FunctionName=func_name, Name="CANARY") self.log("CANARY Alias starting at version: %s" % canary_alias["FunctionVersion"]) # Create new version self.stdout.write("Publishing new version...") new_version = LAMBDA.publish_version(FunctionName=func_name) new_version_num = new_version["Version"] self.log("New version is: %s" % new_version_num) # This causes the new code to start getting used on canary upload events. canary_period_start = now() self.set_canary_version(new_version_num) if options["bypass_canary"]: # We promote all aliases and return immediately. self.log("Bypassing canary stage and promoting immediately") self.set_prod_version(new_version_num) self.log("Finished.") return if is_processing_disabled(): self.log("Processing is disabled so we will not see any canaries") self.log("Bypassing canary stage and promoting immediately") self.set_prod_version(new_version_num) self.log("Finished.") return # If we did not exit already, then we are doing a canary deployment. max_wait_seconds = options["max_wait_seconds"] max_wait_time = now() + timedelta(seconds=max_wait_seconds) min_canary_uploads = options["min_canary_uploads"] wait_for_more_canary_uploads = True try: while wait_for_more_canary_uploads: canary_uploads = UploadEvent.objects.filter( canary=True, created__gte=canary_period_start, ).exclude(status__in=UploadEventStatus.processing_statuses()) if canary_uploads.count() >= min_canary_uploads: wait_for_more_canary_uploads = False elif len(self.get_failed_canaries(canary_uploads.all())): # Exit early since some canaries have already failed wait_for_more_canary_uploads = False else: if now() > max_wait_time: msg = "Waited too long for canary events. Exiting." self.log(msg) raise RuntimeError(msg) self.log( "Found %i uploads... sleeping 5 seconds" % (canary_uploads.count(),) ) time.sleep(5) self.log( "%s canary upload events have been found" % canary_uploads.count() ) canary_failures = self.get_canary_failures_since(canary_period_start) if canary_failures: # We have canary failures, time to rollback. self.log("The following canary uploads have failed:") self.log(", ".join(u.shortid for u in canary_failures)) raise RuntimeError("Failed canary events detected. Rolling back.") except Exception: # Revert the canary alias back to what PROD still points to prod_version = prod_alias["FunctionVersion"] self.log("CANARY will be reverted to version: %s" % prod_version) self.set_canary_version(prod_version) self.log("Initiating reprocessing to fix canary uploads") # Query again for all canary failures after reverting so we don't miss any canary_failures = self.get_canary_failures_since(canary_period_start) queue_upload_events_for_reprocessing(canary_failures, use_kinesis=True) self.log("Finished queuing canaries for reprocessing.") sys.exit(1) # We didn't have any canary failures so it's time to promote PROD. self.log("The canary version is a success! Promoting PROD.") self.set_prod_version(new_version_num) self.log("Finished.")