def handle(self, *args, **options):
        enabled = True if options["enabled"].lower() == "true" else False
        event_source_list = LAMBDA.list_event_source_mappings(FunctionName=options["lambda"])
        self.stdout.write("Stream processing for %r will be set to %r" % (options["lambda"], options["enabled"]))

        for mapping in event_source_list["EventSourceMappings"]:
            LAMBDA.update_event_source_mapping(UUID=mapping["UUID"], Enabled=enabled)
        self.stdout.write("Finished.")
Esempio n. 2
0
    def schedule_lambda_trigger(self, url, payload):
        from hsreplaynet.utils.aws.clients import LAMBDA

        final_payload = self._serialize_payload(payload)

        LAMBDA.invoke(
            FunctionName="trigger_webhook",
            InvocationType="Event",  # Triggers asynchronous invocation
            Payload=final_payload,
        )
Esempio n. 3
0
	def schedule_lambda_trigger(self, url, payload):
		from hsreplaynet.utils.aws.clients import LAMBDA

		final_payload = self._serialize_payload(payload)

		LAMBDA.invoke(
			FunctionName="trigger_webhook",
			InvocationType="Event",  # Triggers asynchronous invocation
			Payload=final_payload,
		)
Esempio n. 4
0
	def lambda_invoker(payload, shortid):
		try:
			LAMBDA.invoke(
				FunctionName="process_single_replay_upload_stream_handler",
				InvocationType="RequestResponse",  # Triggers synchronous invocation
				Payload=payload,
			)
		finally:
			logger.debug("Lambda completed for %s Decrementing latch.", shortid)
			countdown_latch.count_down()
Esempio n. 5
0
 def lambda_invoker(payload, shortid):
     try:
         LAMBDA.invoke(
             FunctionName="process_single_replay_upload_stream_handler",
             InvocationType=
             "RequestResponse",  # Triggers synchronous invocation
             Payload=payload,
         )
     finally:
         logger.info("Lambda completed for %s Decrementing latch.", shortid)
         countdown_latch.count_down()
Esempio n. 6
0
	def handle(self, *args, **options):
		enabled = True if options["enabled"].lower() == "true" else False
		event_source_list = LAMBDA.list_event_source_mappings(
			FunctionName=options["lambda"]
		)
		self.stdout.write("Stream processing for %r will be set to %r" % (
			options["lambda"],
			options["enabled"]
		))

		for mapping in event_source_list["EventSourceMappings"]:
			LAMBDA.update_event_source_mapping(
				UUID=mapping["UUID"],
				Enabled=enabled,
			)
		self.stdout.write("Finished.")
Esempio n. 7
0
    def predict_archetype_id(self, deck):
        event = self._to_prediction_event(deck)

        if settings.USE_ARCHETYPE_PREDICTION_LAMBDA or settings.ENV_AWS:
            with influx_timer("callout_to_predict_deck_archetype"):
                response = LAMBDA.invoke(
                    FunctionName="predict_deck_archetype",
                    InvocationType="RequestResponse",  # Synchronous invocation
                    Payload=json.dumps(event),
                )
                if response[
                        "StatusCode"] == 200 and "FunctionError" not in response:
                    result = json.loads(
                        response["Payload"].read().decode("utf8"))
                else:
                    raise RuntimeError(
                        response["Payload"].read().decode("utf8"))
        else:
            from keras_handler import handler
            result = handler(event, None)

        predicted_class = result["predicted_class"]
        id_encoding = self.one_hot_external_ids(inverse=True)
        predicted_archetype_id = id_encoding[predicted_class]

        if predicted_archetype_id == -1:
            return None
        else:
            return predicted_archetype_id
 def set_prod_version(self, version):
     update_prod_result = LAMBDA.update_alias(
         FunctionName=self.canary_function_name,
         Name="PROD",
         FunctionVersion=version,
     )
     self.stdout.write("PROD Alias updated to version: %s" %
                       update_prod_result["FunctionVersion"])
     return update_prod_result["FunctionVersion"]
 def set_canary_version(self, version):
     update_canary_result = LAMBDA.update_alias(
         FunctionName=self.canary_function_name,
         Name="CANARY",
         FunctionVersion=version,
     )
     self.log("CANARY Alias updated to version: %s" %
              update_canary_result["FunctionVersion"])
     return update_canary_result["FunctionVersion"]
Esempio n. 10
0
    def schedule_delivery(self):
        """
		Schedule the webhook for delivery.

		On ENV_AWS, this schedules a Lambda trigger.
		Otherwise, triggers immediately.
		"""
        self.status = WebhookStatus.PENDING
        self.save()

        if settings.WEBHOOKS["USE_LAMBDA"]:
            from hsreplaynet.utils.aws.clients import LAMBDA
            LAMBDA.invoke(
                FunctionName="trigger_webhook",
                InvocationType="Event",
                Payload=json.dumps({"webhook": str(self.pk)}),
            )
        else:
            self.deliver()
	def set_canary_version(self, version):
		update_canary_result = LAMBDA.update_alias(
			FunctionName=self.canary_function_name,
			Name="CANARY",
			FunctionVersion=version,
		)
		self.log(
			"CANARY Alias updated to version: %s" % update_canary_result["FunctionVersion"]
		)
		return update_canary_result["FunctionVersion"]
	def set_prod_version(self, version):
		update_prod_result = LAMBDA.update_alias(
			FunctionName=self.canary_function_name,
			Name="PROD",
			FunctionVersion=version,
		)
		self.stdout.write(
			"PROD Alias updated to version: %s" % update_prod_result["FunctionVersion"]
		)
		return update_prod_result["FunctionVersion"]
Esempio n. 13
0
def execute_query(parameterized_query, run_local=False):
    if run_local:
        _do_execute_query_work(parameterized_query)

    # It's safe to launch multiple attempts to execute for the same query
    # Because the dogpile lock will only allow one to execute
    # But we can save resources by not even launching the attempt
    # If we see that the lock already exists
    if not _lock_exists(parameterized_query.cache_key):
        log.info(
            "No lock already exists for query. Will attempt to execute async.")

        if settings.ENV_AWS and settings.PROCESS_REDSHIFT_QUERIES_VIA_LAMBDA:
            # In PROD use Lambdas so the web-servers don't get overloaded
            LAMBDA.invoke(
                FunctionName="execute_redshift_query",
                InvocationType="Event",  # Triggers asynchronous invocation
                Payload=_to_lambda_payload(parameterized_query),
            )
        else:
            _do_execute_query_work(parameterized_query)
    else:
        msg = "An async attempt to run this query is in-flight. Will not launch another."
        log.info(msg)
Esempio n. 14
0
	def handle(self, *args, **options):
		for module_name in options["module"].split(","):
			importlib.import_module(module_name)

		descriptors = get_lambda_descriptors()
		if not descriptors:
			self.output("No descriptors found. Exiting.")
			return

		all_lambdas = LAMBDA.list_functions()

		iam_default_role_response = IAM.get_role(
			RoleName=settings.LAMBDA_DEFAULT_EXECUTION_ROLE_NAME
		)
		execution_role_arn = iam_default_role_response["Role"]["Arn"]

		iam_private_role_response = IAM.get_role(
			RoleName=settings.LAMBDA_PRIVATE_EXECUTION_ROLE_NAME
		)
		private_execution_role_arn = iam_private_role_response["Role"]["Arn"]
		self.output("Execution Role Arn: %r" % (execution_role_arn))
		self.output("Private Execution Role Arn: %r" % (private_execution_role_arn))

		artifact_obj = options["artifact"]
		artifact_bucket = options["artifact_bucket"]
		self.output("Using code at S3 path: %r/%r" % (artifact_bucket, artifact_obj))

		for descriptor in descriptors:
			self.output("About to deploy: %r" % (descriptor["name"]))

			existing_lambda = None
			for func in all_lambdas["Functions"]:
				if func["FunctionName"] == descriptor["name"]:
					existing_lambda = func

			lambda_args = {
				"Runtime": descriptor["runtime"],
				"FunctionName": descriptor["name"],
				"Handler": descriptor["handler"],
				"Timeout": descriptor["cpu_seconds"],
				"MemorySize": descriptor["memory"],
			}

			if descriptor["requires_vpc_access"]:
				lambda_args["Role"] = private_execution_role_arn
				lambda_args["VpcConfig"] = {
					"SubnetIds": [settings.AWS_PROD_PRIVATE_SUBNET],
					"SecurityGroupIds": [settings.AWS_PROD_SECURITY_GROUP_ID],
				}
			else:
				lambda_args["Role"] = execution_role_arn

			if existing_lambda:
				self.output("Updating lambda with arguments %r" % (lambda_args))
				LAMBDA.update_function_configuration(**lambda_args)
				LAMBDA.update_function_code(
					FunctionName=descriptor["name"],
					S3Bucket=artifact_bucket,
					S3Key=artifact_obj,
				)
			else:
				lambda_args["Code"] = {
					"S3Bucket": artifact_bucket,
					"S3Key": artifact_obj,
				}
				self.output("Creating lambda with arguments %r" % (lambda_args))
				LAMBDA.create_function(**lambda_args)

			if descriptor["stream_name"]:
				# This lambda would like to be registered as a listener on a kinesis stream
				stream_name = descriptor["stream_name"]
				batch_size = descriptor["stream_batch_size"]
				self.output("Applying event source mapping for stream: %s" % (stream_name))
				target_event_source = get_kinesis_stream_arn_from_name(stream_name)

				event_source_list = LAMBDA.list_event_source_mappings(
					FunctionName=descriptor["name"]
				)

				if event_source_list["EventSourceMappings"]:
					# We need to update the existing mappings
					# First we need to remove any stale event source mappings.
					# Then we need to look for an existing match and update it.
					# Finally if we didn't update an existing match, we need to create a new mapping.
					# So long as we don't DELETE an in-use mapping
					# it will not loose its place in the stream.

					update_existing_mapping_success = False

					for mapping in event_source_list["EventSourceMappings"]:
						mapping_uuid = mapping["UUID"]
						mapping_event_source = mapping["EventSourceArn"]
						mapping_batch_size = mapping["BatchSize"]

						if mapping_event_source != target_event_source:
							# Delete this event source, it's stale.
							self.output("Deleting stale mapping: %r" % (mapping_event_source))
							LAMBDA.delete_event_source_mapping(UUID=mapping_uuid)
						else:
							update_existing_mapping_success = True

							if mapping_batch_size != batch_size:
								# The batch size is the only thing that might have changed
								self.output("Updating existing stream batch size from %s to %s" % (
									mapping_batch_size,
									batch_size
								))
								LAMBDA.update_event_source_mapping(
									UUID=mapping_uuid,
									BatchSize=batch_size
								)
							else:
								# Nothing has changed.
								self.output("No changes required.")

					if not update_existing_mapping_success:
						# We didn't find an existing mapping to update, so we still must create one
						self.output("Creating new mapping for event source: %s" % (target_event_source))
						LAMBDA.create_event_source_mapping(
							EventSourceArn=target_event_source,
							FunctionName=descriptor["name"],
							BatchSize=batch_size,
							StartingPosition="TRIM_HORIZON"
						)
				else:
					# No mappings currently exist, so we need to create a new mapping
					self.output("Creating new mapping for event source: %s" % (target_event_source))
					LAMBDA.create_event_source_mapping(
						EventSourceArn=target_event_source,
						FunctionName=descriptor["name"],
						BatchSize=batch_size,
						StartingPosition="TRIM_HORIZON"
					)

		if options["wait"]:
			self.wait_for_complete_deployment(timeout=30)
    def handle(self, *args, **options):
        func_name = self.canary_function_name

        prod_alias = LAMBDA.get_alias(FunctionName=func_name, Name="PROD")
        self.log("PROD Alias starting at version: %s" %
                 prod_alias["FunctionVersion"])

        canary_alias = LAMBDA.get_alias(FunctionName=func_name, Name="CANARY")
        self.log("CANARY Alias starting at version: %s" %
                 canary_alias["FunctionVersion"])

        # Create new version
        self.stdout.write("Publishing new version...")
        new_version = LAMBDA.publish_version(FunctionName=func_name)
        new_version_num = new_version["Version"]
        self.log("New version is: %s" % new_version_num)

        # This causes the new code to start getting used on canary upload events.
        canary_period_start = now()
        self.set_canary_version(new_version_num)

        if options["bypass_canary"]:
            # We promote all aliases and return immediately.
            self.log("Bypassing canary stage and promoting immediately")
            self.set_prod_version(new_version_num)
            self.log("Finished.")
            return

        if is_processing_disabled():
            self.log("Processing is disabled so we will not see any canaries")
            self.log("Bypassing canary stage and promoting immediately")

            self.set_prod_version(new_version_num)

            self.log("Finished.")
            return

        # If we did not exit already, then we are doing a canary deployment.
        max_wait_seconds = options["max_wait_seconds"]
        max_wait_time = now() + timedelta(seconds=max_wait_seconds)

        min_canary_uploads = options["min_canary_uploads"]

        wait_for_more_canary_uploads = True
        try:
            while wait_for_more_canary_uploads:
                canary_uploads = UploadEvent.objects.filter(
                    canary=True,
                    created__gte=canary_period_start,
                ).exclude(status__in=UploadEventStatus.processing_statuses())

                if canary_uploads.count() >= min_canary_uploads:
                    wait_for_more_canary_uploads = False
                elif len(self.get_failed_canaries(canary_uploads.all())):
                    # Exit early since some canaries have already failed
                    wait_for_more_canary_uploads = False
                else:
                    if now() > max_wait_time:
                        msg = "Waited too long for canary events. Exiting."
                        self.log(msg)
                        raise RuntimeError(msg)

                    self.log("Found %i uploads... sleeping 5 seconds" %
                             (canary_uploads.count()))
                    time.sleep(5)

            self.log("%s canary upload events have been found" %
                     (canary_uploads.count()))

            canary_failures = self.get_canary_failures_since(
                canary_period_start)

            if canary_failures:
                # We have canary failures, time to rollback.
                self.log("The following canary uploads have failed:")
                self.log(", ".join(u.shortid for u in canary_failures))
                raise RuntimeError(
                    "Failed canary events detected. Rolling back.")
        except Exception:

            # Revert the canary alias back to what PROD still points to
            prod_version = prod_alias["FunctionVersion"]
            self.log("CANARY will be reverted to version: %s" % prod_version)
            self.set_canary_version(prod_version)

            self.log("Initiating reprocessing to fix canary uploads")
            # Query again for all canary failures after reverting so we don't miss any
            canary_failures = self.get_canary_failures_since(
                canary_period_start)
            queue_upload_events_for_reprocessing(canary_failures,
                                                 use_kinesis=True)
            self.log("Finished queuing canaries for reprocessing.")
            sys.exit(1)

        # We didn't have any canary failures so it's time to promote PROD.
        self.log("The canary version is a success! Promoting PROD.")
        self.set_prod_version(new_version_num)
        self.log("Finished.")
	def handle(self, *args, **options):
		for module_name in options["module"].split(","):
			importlib.import_module(module_name)

		descriptors = get_lambda_descriptors()
		if not descriptors:
			self.stdout.write("No descriptors found. Exiting.")
			return

		all_lambdas = LAMBDA.list_functions()

		response = IAM.get_role(RoleName=settings.LAMBDA_DEFAULT_EXECUTION_ROLE_NAME)
		execution_role_arn = response["Role"]["Arn"]
		self.stdout.write("Execution Role Arn: %r" % (execution_role_arn))

		artifact_obj = options["artifact"]
		artifact_bucket = settings.AWS_LAMBDA_ARTIFACTS_BUCKET
		self.stdout.write(
			"Using code at S3 path: %r/%r" % (artifact_bucket, artifact_obj)
		)

		for descriptor in descriptors:
			self.stdout.write("About to deploy: %s" % (descriptor["name"]))

			existing_lambda = None
			for func in all_lambdas["Functions"]:
				if func["FunctionName"] == descriptor["name"]:
					existing_lambda = func

			if existing_lambda:
				self.stdout.write("Lambda exists - will update.")

				LAMBDA.update_function_configuration(
					FunctionName=descriptor["name"],
					Role=execution_role_arn,
					Handler=descriptor["handler"],
					Timeout=descriptor["cpu_seconds"],
					MemorySize=descriptor["memory"],
				)

				LAMBDA.update_function_code(
					FunctionName=descriptor["name"],
					S3Bucket=artifact_bucket,
					S3Key=artifact_obj,
				)

			else:
				self.stdout.write("New Lambda - will create.")

				LAMBDA.create_function(
					FunctionName=descriptor["name"],
					Runtime="python2.7",
					Role=execution_role_arn,
					Handler=descriptor["handler"],
					Code={
						"S3Bucket": artifact_bucket,
						"S3Key": artifact_obj,
					},
					Timeout=descriptor["cpu_seconds"],
					MemorySize=descriptor["memory"],
				)

			if descriptor["stream_name"]:
				# This lambda would like to be registered as a listener on a kinesis stream
				stream_name = descriptor["stream_name"]
				batch_size = descriptor["stream_batch_size"]
				self.stdout.write("Applying event source mapping for stream: %s" % (stream_name))
				target_event_source = get_kinesis_stream_arn_from_name(stream_name)

				event_source_list = LAMBDA.list_event_source_mappings(
					FunctionName=descriptor["name"]
				)

				if event_source_list["EventSourceMappings"]:
					# We need to update the existing mappings
					# First we need to remove any stale event source mappings.
					# Then we need to look for an existing match and update it.
					# Finally if we didn't update an existing match, we need to create a new mapping.
					# So long as we don't DELETE an in-use mapping
					# it will not loose its place in the stream.

					update_existing_mapping_success = False

					for mapping in event_source_list["EventSourceMappings"]:
						mapping_uuid = mapping["UUID"]
						mapping_event_source = mapping["EventSourceArn"]
						mapping_batch_size = mapping["BatchSize"]

						if mapping_event_source != target_event_source:
							# Delete this event source, it's stale.
							self.stdout.write("Deleting stale mapping: %r" % (mapping_event_source))
							LAMBDA.delete_event_source_mapping(UUID=mapping_uuid)
						else:
							update_existing_mapping_success = True

							if mapping_batch_size != batch_size:
								# The batch size is the only thing that might have changed
								self.stdout.write("Updating existing stream batch size from %s to %s" % (
									mapping_batch_size,
									batch_size
								))
								LAMBDA.update_event_source_mapping(
									UUID=mapping_uuid,
									BatchSize=batch_size
								)
							else:
								# Nothing has changed.
								self.stdout.write("No changes required.")

					if not update_existing_mapping_success:
						# We didn't find an existing mapping to update, so we still must create one
						self.stdout.write("Creating new mapping for event source: %s" % target_event_source)
						LAMBDA.create_event_source_mapping(
							EventSourceArn=target_event_source,
							FunctionName=descriptor["name"],
							BatchSize=batch_size,
							StartingPosition="TRIM_HORIZON"
						)
				else:
					# No mappings currently exist, so we need to create a new mapping
					self.stdout.write("Creating new mapping for event source: %s" % target_event_source)
					LAMBDA.create_event_source_mapping(
						EventSourceArn=target_event_source,
						FunctionName=descriptor["name"],
						BatchSize=batch_size,
						StartingPosition="TRIM_HORIZON"
					)

		if options["wait"]:
			self.wait_for_complete_deployment(timeout=30)
    def handle(self, *args, **options):
        for module_name in options["module"].split(","):
            importlib.import_module(module_name)

        descriptors = get_lambda_descriptors()
        if not descriptors:
            self.stdout.write("No descriptors found. Exiting.")
            return

        all_lambdas = LAMBDA.list_functions()

        response = IAM.get_role(
            RoleName=settings.LAMBDA_DEFAULT_EXECUTION_ROLE_NAME)
        execution_role_arn = response["Role"]["Arn"]
        self.stdout.write("Execution Role Arn: %r" % (execution_role_arn))

        artifact_path = options["artifact"]
        self.stdout.write("Using code at path: %r" % (artifact_path))

        with open(artifact_path, "rb") as artifact:
            code_payload_bytes = artifact.read()
        self.stdout.write("Code Payload Bytes: %s" % (len(code_payload_bytes)))

        for descriptor in descriptors:
            self.stdout.write("About to deploy: %s" % (descriptor["name"]))

            existing_lambda = None
            for func in all_lambdas["Functions"]:
                if func["FunctionName"] == descriptor["name"]:
                    existing_lambda = func

            if existing_lambda:
                self.stdout.write("Lambda exists - will update.")

                LAMBDA.update_function_configuration(
                    FunctionName=descriptor["name"],
                    Role=execution_role_arn,
                    Handler=descriptor["handler"],
                    Timeout=descriptor["cpu_seconds"],
                    MemorySize=descriptor["memory"],
                )

                LAMBDA.update_function_code(
                    FunctionName=descriptor["name"],
                    ZipFile=code_payload_bytes,
                )

            else:
                self.stdout.write("New Lambda - will create.")

                LAMBDA.create_function(
                    FunctionName=descriptor["name"],
                    Runtime="python2.7",
                    Role=execution_role_arn,
                    Handler=descriptor["handler"],
                    Code={"ZipFile": code_payload_bytes},
                    Timeout=descriptor["cpu_seconds"],
                    MemorySize=descriptor["memory"],
                )

            if descriptor["stream_name"]:
                # This lambda would like to be registered as a listener on a kinesis stream
                stream_name = descriptor["stream_name"]
                batch_size = descriptor["stream_batch_size"]
                self.stdout.write(
                    "Applying event source mapping for stream: %s" %
                    (stream_name))
                target_event_source = get_kinesis_stream_arn_from_name(
                    stream_name)

                event_source_list = LAMBDA.list_event_source_mappings(
                    FunctionName=descriptor["name"])

                if event_source_list["EventSourceMappings"]:
                    # We need to update the existing mappings
                    # First we need to remove any stale event source mappings.
                    # Then we need to look for an existing match and update it.
                    # Finally if we didn't update an existing match, we need to create a new mapping.
                    # So long as we don't DELETE an in-use mapping
                    # it will not loose its place in the stream.

                    update_existing_mapping_success = False

                    for mapping in event_source_list["EventSourceMappings"]:
                        mapping_uuid = mapping["UUID"]
                        mapping_event_source = mapping["EventSourceArn"]
                        mapping_batch_size = mapping["BatchSize"]

                        if mapping_event_source != target_event_source:
                            # Delete this event source, it's stale.
                            self.stdout.write("Deleting stale mapping: %r" %
                                              (mapping_event_source))
                            LAMBDA.delete_event_source_mapping(
                                UUID=mapping_uuid)
                        else:
                            update_existing_mapping_success = True

                            if mapping_batch_size != batch_size:
                                # The batch size is the only thing that might have changed
                                self.stdout.write(
                                    "Updating existing stream batch size from %s to %s"
                                    % (mapping_batch_size, batch_size))
                                LAMBDA.update_event_source_mapping(
                                    UUID=mapping_uuid, BatchSize=batch_size)
                            else:
                                # Nothing has changed.
                                self.stdout.write("No changes required.")

                    if not update_existing_mapping_success:
                        # We didn't find an existing mapping to update, so we still must create one
                        self.stdout.write(
                            "Creating new mapping for event source: %s" %
                            target_event_source)
                        LAMBDA.create_event_source_mapping(
                            EventSourceArn=target_event_source,
                            FunctionName=descriptor["name"],
                            BatchSize=batch_size,
                            StartingPosition="TRIM_HORIZON")
                else:
                    # No mappings currently exist, so we need to create a new mapping
                    self.stdout.write(
                        "Creating new mapping for event source: %s" %
                        target_event_source)
                    LAMBDA.create_event_source_mapping(
                        EventSourceArn=target_event_source,
                        FunctionName=descriptor["name"],
                        BatchSize=batch_size,
                        StartingPosition="TRIM_HORIZON")

        if options["wait"]:
            self.wait_for_complete_deployment(timeout=30)
Esempio n. 18
0
def get_meta_preview(num_items=10):
    from hsreplaynet.utils.aws.clients import LAMBDA
    from hsreplaynet.utils.aws.redshift import get_redshift_query

    LAMBDA.invoke(FunctionName="do_refresh_meta_preview",
                  InvocationType="Event")

    query = get_redshift_query("archetype_popularity_distribution_stats")

    unique_archetypes = set()
    data = []

    ranks = [x for x in range(0, 21)]

    for rank in ranks:
        regions = ["REGION_EU", "REGION_US", "REGION_KR", "REGION_CN"]
        for region in regions:
            parameterized_query = query.build_full_params(
                dict(TimeRange="LAST_1_DAY",
                     GameType="RANKED_STANDARD",
                     RankRange=RANK_MAP[rank],
                     Region=region))

            if not parameterized_query.result_available:
                continue

            response = parameterized_query.response_payload

            archetypes = []
            for class_values in response["series"]["data"].values():
                for value in class_values:
                    if (value["archetype_id"] > 0
                            and value["pct_of_total"] > 0.5
                            and value["total_games"] > 30
                            and value["win_rate"] > 51):
                        archetypes.append(value)
            if not len(archetypes):
                continue

            archetype = list(
                sorted(archetypes, key=lambda a: a["win_rate"],
                       reverse=True))[0]
            unique_archetypes.add(archetype["archetype_id"])
            data.append({
                "rank": rank,
                "region": region,
                "data": archetype,
                "as_of": response["as_of"]
            })

    results = []
    for archetype_id in unique_archetypes:
        for datum in data:
            if datum["data"]["archetype_id"] == archetype_id:
                results.append(datum)
                break
        data.remove(datum)

    hour = datetime.utcnow().replace(minute=0, second=0, microsecond=0)
    random = seed(int(hour.timestamp()))

    if len(results) < num_items:
        shuffle(data, random=random)
        for i in range(0, num_items - len(results)):
            results.append(data[i])

    shuffle(results, random=random)

    return results[:num_items]
	def handle(self, *args, **options):
		func_name = self.canary_function_name

		prod_alias = LAMBDA.get_alias(FunctionName=func_name, Name="PROD")
		self.log("PROD Alias starting at version: %s" % prod_alias["FunctionVersion"])

		canary_alias = LAMBDA.get_alias(FunctionName=func_name, Name="CANARY")
		self.log("CANARY Alias starting at version: %s" % canary_alias["FunctionVersion"])

		# Create new version
		self.stdout.write("Publishing new version...")
		new_version = LAMBDA.publish_version(FunctionName=func_name)
		new_version_num = new_version["Version"]
		self.log("New version is: %s" % new_version_num)

		# This causes the new code to start getting used on canary upload events.
		canary_period_start = now()
		self.set_canary_version(new_version_num)

		if options["bypass_canary"]:
			# We promote all aliases and return immediately.
			self.log("Bypassing canary stage and promoting immediately")
			self.set_prod_version(new_version_num)
			self.log("Finished.")
			return

		if is_processing_disabled():
			self.log("Processing is disabled so we will not see any canaries")
			self.log("Bypassing canary stage and promoting immediately")

			self.set_prod_version(new_version_num)

			self.log("Finished.")
			return

		# If we did not exit already, then we are doing a canary deployment.
		max_wait_seconds = options["max_wait_seconds"]
		max_wait_time = now() + timedelta(seconds=max_wait_seconds)

		min_canary_uploads = options["min_canary_uploads"]

		wait_for_more_canary_uploads = True
		try:
			while wait_for_more_canary_uploads:
				canary_uploads = UploadEvent.objects.filter(
					canary=True,
					created__gte=canary_period_start,
				).exclude(status__in=UploadEventStatus.processing_statuses())

				if canary_uploads.count() >= min_canary_uploads:
					wait_for_more_canary_uploads = False
				elif len(self.get_failed_canaries(canary_uploads.all())):
					# Exit early since some canaries have already failed
					wait_for_more_canary_uploads = False
				else:
					if now() > max_wait_time:
						msg = "Waited too long for canary events. Exiting."
						self.log(msg)
						raise RuntimeError(msg)

					self.log(
						"Found %i uploads... sleeping 5 seconds" % (canary_uploads.count(),)
					)
					time.sleep(5)

			self.log(
				"%s canary upload events have been found" % canary_uploads.count()
			)

			canary_failures = self.get_canary_failures_since(canary_period_start)

			if canary_failures:
				# We have canary failures, time to rollback.
				self.log("The following canary uploads have failed:")
				self.log(", ".join(u.shortid for u in canary_failures))
				raise RuntimeError("Failed canary events detected. Rolling back.")
		except Exception:

			# Revert the canary alias back to what PROD still points to
			prod_version = prod_alias["FunctionVersion"]
			self.log("CANARY will be reverted to version: %s" % prod_version)
			self.set_canary_version(prod_version)

			self.log("Initiating reprocessing to fix canary uploads")
			# Query again for all canary failures after reverting so we don't miss any
			canary_failures = self.get_canary_failures_since(canary_period_start)
			queue_upload_events_for_reprocessing(canary_failures, use_kinesis=True)
			self.log("Finished queuing canaries for reprocessing.")
			sys.exit(1)

		# We didn't have any canary failures so it's time to promote PROD.
		self.log("The canary version is a success! Promoting PROD.")
		self.set_prod_version(new_version_num)
		self.log("Finished.")