def post(self, request, ab_test_id, format=None):
try:
ab_test = ABTest.objects.get(pk=ab_test_id)
if ab_test.ended_at is not None:
return Response({"message": "AB Test already finished."})
date_now = datetime.datetime.now()
# alg #1 accuracy
all_responses_1 = MLRequest.objects.filter(parent_mlalgorithm=ab_test.parent_mlalgorithm_1, created_at__gt = ab_test.created_at, created_at__lt = date_now).count()
correct_responses_1 = MLRequest.objects.filter(parent_mlalgorithm=ab_test.parent_mlalgorithm_1, created_at__gt = ab_test.created_at, created_at__lt = date_now, response=F('feedback')).count()
accuracy_1 = correct_responses_1 / float(all_responses_1)
print(all_responses_1, correct_responses_1, accuracy_1)
# alg #2 accuracy
all_responses_2 = MLRequest.objects.filter(parent_mlalgorithm=ab_test.parent_mlalgorithm_2, created_at__gt = ab_test.created_at, created_at__lt = date_now).count()
correct_responses_2 = MLRequest.objects.filter(parent_mlalgorithm=ab_test.parent_mlalgorithm_2, created_at__gt = ab_test.created_at, created_at__lt = date_now, response=F('feedback')).count()
accuracy_2 = correct_responses_2 / float(all_responses_2)
print(all_responses_2, correct_responses_2, accuracy_2)
# select algorithm with higher accuracy
alg_id_1, alg_id_2 = ab_test.parent_mlalgorithm_1, ab_test.parent_mlalgorithm_2
# swap
if accuracy_1 < accuracy_2:
alg_id_1, alg_id_2 = alg_id_2, alg_id_1
status_1 = MLAlgorithmStatus(status = "production",
created_by=ab_test.created_by,
parent_mlalgorithm = alg_id_1,
active=True)
status_1.save()
deactivate_other_statuses(status_1)
# update status for second algorithm
status_2 = MLAlgorithmStatus(status = "testing",
created_by=ab_test.created_by,
parent_mlalgorithm = alg_id_2,
active=True)
status_2.save()
deactivate_other_statuses(status_2)
summary = "Algorithm #1 accuracy: {}, Algorithm #2 accuracy: {}".format(accuracy_1, accuracy_2)
ab_test.ended_at = date_now
ab_test.summary = summary
ab_test.save()
except Exception as e:
return Response({"status": "Error", "message": str(e)},
status=status.HTTP_400_BAD_REQUEST
)
return Response({"message": "AB Test finished.", "summary": summary})
def perform_create(self, serializer):
try:
with transaction.atomic():
instance = serializer.save()
# update status for the first algo
status_1 = MLAlgorithmStatus(status = "ab_testing", created_by=instance.created_by,
parent_mlalgorithm = instance.parent_mlalgorithm_1, active=True)
status_1.save()
deactivate_other_statuses(status_1)
# update status for second algo
status_2 = MLAlgorithmStatus(status = "ab_testing", created_by=instance.created_by,
parent_mlalgorithm = instance.parent_mlalgorithm_2, active=True)
status_2.save()
deactivate_other_statuses(status_2)
except Exception as e:
raise APIException(str(e))
def add_algorithm(self, endpoint_name, algorithm_object, algorithm_name,
algorithm_status, algorithm_version, owner,
algorithm_description, algorithm_code):
# get endpoint
endpoint, _ = Endpoint.objects.get_or_create(name=endpoint_name, owner=owner)
# get algorithm
database_object, algorithm_created = MLAlgorithm.objects.get_or_create(
name=algorithm_name,
description=algorithm_description,
code=algorithm_code,
version=algorithm_version,
owner=owner,
parent_endpoint=endpoint)
if algorithm_created:
status = MLAlgorithmStatus(status = algorithm_status,
created_by = owner,
parent_mlalgorithm = database_object,
active = True)
status.save()
# add to registry
self.endpoints[database_object.id] = algorithm_object