def add_algorithm(self, endpoint_name, algorithm_object, algorithm_name,
algorithm_status, algorithm_version, owner,
algorithm_description, algorithm_code):
#get endpoint
endpoint, _ = Endpoint.objects.get_or_create(name=endpoint_name,
owner=owner)
#get algorithm
database_object, algorithm_created = MLAlgorithm.objects.get_or_create(
name=algorithm_name,
description=algorithm_description,
code=algorithm_code,
version=algorithm_version,
owner=owner,
parent_endpoint=endpoint)
if algorithm_created:
status = MLAlgorithmStatus(status=algorithm_status,
created_by=owner,
parent_mlalgorithm=database_object,
active=True)
status.save()
#add to registry
self.endpoints[database_object.id] = algorithm_object
def perform_create(self, serializer):
try:
with transaction.atomic():
instance = serializer.save()
# update status for first algorithm
status_1 = MLAlgorithmStatus(status = "ab_testing",
created_by=instance.created_by,
parent_mlalgorithm = instance.parent_mlalgorithm_1,
active=True)
status_1.save()
deactivate_other_statuses(status_1)
# update status for second algorithm
status_2 = MLAlgorithmStatus(status = "ab_testing",
created_by=instance.created_by,
parent_mlalgorithm = instance.parent_mlalgorithm_2,
active=True)
status_2.save()
deactivate_other_statuses(status_2)
except Exception as e:
raise APIException(str(e))
def post(self, request, ab_test_id, format=None):
try:
ab_test = ABTest.objects.get(pk=ab_test_id)
if ab_test.ended_at is not None:
return Response({"message": "AB Test already finished."})
date_now = datetime.datetime.now()
# alg #1 accuracy
all_responses_1 = MLRequest.objects.filter(
parent_mlalgorithm=ab_test.parent_mlalgorithm_1,
created_at__gt=ab_test.created_at,
created_at__lt=date_now).count()
correct_responses_1 = MLRequest.objects.filter(
parent_mlalgorithm=ab_test.parent_mlalgorithm_1,
created_at__gt=ab_test.created_at,
created_at__lt=date_now,
response=F('feedback')).count()
accuracy_1 = correct_responses_1 / float(all_responses_1)
print(all_responses_1, correct_responses_1, accuracy_1)
# alg #2 accuracy
all_responses_2 = MLRequest.objects.filter(
parent_mlalgorithm=ab_test.parent_mlalgorithm_2,
created_at__gt=ab_test.created_at,
created_at__lt=date_now).count()
correct_responses_2 = MLRequest.objects.filter(
parent_mlalgorithm=ab_test.parent_mlalgorithm_2,
created_at__gt=ab_test.created_at,
created_at__lt=date_now,
response=F('feedback')).count()
accuracy_2 = correct_responses_2 / float(all_responses_2)
print(all_responses_2, correct_responses_2, accuracy_2)
# select algorithm with higher accuracy
alg_id_1, alg_id_2 = ab_test.parent_mlalgorithm_1, ab_test.parent_mlalgorithm_2
# swap
if accuracy_1 < accuracy_2:
alg_id_1, alg_id_2 = alg_id_2, alg_id_1
status_1 = MLAlgorithmStatus(status="production",
created_by=ab_test.created_by,
parent_mlalgorithm=alg_id_1,
active=True)
status_1.save()
deactivate_other_statuses(status_1)
# update status for second algorithm
status_2 = MLAlgorithmStatus(status="testing",
created_by=ab_test.created_by,
parent_mlalgorithm=alg_id_2,
active=True)
status_2.save()
deactivate_other_statuses(status_2)
summary = "Algorithm #1 accuracy: {}, Algorithm #2 accuracy: {}".format(
accuracy_1, accuracy_2)
ab_test.ended_at = date_now
ab_test.summary = summary
ab_test.save()
except Exception as e:
return Response({
"status": "Error",
"message": str(e)
},
status=status.HTTP_400_BAD_REQUEST)
return Response({"message": "AB Test finished.", "summary": summary})