def get_submissions_for_challenge(request, challenge_pk): challenge = get_challenge_model(challenge_pk) if not is_user_a_host_of_challenge(request.user, challenge.id): response_data = { "error": "Sorry, you are not authorized to make this request!" } return Response(response_data, status=status.HTTP_400_BAD_REQUEST) submission_status = request.query_params.get("status", None) valid_submission_status = [ Submission.SUBMITTED, Submission.RUNNING, Submission.FAILED, Submission.CANCELLED, Submission.FINISHED, Submission.SUBMITTING, ] if submission_status not in valid_submission_status: response_data = { "error": "Invalid submission status {}".format(submission_status) } return Response(response_data, status=status.HTTP_400_BAD_REQUEST) submissions_done_in_challenge = Submission.objects.filter( challenge_phase__challenge=challenge.id, status=submission_status) serializer = SubmissionSerializer(submissions_done_in_challenge, many=True, context={"request": request}) return Response(serializer.data, status=status.HTTP_200_OK)
def re_run_submission(request, submission_pk): """ API endpoint to re-run a submission. Only challenge host has access to this endpoint. """ try: submission = Submission.objects.get(pk=submission_pk) except Submission.DoesNotExist: response_data = { 'error': 'Submission {} does not exist'.format(submission_pk) } return Response(response_data, status=status.HTTP_404_NOT_FOUND) # get the challenge and challenge phase object challenge_phase = submission.challenge_phase challenge = challenge_phase.challenge if not is_user_a_host_of_challenge(request.user, challenge.pk): response_data = { "error": "Only challenge hosts are allowed to re-run a submission" } return Response(response_data, status=status.HTTP_403_FORBIDDEN) if not challenge.is_active: response_data = { 'error': 'Challenge {} is not active'.format(challenge.title) } return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE) publish_submission_message(challenge.pk, challenge_phase.pk, submission.pk) response_data = { 'success': 'Submission is successfully submitted for re-running' } return Response(response_data, status=status.HTTP_200_OK)
def get_teams_and_corresponding_challenges_for_a_participant( request, challenge_pk): """ Returns list of teams and corresponding challenges for a participant """ # first get list of all the participants and teams related to the user participant_objs = Participant.objects.filter( user=request.user).prefetch_related('team') is_challenge_host = is_user_a_host_of_challenge(user=request.user, challenge_pk=challenge_pk) challenge_participated_teams = [] for participant_obj in participant_objs: participant_team = participant_obj.team challenges = Challenge.objects.filter( participant_teams=participant_team) if challenges.count(): for challenge in challenges: challenge_participated_teams.append( ChallengeParticipantTeam(challenge, participant_team)) else: challenge = None challenge_participated_teams.append( ChallengeParticipantTeam(challenge, participant_team)) serializer = ChallengeParticipantTeamListSerializer( ChallengeParticipantTeamList(challenge_participated_teams)) response_data = serializer.data response_data['is_challenge_host'] = is_challenge_host return Response(response_data, status=status.HTTP_200_OK)
def download_all_participants(request, challenge_pk): """ Returns the List of Participant Teams and its details in csv format """ if is_user_a_host_of_challenge(user=request.user, challenge_pk=challenge_pk): challenge = get_challenge_model(challenge_pk) participant_teams = challenge.participant_teams.all().order_by( "-team_name") teams = ChallengeParticipantSerializer(participant_teams, many=True, context={"request": request}) response = HttpResponse(content_type="text/csv") response[ "Content-Disposition"] = "attachment; filename=participant_teams_{0}.csv".format( challenge_pk) writer = csv.writer(response) writer.writerow([ "Team Name", "Team Members", "Email Id", ]) for team in teams.data: writer.writerow([ team["team_name"], ",".join(team["team_members"]), ",".join(team["team_members_email_ids"]), ]) return response else: response_data = { "error": "Sorry, you are not authorized to make this request" } return Response(response_data, status=status.HTTP_400_BAD_REQUEST)
def test_is_user_a_host_of_challenge_with_authenticated_not_host_user( self): self.client.force_authenticate(user=self.user) expected = False output = is_user_a_host_of_challenge(self.user, self.challenge.pk) self.assertEqual(output, expected)
def get_remaining_submissions(request, challenge_pk): ''' API to get the number of remaining submission for all phases. Below is the sample response returned by the API { "participant_team": "Sample_Participant_Team", "participant_team_id": 2, "phases": [ { "id": 1, "name": "Megan Phase", "start_date": "2018-10-28T14:22:53.022639Z", "end_date": "2020-06-19T14:22:53.022660Z", "limits": { "remaining_submissions_this_month_count": 9, "remaining_submissions_today_count": 5, "remaining_submissions_count": 29 } }, { "id": 2, "name": "Molly Phase", "start_date": "2018-10-28T14:22:53Z", "end_date": "2020-06-19T14:22:53Z", "limits": { "message": "You have exhausted this month's submission limit!", "remaining_time": "1481076.929224" // remaining_time is in seconds } } ] } ''' phases_data = {} challenge = get_challenge_model(challenge_pk) challenge_phases = ChallengePhase.objects.filter( challenge=challenge).order_by('pk') if not is_user_a_host_of_challenge(request.user, challenge_pk): challenge_phases = challenge_phases.filter( challenge=challenge, is_public=True).order_by('pk') phase_data_list = list() for phase in challenge_phases: remaining_submission_message, response_status = get_remaining_submission_for_a_phase( request.user, phase.id, challenge_pk) if response_status != status.HTTP_200_OK: return Response(remaining_submission_message, status=response_status) phase_data_list.append( RemainingSubmissionDataSerializer(phase, context={ 'limits': remaining_submission_message }).data) phases_data["phases"] = phase_data_list participant_team = get_participant_team_of_user_for_a_challenge( request.user, challenge_pk) phases_data['participant_team'] = participant_team.team_name phases_data['participant_team_id'] = participant_team.id return Response(phases_data, status=status.HTTP_200_OK)
def get_signed_url_for_submission_related_file(request): """Returns S3 signed URL for a particular file residing on S3 bucket Arguments: request {object} -- Request object Returns: Response object -- Response object with appropriate response code (200/400/403/404) """ # Assumption: file will be stored in this format: 'team_{id}/submission_{id}/.../file.log' bucket = request.query_params.get("bucket", None) key = request.query_params.get("key", None) if not bucket or not key: response_data = {"error": "key and bucket names can't be empty"} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) try: splits = key.split("/") participant_team_id, submission_id = ( splits[0].replace("team_", ""), splits[1].replace("submission_", ""), ) except Exception: response_data = { "error": "Invalid file path format. Please try again with correct file path format." } return Response(response_data, status=status.HTTP_400_BAD_REQUEST) participant_team = get_participant_team_model(participant_team_id) submission = get_submission_model(submission_id) challenge_pk = submission.challenge_phase.challenge.pk if submission.participant_team != participant_team: response_data = { "error": "You are not authorized to access this file." } return Response(response_data, status=status.HTTP_403_FORBIDDEN) if is_user_part_of_participant_team( request.user, participant_team) or is_user_a_host_of_challenge( request.user, challenge_pk): aws_keys = get_aws_credentials_for_challenge(challenge_pk) s3 = get_boto3_client("s3", aws_keys) url = s3.generate_presigned_url(ClientMethod="get_object", Params={ "Bucket": bucket, "Key": key }) response_data = {"signed_url": url} return Response(response_data, status=status.HTTP_200_OK) else: response_data = { "error": "You are not authorized to access this file." } return Response(response_data, status=status.HTTP_403_FORBIDDEN)
def download_all_submissions_file(request, challenge_pk, file_type): # To check for the corresponding challenge from challenge_pk. challenge = get_challenge_model(challenge_pk) if file_type == 'csv': if is_user_a_host_of_challenge(user=request.user, challenge_pk=challenge_pk): submissions = Submission.objects.filter( challenge_phase__challenge=challenge).order_by('-submitted_at') response = HttpResponse(content_type='text/csv') response[ 'Content-Disposition'] = 'attachment; filename=all_submissions.csv' writer = csv.writer(response) writer.writerow([ 'id', 'Team Name', 'Challenge Phase', 'Status', 'Created By', 'Execution Time(sec.)', 'Submission Number', 'Submitted File', 'Stdout File', 'Stderr File', 'Submitted At', 'Submission Result File', 'Submission Metadata File', ]) for submission in submissions: writer.writerow([ submission.id, submission.participant_team, submission.challenge_phase, submission.status, submission.created_by, submission.execution_time, submission.submission_number, submission.input_file, submission.stdout_file, submission.stderr_file, submission.created_at, submission.submission_result_file, submission.submission_metadata_file, ]) return response else: response_data = { 'error': 'Only Challenge Hosts can use the export feature!' } return Response(response_data, status=status.HTTP_400_BAD_REQUEST) else: response_data = {'error': 'The file type requested is not valid!'} return Response(response_data, status=status.HTTP_400_BAD_REQUEST)
def get_all_submissions_of_challenge(request, challenge_pk, challenge_phase_pk): """ Returns all the submissions for a particular challenge """ # To check for the corresponding challenge from challenge_pk. challenge = get_challenge_model(challenge_pk) # To check for the corresponding challenge phase from the challenge_phase_pk and challenge. try: challenge_phase = ChallengePhase.objects.get(pk=challenge_phase_pk, challenge=challenge) except ChallengePhase.DoesNotExist: response_data = {'error': 'Challenge Phase {} does not exist'.format(challenge_phase_pk)} return Response(response_data, status=status.HTTP_404_NOT_FOUND) # To check for the user as a host of the challenge from the request and challenge_pk. if is_user_a_host_of_challenge(user=request.user, challenge_pk=challenge_pk): # Filter submissions on the basis of challenge for host for now. Later on, the support for query # parameters like challenge phase, date is to be added. submissions = Submission.objects.filter(challenge_phase__challenge=challenge).order_by('-submitted_at') paginator, result_page = paginated_queryset(submissions, request) try: serializer = ChallengeSubmissionManagementSerializer(result_page, many=True, context={'request': request}) response_data = serializer.data return paginator.get_paginated_response(response_data) except: return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) # To check for the user as a participant of the challenge from the request and challenge_pk. elif has_user_participated_in_challenge(user=request.user, challenge_id=challenge_pk): # get participant team object for the user for a particular challenge. participant_team_pk = get_participant_team_id_of_user_for_a_challenge( request.user, challenge_pk) # Filter submissions on the basis of challenge phase for a participant. submissions = Submission.objects.filter(participant_team=participant_team_pk, challenge_phase=challenge_phase).order_by('-submitted_at') paginator, result_page = paginated_queryset(submissions, request) try: serializer = SubmissionSerializer(result_page, many=True, context={'request': request}) response_data = serializer.data return paginator.get_paginated_response(response_data) except: return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) # when user is neither host not participant of the challenge. else: response_data = {'error': 'You are neither host nor participant of the challenge!'} return Response(response_data, status=status.HTTP_400_BAD_REQUEST)
def get_submission_message_from_queue(request, queue_name): """ API to fetch submission message from AWS SQS queue Arguments: queue_name -- The unique authentication token provided by challenge hosts """ try: challenge = Challenge.objects.get(queue=queue_name) # noqa except Challenge.DoesNotExist: response_data = { "error": "Challenge with queue name {} does not exist".format(queue_name) } return Response(response_data, status=status.HTTP_400_BAD_REQUEST) if not is_user_a_host_of_challenge(request.user, challenge.pk): response_data = { "error": "Sorry, you are not authorized to access this resource" } return Response(response_data, status=status.HTTP_401_UNAUTHORIZED) queue = get_sqs_queue_object() try: messages = queue.receive_messages() if len(messages): message_receipt_handle = messages[0].receipt_handle message_body = eval(messages[0].body) logger.info("A submission is received with pk {}".format( message_body.get("submission_pk"))) else: logger.info("No submission received") message_receipt_handle = None message_body = None response_data = { "body": message_body, "receipt_handle": message_receipt_handle, } return Response(response_data, status=status.HTTP_200_OK) except botocore.exceptions.ClientError as ex: response_data = ex logger.exception("Exception raised: {}".format(ex)) return Response(response_data, status=status.HTTP_400_BAD_REQUEST)
def has_permission(self, request, view): if request.method in permissions.SAFE_METHODS: return True elif request.method in ["DELETE", "PATCH", "PUT", "POST"]: try: challenge = Challenge.objects.get( pk=request.parser_context["kwargs"]["challenge_pk"]) except Challenge.DoesNotExist: return False if request.user.id == challenge.creator.created_by.id: return True elif is_user_a_host_of_challenge(request.user, challenge.id): return True else: return False else: return False
def get_submissions_for_challenge(request, challenge_pk): challenge = get_challenge_model(challenge_pk) if not is_user_a_host_of_challenge(request.user, challenge.id): response_data = { "error": "Sorry, you are not authorized to make this request!" } return Response(response_data, status=status.HTTP_400_BAD_REQUEST) submissions_done_in_challenge = SubmissionFilter( request.GET, queryset=Submission.objects.filter( challenge_phase__challenge=challenge.id)) serializer = SubmissionSerializer(submissions_done_in_challenge.qs, many=True, context={'request': request}) return Response(serializer.data, status=status.HTTP_200_OK)
def delete_submission_message_from_queue(request, queue_name, receipt_handle): """ API to delete submission message from AWS SQS queue Arguments: queue_name -- The unique authentication token provided by challenge hosts receipt_handle -- The receipt handle of the message to be deleted """ try: challenge = Challenge.objects.get(queue=queue_name) except Challenge.DoesNotExist: response_data = { "error": "Challenge with queue name {} does not exists".format(queue_name) } return Response(response_data, status=status.HTTP_400_BAD_REQUEST) challenge_pk = challenge.pk if not is_user_a_host_of_challenge(request.user, challenge_pk): response_data = { "error": "Sorry, you are not authorized to access this resource" } return Response(response_data, status=status.HTTP_401_UNAUTHORIZED) queue = get_sqs_queue_object() try: message = queue.Message(receipt_handle) message.delete() response_data = { "success": "Message deleted successfully from the queue: {}".format( queue_name) } return Response(response_data, status=status.HTTP_200_OK) except botocore.exceptions.ClientError as ex: response_data = ex logger.exception( "SQS message is not deleted due to {}".format(response_data)) return Response(response_data, status=status.HTTP_400_BAD_REQUEST)
def leaderboard(request, challenge_phase_split_id): """Returns leaderboard for a corresponding Challenge Phase Split""" # check if the challenge exists or not try: challenge_phase_split = ChallengePhaseSplit.objects.get( pk=challenge_phase_split_id) except ChallengePhaseSplit.DoesNotExist: response_data = {'error': 'Challenge Phase Split does not exist'} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) # Get the leaderboard associated with the Challenge Phase Split leaderboard = challenge_phase_split.leaderboard # Get the default order by key to rank the entries on the leaderboard try: default_order_by = leaderboard.schema['default_order_by'] except: response_data = {'error': 'Sorry, Default filtering key not found in leaderboard schema!'} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) # Exclude the submissions done by members of the host team # while populating leaderboard challenge_obj = challenge_phase_split.challenge_phase.challenge challenge_hosts_emails = challenge_obj.creator.get_all_challenge_host_email() is_challenge_phase_public = challenge_phase_split.challenge_phase.is_public # Exclude the submissions from challenge host team to be displayed on the leaderboard of public phases challenge_hosts_emails = [] if not is_challenge_phase_public else challenge_hosts_emails leaderboard_data = LeaderboardData.objects.exclude( submission__created_by__email__in=challenge_hosts_emails) # Get all the successful submissions related to the challenge phase split leaderboard_data = leaderboard_data.filter( challenge_phase_split=challenge_phase_split, submission__is_public=True, submission__is_flagged=False, submission__status=Submission.FINISHED).order_by('created_at') leaderboard_data = leaderboard_data.annotate( filtering_score=RawSQL('result->>%s', (default_order_by, ), output_field=FloatField())).values( 'id', 'submission__participant_team__team_name', 'challenge_phase_split', 'result', 'filtering_score', 'leaderboard__schema', 'submission__submitted_at') sorted_leaderboard_data = sorted(leaderboard_data, key=lambda k: float(k['filtering_score']), reverse=True) distinct_sorted_leaderboard_data = [] team_list = [] for data in sorted_leaderboard_data: if data['submission__participant_team__team_name'] in team_list: continue else: distinct_sorted_leaderboard_data.append(data) team_list.append(data['submission__participant_team__team_name']) leaderboard_labels = challenge_phase_split.leaderboard.schema['labels'] for item in distinct_sorted_leaderboard_data: item['result'] = [item['result'][index] for index in leaderboard_labels] paginator, result_page = paginated_queryset( distinct_sorted_leaderboard_data, request, pagination_class=StandardResultSetPagination()) challenge_host_user = is_user_a_host_of_challenge(request.user, challenge_obj.pk) # Show the Private leaderboard only if the user is a challenge host if challenge_host_user: response_data = result_page return paginator.get_paginated_response(response_data) # Check if challenge phase leaderboard is public for participant user or not elif challenge_phase_split.visibility != ChallengePhaseSplit.PUBLIC: response_data = {'error': 'Sorry, the leaderboard is not public!'} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) else: response_data = result_page return paginator.get_paginated_response(response_data)
def change_submission_data_and_visibility(request, challenge_pk, challenge_phase_pk, submission_pk): """ API Endpoint for updating the submission meta data and changing submission visibility. """ # check if the challenge exists or not challenge = get_challenge_model(challenge_pk) # check if the challenge phase exists or not challenge_phase = get_challenge_phase_model(challenge_phase_pk) if not challenge.is_active: response_data = {'error': 'Challenge is not active'} return Response(response_data, status=status.HTTP_403_FORBIDDEN) # check if challenge phase is public and accepting solutions if not is_user_a_host_of_challenge(request.user, challenge_pk): if not challenge_phase.is_public: response_data = { 'error': 'Sorry, cannot accept submissions since challenge phase is not public'} return Response(response_data, status=status.HTTP_403_FORBIDDEN) participant_team_pk = get_participant_team_id_of_user_for_a_challenge( request.user, challenge_pk) try: participant_team = ParticipantTeam.objects.get(pk=participant_team_pk) except ParticipantTeam.DoesNotExist: response_data = {'error': 'You haven\'t participated in the challenge'} return Response(response_data, status=status.HTTP_403_FORBIDDEN) try: submission = Submission.objects.get(participant_team=participant_team, challenge_phase=challenge_phase, id=submission_pk) except Submission.DoesNotExist: response_data = {'error': 'Submission does not exist'} return Response(response_data, status=status.HTTP_403_FORBIDDEN) try: is_public = request.data['is_public'] if is_public is True: when_made_public = datetime.datetime.now() request.data['when_made_public'] = when_made_public except KeyError: pass serializer = SubmissionSerializer(submission, data=request.data, context={ 'participant_team': participant_team, 'challenge_phase': challenge_phase, 'request': request }, partial=True) if serializer.is_valid(): serializer.save() response_data = serializer.data return Response(response_data, status=status.HTTP_200_OK) else: return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def download_all_submissions(request, challenge_pk, challenge_phase_pk, file_type): # To check for the corresponding challenge from challenge_pk. challenge = get_challenge_model(challenge_pk) # To check for the corresponding challenge phase from the challenge_phase_pk and challenge. try: challenge_phase = ChallengePhase.objects.get(pk=challenge_phase_pk, challenge=challenge) except ChallengePhase.DoesNotExist: response_data = { 'error': 'Challenge Phase {} does not exist'.format(challenge_phase_pk) } return Response(response_data, status=status.HTTP_404_NOT_FOUND) if file_type == 'csv': if is_user_a_host_of_challenge(user=request.user, challenge_pk=challenge_pk): submissions = Submission.objects.filter( challenge_phase__challenge=challenge).order_by('-submitted_at') submissions = ChallengeSubmissionManagementSerializer( submissions, many=True, context={'request': request}) response = HttpResponse(content_type='text/csv') response[ 'Content-Disposition'] = 'attachment; filename=all_submissions.csv' writer = csv.writer(response) writer.writerow([ 'id', 'Team Name', 'Team Members', 'Team Members Email Id', 'Challenge Phase', 'Status', 'Created By', 'Execution Time(sec.)', 'Submission Number', 'Submitted File', 'Stdout File', 'Stderr File', 'Submitted At', 'Submission Result File', 'Submission Metadata File', ]) for submission in submissions.data: writer.writerow([ submission['id'], submission['participant_team'], ",".join(username['username'] for username in submission['participant_team_members']), ",".join( email['email'] for email in submission['participant_team_members']), submission['challenge_phase'], submission['status'], submission['created_by'], submission['execution_time'], submission['submission_number'], submission['input_file'], submission['stdout_file'], submission['stderr_file'], submission['created_at'], submission['submission_result_file'], submission['submission_metadata_file'], ]) return response elif has_user_participated_in_challenge(user=request.user, challenge_id=challenge_pk): # get participant team object for the user for a particular challenge. participant_team_pk = get_participant_team_id_of_user_for_a_challenge( request.user, challenge_pk) # Filter submissions on the basis of challenge phase for a participant. submissions = Submission.objects.filter( participant_team=participant_team_pk, challenge_phase=challenge_phase).order_by('-submitted_at') submissions = ChallengeSubmissionManagementSerializer( submissions, many=True, context={'request': request}) response = HttpResponse(content_type='text/csv') response[ 'Content-Disposition'] = 'attachment; filename=all_submissions.csv' writer = csv.writer(response) writer.writerow([ 'Team Name', 'Method Name', 'Status', 'Execution Time(sec.)', 'Submitted File', 'Result File', 'Stdout File', 'Stderr File', 'Submitted At', ]) for submission in submissions.data: writer.writerow([ submission['participant_team'], submission['method_name'], submission['status'], submission['execution_time'], submission['input_file'], submission['submission_result_file'], submission['stdout_file'], submission['stderr_file'], submission['created_at'], ]) return response else: response_data = { 'error': 'You are neither host nor participant of the challenge!' } return Response(response_data, status=status.HTTP_400_BAD_REQUEST) else: response_data = {'error': 'The file type requested is not valid!'} return Response(response_data, status=status.HTTP_400_BAD_REQUEST)
def calculate_distinct_sorted_leaderboard_data(user, challenge_obj, challenge_phase_split, only_public_entries): """ Function to calculate and return the sorted leaderboard data Arguments: user {[Class object]} -- User model object challenge_obj {[Class object]} -- Challenge model object challenge_phase_split {[Class object]} -- Challenge phase split model object only_public_entries {[Boolean]} -- Boolean value to determine if the user wants to include private entries or not Returns: [list] -- Ranked list of participant teams to be shown on leaderboard [status] -- HTTP status code (200/400) """ # Get the leaderboard associated with the Challenge Phase Split leaderboard = challenge_phase_split.leaderboard # Get the default order by key to rank the entries on the leaderboard try: default_order_by = leaderboard.schema["default_order_by"] except KeyError: response_data = { "error": "Sorry, default_order_by key is missing in leaderboard schema!" } return response_data, status.HTTP_400_BAD_REQUEST # Exclude the submissions done by members of the host team # while populating leaderboard challenge_hosts_emails = ( challenge_obj.creator.get_all_challenge_host_email()) is_challenge_phase_public = challenge_phase_split.challenge_phase.is_public # Exclude the submissions from challenge host team to be displayed on the leaderboard of public phases challenge_hosts_emails = ([] if not is_challenge_phase_public else challenge_hosts_emails) challenge_host_user = is_user_a_host_of_challenge(user, challenge_obj.pk) all_banned_email_ids = challenge_obj.banned_email_ids # Check if challenge phase leaderboard is public for participant user or not if (challenge_phase_split.visibility != ChallengePhaseSplit.PUBLIC and not challenge_host_user): response_data = {"error": "Sorry, the leaderboard is not public!"} return response_data, status.HTTP_400_BAD_REQUEST leaderboard_data = LeaderboardData.objects.exclude( Q(submission__created_by__email__in=challenge_hosts_emails) & Q(submission__is_baseline=False)) # Get all the successful submissions related to the challenge phase split leaderboard_data = leaderboard_data.filter( challenge_phase_split=challenge_phase_split, submission__is_flagged=False, submission__status=Submission.FINISHED, ).order_by("-created_at") leaderboard_data = leaderboard_data.annotate( filtering_score=RawSQL("result->>%s", (default_order_by, ), output_field=FloatField()), filtering_error=RawSQL( "error->>%s", ("error_{0}".format(default_order_by), ), output_field=FloatField(), ), ).values( "id", "submission__participant_team", "submission__participant_team__team_name", "submission__participant_team__team_url", "submission__is_baseline", "submission__is_public", "challenge_phase_split", "result", "error", "filtering_score", "filtering_error", "leaderboard__schema", "submission__submitted_at", "submission__method_name", ) if only_public_entries: if challenge_phase_split.visibility == ChallengePhaseSplit.PUBLIC: leaderboard_data = leaderboard_data.filter( submission__is_public=True) all_banned_participant_team = [] for leaderboard_item in leaderboard_data: participant_team_id = leaderboard_item["submission__participant_team"] participant_team = ParticipantTeam.objects.get(id=participant_team_id) all_participants_email_ids = ( participant_team.get_all_participants_email()) for participant_email in all_participants_email_ids: if participant_email in all_banned_email_ids: all_banned_participant_team.append(participant_team_id) break if leaderboard_item["error"] is None: leaderboard_item.update(filtering_error=0) if challenge_phase_split.show_leaderboard_by_latest_submission: sorted_leaderboard_data = leaderboard_data else: sorted_leaderboard_data = sorted( leaderboard_data, key=lambda k: ( float(k["filtering_score"]), float(-k["filtering_error"]), ), reverse=True if challenge_phase_split.is_leaderboard_order_descending else False, ) distinct_sorted_leaderboard_data = [] team_list = [] for data in sorted_leaderboard_data: if (data["submission__participant_team__team_name"] in team_list or data["submission__participant_team"] in all_banned_participant_team): continue elif data["submission__is_baseline"] is True: distinct_sorted_leaderboard_data.append(data) else: distinct_sorted_leaderboard_data.append(data) team_list.append(data["submission__participant_team__team_name"]) leaderboard_labels = challenge_phase_split.leaderboard.schema["labels"] for item in distinct_sorted_leaderboard_data: item["result"] = [ item["result"][index] for index in leaderboard_labels ] if item["error"] is not None: item["error"] = [ item["error"]["error_{0}".format(index)] for index in leaderboard_labels ] return distinct_sorted_leaderboard_data, status.HTTP_200_OK
def challenge_submission(request, challenge_id, challenge_phase_id): """API Endpoint for making a submission to a challenge""" # check if the challenge exists or not try: challenge = Challenge.objects.get(pk=challenge_id) except Challenge.DoesNotExist: response_data = {"error": "Challenge does not exist"} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) # check if the challenge phase exists or not try: challenge_phase = ChallengePhase.objects.get(pk=challenge_phase_id, challenge=challenge) except ChallengePhase.DoesNotExist: response_data = {"error": "Challenge Phase does not exist"} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) if request.method == "GET": # getting participant team object for the user for a particular challenge. participant_team_id = get_participant_team_id_of_user_for_a_challenge( request.user, challenge_id) # check if participant team exists or not. try: ParticipantTeam.objects.get(pk=participant_team_id) except ParticipantTeam.DoesNotExist: response_data = { "error": "You haven't participated in the challenge" } return Response(response_data, status=status.HTTP_403_FORBIDDEN) submission = Submission.objects.filter( participant_team=participant_team_id, challenge_phase=challenge_phase, ).order_by("-submitted_at") filtered_submissions = SubmissionFilter(request.GET, queryset=submission) paginator, result_page = paginated_queryset(filtered_submissions.qs, request) serializer = SubmissionSerializer(result_page, many=True, context={"request": request}) response_data = serializer.data return paginator.get_paginated_response(response_data) elif request.method == "POST": # check if the challenge is active or not if not challenge.is_active: response_data = {"error": "Challenge is not active"} return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE) # check if challenge phase is active if not challenge_phase.is_active: response_data = { "error": "Sorry, cannot accept submissions since challenge phase is not active" } return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE) # check if user is a challenge host or a participant if not is_user_a_host_of_challenge(request.user, challenge_id): # check if challenge phase is public and accepting solutions if not challenge_phase.is_public: response_data = { "error": "Sorry, cannot accept submissions since challenge phase is not public" } return Response(response_data, status=status.HTTP_403_FORBIDDEN) # if allowed email ids list exist, check if the user exist in that list or not if challenge_phase.allowed_email_ids: if request.user.email not in challenge_phase.allowed_email_ids: response_data = { "error": "Sorry, you are not allowed to participate in this challenge phase" } return Response(response_data, status=status.HTTP_403_FORBIDDEN) participant_team_id = get_participant_team_id_of_user_for_a_challenge( request.user, challenge_id) try: participant_team = ParticipantTeam.objects.get( pk=participant_team_id) except ParticipantTeam.DoesNotExist: response_data = { "error": "You haven't participated in the challenge" } return Response(response_data, status=status.HTTP_403_FORBIDDEN) all_participants_email = participant_team.get_all_participants_email() for participant_email in all_participants_email: if participant_email in challenge.banned_email_ids: message = "You're a part of {} team and it has been banned from this challenge. \ Please contact the challenge host.".format( participant_team.team_name) response_data = {"error": message} return Response(response_data, status=status.HTTP_403_FORBIDDEN) # Fetch the number of submissions under progress. submissions_in_progress_status = [ Submission.SUBMITTED, Submission.SUBMITTING, Submission.RUNNING, ] submissions_in_progress = Submission.objects.filter( participant_team=participant_team_id, challenge_phase=challenge_phase, status__in=submissions_in_progress_status, ).count() if (submissions_in_progress >= challenge_phase.max_concurrent_submissions_allowed): message = "You have {} submissions that are being processed. \ Please wait for them to finish and then try again." response_data = {"error": message.format(submissions_in_progress)} return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE) if not request.FILES: if not is_url_valid(request.data['file_url']): response_data = {'error': 'The file URL does not exists!'} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) download_file_and_publish_submission_message.delay( request.data, request.user.id, request.method, challenge_phase_id) response_data = { 'message': 'Please wait while your submission being evaluated!' } return Response(response_data, status=status.HTTP_200_OK) serializer = SubmissionSerializer( data=request.data, context={ "participant_team": participant_team, "challenge_phase": challenge_phase, "request": request, }, ) if serializer.is_valid(): serializer.save() response_data = serializer.data submission = serializer.instance # publish message in the queue publish_submission_message(challenge_id, challenge_phase_id, submission.id) return Response(response_data, status=status.HTTP_201_CREATED) return Response(serializer.errors, status=status.HTTP_406_NOT_ACCEPTABLE)
def leaderboard(request, challenge_phase_split_id): """Returns leaderboard for a corresponding Challenge Phase Split""" # check if the challenge exists or not try: challenge_phase_split = ChallengePhaseSplit.objects.get( pk=challenge_phase_split_id) except ChallengePhaseSplit.DoesNotExist: response_data = {"error": "Challenge Phase Split does not exist"} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) # Get the leaderboard associated with the Challenge Phase Split leaderboard = challenge_phase_split.leaderboard # Get the default order by key to rank the entries on the leaderboard try: default_order_by = leaderboard.schema["default_order_by"] except KeyError: response_data = { "error": "Sorry, Default filtering key not found in leaderboard schema!" } return Response(response_data, status=status.HTTP_400_BAD_REQUEST) # Exclude the submissions done by members of the host team # while populating leaderboard challenge_obj = challenge_phase_split.challenge_phase.challenge challenge_hosts_emails = ( challenge_obj.creator.get_all_challenge_host_email()) is_challenge_phase_public = challenge_phase_split.challenge_phase.is_public # Exclude the submissions from challenge host team to be displayed on the leaderboard of public phases challenge_hosts_emails = ([] if not is_challenge_phase_public else challenge_hosts_emails) challenge_host_user = is_user_a_host_of_challenge(request.user, challenge_obj.pk) # Check if challenge phase leaderboard is public for participant user or not if (challenge_phase_split.visibility != ChallengePhaseSplit.PUBLIC and not challenge_host_user): response_data = {"error": "Sorry, the leaderboard is not public!"} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) leaderboard_data = LeaderboardData.objects.exclude( Q(submission__created_by__email__in=challenge_hosts_emails) & Q(submission__is_baseline=False)) # Get all the successful submissions related to the challenge phase split leaderboard_data = leaderboard_data.filter( challenge_phase_split=challenge_phase_split, submission__is_flagged=False, submission__status=Submission.FINISHED, ).order_by("created_at") leaderboard_data = leaderboard_data.annotate( filtering_score=RawSQL("result->>%s", (default_order_by, ), output_field=FloatField())).values( "id", "submission__participant_team__team_name", "submission__participant_team__team_url", "submission__is_baseline", "challenge_phase_split", "result", "filtering_score", "leaderboard__schema", "submission__submitted_at", "submission__method_name", ) if challenge_phase_split.visibility == ChallengePhaseSplit.PUBLIC: leaderboard_data = leaderboard_data.filter(submission__is_public=True) sorted_leaderboard_data = sorted( leaderboard_data, key=lambda k: float(k["filtering_score"]), reverse=True, ) distinct_sorted_leaderboard_data = [] team_list = [] for data in sorted_leaderboard_data: if data["submission__participant_team__team_name"] in team_list: continue elif data["submission__is_baseline"] is True: distinct_sorted_leaderboard_data.append(data) else: distinct_sorted_leaderboard_data.append(data) team_list.append(data["submission__participant_team__team_name"]) leaderboard_labels = challenge_phase_split.leaderboard.schema["labels"] for item in distinct_sorted_leaderboard_data: item["result"] = [ item["result"][index] for index in leaderboard_labels ] paginator, result_page = paginated_queryset( distinct_sorted_leaderboard_data, request, pagination_class=StandardResultSetPagination(), ) response_data = result_page return paginator.get_paginated_response(response_data)
def update_submission(request, challenge_pk): """ API endpoint to update submission related attributes Query Parameters: - ``challenge_phase``: challenge phase id, e.g. 123 (**required**) - ``submission``: submission id, e.g. 123 (**required**) - ``stdout``: Stdout after evaluation, e.g. "Evaluation completed in 2 minutes" (**required**) - ``stderr``: Stderr after evaluation, e.g. "Failed due to incorrect file format" (**required**) - ``submission_status``: Status of submission after evaluation (can take one of the following values: `FINISHED`/`CANCELLED`/`FAILED`), e.g. FINISHED (**required**) - ``result``: contains accuracies for each metric, (**required**) e.g. [ { "split": "split1-codename", "show_to_participant": True, "accuracies": { "metric1": 90 } }, { "split": "split2-codename", "show_to_participant": False, "accuracies": { "metric1": 50, "metric2": 40 } } ] - ``metadata``: Contains the metadata related to submission (only visible to challenge hosts) e.g: { "average-evaluation-time": "5 sec", "foo": "bar" } """ if not is_user_a_host_of_challenge(request.user, challenge_pk): response_data = {'error': 'Sorry, you are not authorized to make this request!'} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) challenge_phase_pk = request.data.get('challenge_phase') submission_pk = request.data.get('submission') submission_status = request.data.get('submission_status', '').lower() stdout_content = request.data.get('stdout', '') stderr_content = request.data.get('stderr', '') submission_result = request.data.get('result', '') metadata = request.data.get('metadata', '') submission = get_submission_model(submission_pk) public_results = [] successful_submission = True if submission_status == Submission.FINISHED else False if submission_status not in [Submission.FAILED, Submission.CANCELLED, Submission.FINISHED]: response_data = {'error': 'Sorry, submission status is invalid'} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) if successful_submission: try: results = json.loads(submission_result) except ValueError: response_data = {'error': '`result` key contains invalid data. Please try again with correct format!'} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) leaderboard_data_list = [] for phase_result in results: split = phase_result.get('split') accuracies = phase_result.get('accuracies') show_to_participant = phase_result.get('show_to_participant', False) try: challenge_phase_split = ChallengePhaseSplit.objects.get( challenge_phase__pk=challenge_phase_pk, dataset_split__codename=split) except ChallengePhaseSplit.DoesNotExist: response_data = {'error': 'Challenge Phase Split does not exist with phase_id: {} and' 'split codename: {}'.format(challenge_phase_pk, split)} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) leaderboard_metrics = challenge_phase_split.leaderboard.schema.get('labels') missing_metrics = [] malformed_metrics = [] for metric, value in accuracies.items(): if metric not in leaderboard_metrics: missing_metrics.append(metric) if not (isinstance(value, float) or isinstance(value, int)): malformed_metrics.append((metric, type(value))) if len(missing_metrics): response_data = {'error': 'Following metrics are missing in the' 'leaderboard data: {}'.format(missing_metrics)} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) if len(malformed_metrics): response_data = {'error': 'Values for following metrics are not of' 'float/int: {}'.format(malformed_metrics)} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) data = {'result': accuracies} serializer = CreateLeaderboardDataSerializer( data=data, context={ 'challenge_phase_split': challenge_phase_split, 'submission': submission, 'request': request, } ) if serializer.is_valid(): leaderboard_data_list.append(serializer) else: return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) # Only after checking if the serializer is valid, append the public split results to results file if show_to_participant: public_results.append(accuracies) try: with transaction.atomic(): for serializer in leaderboard_data_list: serializer.save() except IntegrityError: logger.exception('Failed to update submission_id {} related metadata'.format(submission_pk)) response_data = {'error': 'Failed to update submission_id {} related metadata'.format(submission_pk)} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) submission.status = submission_status submission.completed_at = timezone.now() submission.stdout_file.save('stdout.txt', ContentFile(stdout_content)) submission.stderr_file.save('stderr.txt', ContentFile(stderr_content)) submission.submission_result_file.save('submission_result.json', ContentFile(str(public_results))) submission.submission_metadata_file.save('submission_metadata_file.json', ContentFile(str(metadata))) submission.save() response_data = {'success': 'Submission result has been successfully updated'} return Response(response_data, status=status.HTTP_200_OK)
def test_is_user_a_host_of_challenge_with_anonymous_user(self): expected = False user = AnonymousUser() output = is_user_a_host_of_challenge(user, self.challenge.pk) self.assertEqual(output, expected)
def test_is_user_a_host_of_challenge_with_authenticated_host_user(self): expected = True output = is_user_a_host_of_challenge(self.host_user, self.challenge.pk) self.assertEqual(output, expected)
def challenge_submission(request, challenge_id, challenge_phase_id): """API Endpoint for making a submission to a challenge""" # check if the challenge exists or not try: challenge = Challenge.objects.get(pk=challenge_id) except Challenge.DoesNotExist: response_data = {'error': 'Challenge does not exist'} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) # check if the challenge phase exists or not try: challenge_phase = ChallengePhase.objects.get( pk=challenge_phase_id, challenge=challenge) except ChallengePhase.DoesNotExist: response_data = {'error': 'Challenge Phase does not exist'} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) if request.method == 'GET': # getting participant team object for the user for a particular challenge. participant_team_id = get_participant_team_id_of_user_for_a_challenge( request.user, challenge_id) # check if participant team exists or not. try: ParticipantTeam.objects.get(pk=participant_team_id) except ParticipantTeam.DoesNotExist: response_data = {'error': 'You haven\'t participated in the challenge'} return Response(response_data, status=status.HTTP_403_FORBIDDEN) submission = Submission.objects.filter(participant_team=participant_team_id, challenge_phase=challenge_phase).order_by('-submitted_at') paginator, result_page = paginated_queryset(submission, request) try: serializer = SubmissionSerializer(result_page, many=True, context={'request': request}) response_data = serializer.data return paginator.get_paginated_response(response_data) except: return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) elif request.method == 'POST': # check if the challenge is active or not if not challenge.is_active: response_data = {'error': 'Challenge is not active'} return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE) # check if challenge phase is active if not challenge_phase.is_active: response_data = { 'error': 'Sorry, cannot accept submissions since challenge phase is not active'} return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE) # check if user is a challenge host or a participant if not is_user_a_host_of_challenge(request.user, challenge_id): # check if challenge phase is public and accepting solutions if not challenge_phase.is_public: response_data = { 'error': 'Sorry, cannot accept submissions since challenge phase is not public'} return Response(response_data, status=status.HTTP_403_FORBIDDEN) participant_team_id = get_participant_team_id_of_user_for_a_challenge( request.user, challenge_id) try: participant_team = ParticipantTeam.objects.get(pk=participant_team_id) except ParticipantTeam.DoesNotExist: response_data = {'error': 'You haven\'t participated in the challenge'} return Response(response_data, status=status.HTTP_403_FORBIDDEN) # Fetch the number of submissions under progress. submissions_in_progress_status = [Submission.SUBMITTED, Submission.SUBMITTING, Submission.RUNNING] submissions_in_progress = Submission.objects.filter( participant_team=participant_team_id, challenge_phase=challenge_phase, status__in=submissions_in_progress_status).count() if submissions_in_progress >= challenge_phase.max_concurrent_submissions_allowed: message = 'You have {} submissions that are being processed. \ Please wait for them to finish and then try again.' response_data = {'error': message.format(submissions_in_progress)} return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE) serializer = SubmissionSerializer(data=request.data, context={'participant_team': participant_team, 'challenge_phase': challenge_phase, 'request': request }) if serializer.is_valid(): serializer.save() response_data = serializer.data submission = serializer.instance # publish message in the queue publish_submission_message(challenge_id, challenge_phase_id, submission.id) return Response(response_data, status=status.HTTP_201_CREATED) return Response(serializer.errors, status=status.HTTP_406_NOT_ACCEPTABLE)
def change_submission_data_and_visibility(request, challenge_pk, challenge_phase_pk, submission_pk): """ API Endpoint for updating the submission meta data and changing submission visibility. """ # check if the challenge exists or not challenge = get_challenge_model(challenge_pk) # check if the challenge phase exists or not challenge_phase = get_challenge_phase_model(challenge_phase_pk) if not challenge.is_active: response_data = {"error": "Challenge is not active"} return Response(response_data, status=status.HTTP_403_FORBIDDEN) # check if challenge phase is public and accepting solutions if not is_user_a_host_of_challenge(request.user, challenge_pk): if not challenge_phase.is_public: response_data = { "error": "Sorry, cannot accept submissions since challenge phase is not public" } return Response(response_data, status=status.HTTP_403_FORBIDDEN) elif request.data.get("is_baseline"): response_data = { "error": "Sorry, you are not authorized to make this request" } return Response(response_data, status=status.HTTP_400_BAD_REQUEST) participant_team_pk = get_participant_team_id_of_user_for_a_challenge( request.user, challenge_pk) try: participant_team = ParticipantTeam.objects.get(pk=participant_team_pk) except ParticipantTeam.DoesNotExist: response_data = {"error": "You haven't participated in the challenge"} return Response(response_data, status=status.HTTP_403_FORBIDDEN) try: submission = Submission.objects.get( participant_team=participant_team, challenge_phase=challenge_phase, id=submission_pk, ) except Submission.DoesNotExist: response_data = {"error": "Submission does not exist"} return Response(response_data, status=status.HTTP_403_FORBIDDEN) try: is_public = request.data["is_public"] if is_public is True: when_made_public = datetime.datetime.now() request.data["when_made_public"] = when_made_public except KeyError: pass serializer = SubmissionSerializer( submission, data=request.data, context={ "participant_team": participant_team, "challenge_phase": challenge_phase, "request": request, }, partial=True, ) if serializer.is_valid(): serializer.save() response_data = serializer.data return Response(response_data, status=status.HTTP_200_OK) else: return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)