def election_all_ballots_retrieve_view(request, election_local_id=0): """ Reach out to Google and retrieve (for one election): 1) Polling locations (so we can use those addresses to retrieve a representative set of ballots) 2) Cycle through a portion of those polling locations, enough that we are caching all of the possible ballot items :param request: :return: """ authority_required = {'admin'} # admin, verified_volunteer if not voter_has_authority(request, authority_required): return redirect_to_sign_in_page(request, authority_required) google_civic_election_id = convert_to_int( request.GET.get('google_civic_election_id', 0)) try: if positive_value_exists(election_local_id): election_on_stage = Election.objects.get(id=election_local_id) else: election_on_stage = Election.objects.get( google_civic_election_id=google_civic_election_id) election_local_id = election_on_stage.id except Election.MultipleObjectsReturned as e: handle_record_found_more_than_one_exception(e, logger=logger) messages.add_message( request, messages.ERROR, 'Could not retrieve ballot data. More than one election found.') return HttpResponseRedirect(reverse('election:election_list', args=())) except Election.DoesNotExist: messages.add_message( request, messages.ERROR, 'Could not retrieve ballot data. Election could not be found.') return HttpResponseRedirect(reverse('election:election_list', args=())) # Check to see if we have polling location data related to the region(s) covered by this election # We request the ballot data for each polling location as a way to build up our local data state = election_on_stage.get_election_state() try: polling_location_count_query = PollingLocation.objects.all() polling_location_count_query = polling_location_count_query.filter( state__iexact=state) polling_location_count = polling_location_count_query.count() polling_location_list = PollingLocation.objects.all() polling_location_list = polling_location_list.filter( state__iexact=state) # We used to have a limit of 500 ballots to pull per election, but now retrieve all # Ordering by "location_name" creates a bit of (locational) random order polling_location_list = polling_location_list.order_by( 'location_name') # [:500] except PollingLocation.DoesNotExist: messages.add_message( request, messages.INFO, 'Could not retrieve ballot data for the {election_name}. ' 'No polling locations exist for the state \'{state}\'. ' 'Data needed from VIP.'.format( election_name=election_on_stage.election_name, state=state)) return HttpResponseRedirect( reverse('election:election_summary', args=(election_local_id, ))) if polling_location_count == 0: messages.add_message( request, messages.ERROR, 'Could not retrieve ballot data for the {election_name}. ' 'No polling locations returned for the state \'{state}\'. (error 2)' .format(election_name=election_on_stage.election_name, state=state)) return HttpResponseRedirect( reverse('election:election_summary', args=(election_local_id, ))) ballots_retrieved = 0 ballots_not_retrieved = 0 ballots_with_contests_retrieved = 0 # We used to only retrieve up to 500 locations from each state, but we don't limit now # # We retrieve 10% of the total polling locations, which should give us coverage of the entire election # number_of_polling_locations_to_retrieve = int(.1 * polling_location_count) for polling_location in polling_location_list: success = False # Get the address for this polling place, and then retrieve the ballot from Google Civic API text_for_map_search = polling_location.get_text_for_map_search() one_ballot_results = retrieve_one_ballot_from_google_civic_api( text_for_map_search, election_on_stage.google_civic_election_id) if one_ballot_results['success']: one_ballot_json = one_ballot_results['structured_json'] store_one_ballot_results = store_one_ballot_from_google_civic_api( one_ballot_json, 0, polling_location.we_vote_id) if store_one_ballot_results['success']: success = True if success: ballots_retrieved += 1 else: ballots_not_retrieved += 1 if one_ballot_results['contests_retrieved']: ballots_with_contests_retrieved += 1 # We used to only retrieve up to 500 locations from each state, but we don't limit now # # Break out of this loop, assuming we have a minimum number of ballots with contests retrieved # # If we don't achieve the minimum number of ballots_with_contests_retrieved, break out at the emergency level # emergency = (ballots_retrieved + ballots_not_retrieved) >= (3 * number_of_polling_locations_to_retrieve) # if ((ballots_retrieved + ballots_not_retrieved) >= number_of_polling_locations_to_retrieve and # ballots_with_contests_retrieved > 20) or emergency: # break if ballots_retrieved > 0: total_retrieved = ballots_retrieved + ballots_not_retrieved messages.add_message( request, messages.INFO, 'Ballot data retrieved from Google Civic for the {election_name}. ' '(ballots retrieved: {ballots_retrieved} ' '(with contests: {ballots_with_contests_retrieved}), ' 'not retrieved: {ballots_not_retrieved}, ' 'total: {total})'.format( ballots_retrieved=ballots_retrieved, ballots_not_retrieved=ballots_not_retrieved, ballots_with_contests_retrieved=ballots_with_contests_retrieved, election_name=election_on_stage.election_name, total=total_retrieved)) else: messages.add_message( request, messages.ERROR, 'Ballot data NOT retrieved from Google Civic for the {election_name}.' ' (not retrieved: {ballots_not_retrieved})'.format( ballots_not_retrieved=ballots_not_retrieved, election_name=election_on_stage.election_name)) return HttpResponseRedirect( reverse('election:election_summary', args=(election_local_id, )))
def election_all_ballots_retrieve_view(request, election_local_id=0): """ Reach out to Google and retrieve (for one election): 1) Polling locations (so we can use those addresses to retrieve a representative set of ballots) 2) Cycle through a portion of those polling locations, enough that we are caching all of the possible ballot items :param request: :return: """ authority_required = {"admin"} # admin, verified_volunteer if not voter_has_authority(request, authority_required): return redirect_to_sign_in_page(request, authority_required) google_civic_election_id = request.GET.get("google_civic_election_id", 0) # # Testing # messages.add_message(request, messages.INFO, # 'election_local_id: {election_local_id}, ' # 'google_civic_election_id: {google_civic_election_id}.'.format( # election_local_id=election_local_id, # google_civic_election_id=google_civic_election_id, # )) try: if positive_value_exists(election_local_id): election_on_stage = Election.objects.get(id=election_local_id) else: election_on_stage = Election.objects.get(google_civic_election_id=google_civic_election_id) election_local_id = election_on_stage.id except Election.MultipleObjectsReturned as e: handle_record_found_more_than_one_exception(e, logger=logger) messages.add_message(request, messages.ERROR, "Could not retrieve ballot data. More than one election found.") return HttpResponseRedirect(reverse("election:election_list", args=())) except Election.DoesNotExist: messages.add_message(request, messages.ERROR, "Could not retrieve ballot data. Election could not be found.") return HttpResponseRedirect(reverse("election:election_list", args=())) # Check to see if we have polling location data related to the region(s) covered by this election # We request the ballot data for each polling location as a way to build up our local data state = election_on_stage.get_election_state() try: polling_location_count_query = PollingLocation.objects.all() polling_location_count_query = polling_location_count_query.filter(state__iexact=state) polling_location_count = polling_location_count_query.count() polling_location_list = PollingLocation.objects.all() polling_location_list = polling_location_list.filter(state__iexact=state) # Include a limit of 500 ballots to pull per election # Ordering by "location_name" creates a bit of (locational) random order polling_location_list = polling_location_list.order_by("location_name")[:500] except PollingLocation.DoesNotExist: messages.add_message( request, messages.INFO, "Could not retrieve ballot data for the {election_name}. " "No polling locations exist for the state '{state}'. " "Data needed from VIP.".format(election_name=election_on_stage.election_name, state=state), ) return HttpResponseRedirect(reverse("election:election_summary", args=(election_local_id,))) if polling_location_count == 0: messages.add_message( request, messages.ERROR, "Could not retrieve ballot data for the {election_name}. " "No polling locations returned for the state '{state}'. (error 2)".format( election_name=election_on_stage.election_name, state=state ), ) return HttpResponseRedirect(reverse("election:election_summary", args=(election_local_id,))) ballots_retrieved = 0 ballots_not_retrieved = 0 ballots_with_contests_retrieved = 0 # DEPRECATED: We now retrieve up to 500 locations from each state # # We retrieve 10% of the total polling locations, which should give us coverage of the entire election # number_of_polling_locations_to_retrieve = int(.1 * polling_location_count) for polling_location in polling_location_list: success = False # Get the address for this polling place, and then retrieve the ballot from Google Civic API text_for_map_search = polling_location.get_text_for_map_search() one_ballot_results = retrieve_one_ballot_from_google_civic_api( text_for_map_search, election_on_stage.google_civic_election_id ) if one_ballot_results["success"]: one_ballot_json = one_ballot_results["structured_json"] store_one_ballot_results = store_one_ballot_from_google_civic_api(one_ballot_json) if store_one_ballot_results["success"]: success = True # TODO: Record locally the address that failed (or succeeded) here? if success: ballots_retrieved += 1 else: ballots_not_retrieved += 1 if one_ballot_results["contests_retrieved"]: ballots_with_contests_retrieved += 1 # # DEPRECATED: We now retrieve up to 500 locations from each state # # Break out of this loop, assuming we have a minimum number of ballots with contests retrieved # # If we don't achieve the minimum number of ballots_with_contests_retrieved, break out at the emergency level # emergency = (ballots_retrieved + ballots_not_retrieved) >= (3 * number_of_polling_locations_to_retrieve) # if ((ballots_retrieved + ballots_not_retrieved) >= number_of_polling_locations_to_retrieve and # ballots_with_contests_retrieved > 20) or emergency: # break if ballots_retrieved > 0: total_retrieved = ballots_retrieved + ballots_not_retrieved messages.add_message( request, messages.INFO, "Ballot data retrieved from Google Civic for the {election_name}. " "(ballots retrieved: {ballots_retrieved} " "(with contests: {ballots_with_contests_retrieved}), " "not retrieved: {ballots_not_retrieved}, " "total: {total})".format( ballots_retrieved=ballots_retrieved, ballots_not_retrieved=ballots_not_retrieved, ballots_with_contests_retrieved=ballots_with_contests_retrieved, election_name=election_on_stage.election_name, total=total_retrieved, ), ) else: messages.add_message( request, messages.ERROR, "Ballot data NOT retrieved from Google Civic for the {election_name}." " (not retrieved: {ballots_not_retrieved})".format( ballots_not_retrieved=ballots_not_retrieved, election_name=election_on_stage.election_name ), ) return HttpResponseRedirect(reverse("election:election_summary", args=(election_local_id,)))
def election_all_ballots_retrieve_view(request, election_local_id=0): """ Reach out to Google and retrieve (for one election): 1) Polling locations (so we can use those addresses to retrieve a representative set of ballots) 2) Cycle through a portion of those polling locations, enough that we are caching all of the possible ballot items :param request: :return: """ google_civic_election_id = request.GET.get('google_civic_election_id', 0) election_on_stage = Election() # Testing messages.add_message(request, messages.INFO, 'election_local_id: {election_local_id}, ' 'google_civic_election_id: {google_civic_election_id}.'.format( election_local_id=election_local_id, google_civic_election_id=google_civic_election_id, )) try: if positive_value_exists(election_local_id): election_on_stage = Election.objects.get(id=election_local_id) else: election_on_stage = Election.objects.get(google_civic_election_id=google_civic_election_id) election_local_id = election_on_stage.id except Election.MultipleObjectsReturned as e: handle_record_found_more_than_one_exception(e, logger=logger) messages.add_message(request, messages.ERROR, 'Could not retrieve ballot data. More than one election found.') return HttpResponseRedirect(reverse('election:election_list', args=())) except Election.DoesNotExist: messages.add_message(request, messages.ERROR, 'Could not retrieve ballot data. Election could not be found.') return HttpResponseRedirect(reverse('election:election_list', args=())) # Check to see if we have polling location data related to the region(s) covered by this election # We request the ballot data for each polling location as a way to build up our local data state = election_on_stage.get_election_state() try: polling_location_count_query = PollingLocation.objects.all() polling_location_count_query = polling_location_count_query.filter(state__iexact=state) polling_location_count = polling_location_count_query.count() polling_location_list = PollingLocation.objects.all() polling_location_list = polling_location_list.filter(state__iexact=state) # polling_location_list = polling_location_list.filter(city__iexact='san francisco') # TODO DALE For testing # Include a limit of 500 ballots to pull per election polling_location_list = polling_location_list.order_by('location_name')[:500] # Creates a bit of random order except PollingLocation.DoesNotExist: messages.add_message(request, messages.INFO, 'Could not retrieve ballot data for the {election_name}. ' 'No polling locations exist for the state \'{state}\'. ' 'Data needed from VIP.'.format( election_name=election_on_stage.election_name, state=state)) return HttpResponseRedirect(reverse('election:election_summary', args=(election_local_id,))) if polling_location_count == 0: messages.add_message(request, messages.ERROR, 'Could not retrieve ballot data for the {election_name}. ' 'No polling locations returned for the state \'{state}\'. (error 2)'.format( election_name=election_on_stage.election_name, state=state)) return HttpResponseRedirect(reverse('election:election_summary', args=(election_local_id,))) ballots_retrieved = 0 ballots_not_retrieved = 0 # We retrieve 10% of the total polling locations, which should give us coverage of the entire election number_of_polling_locations_to_retrieve = int(.1 * polling_location_count) for polling_location in polling_location_list: success = False # Get the address for this polling place, and then retrieve the ballot from Google Civic API text_for_map_search = polling_location.get_text_for_map_search() one_ballot_results = retrieve_one_ballot_from_google_civic_api( text_for_map_search, election_on_stage.google_civic_election_id) if one_ballot_results['success']: one_ballot_json = one_ballot_results['structured_json'] store_one_ballot_results = store_one_ballot_from_google_civic_api(one_ballot_json) if store_one_ballot_results['success']: success = True if success: ballots_retrieved += 1 else: ballots_not_retrieved += 1 # Break out of this look if (ballots_retrieved + ballots_not_retrieved) >= number_of_polling_locations_to_retrieve: break if ballots_retrieved > 0: messages.add_message(request, messages.INFO, 'Ballot data retrieved from Google Civic for the {election_name}. ' '(ballots retrieved: {ballots_retrieved}, not retrieved: {ballots_not_retrieved}, ' 'total: {total})'.format( ballots_retrieved=ballots_retrieved, ballots_not_retrieved=ballots_not_retrieved, election_name=election_on_stage.election_name, total=number_of_polling_locations_to_retrieve)) else: messages.add_message(request, messages.ERROR, 'Ballot data NOT retrieved from Google Civic for the {election_name}.' ' (not retrieved: {ballots_not_retrieved})'.format( ballots_not_retrieved=ballots_not_retrieved, election_name=election_on_stage.election_name)) return HttpResponseRedirect(reverse('election:election_summary', args=(election_local_id,)))