Beispiel #1
0
    def setUp(self):
        Party(abbreviation='PT', name='PARTIDO DOS TRABALHADORES').save()
        p = Party.objects.filter(abbreviation='PT').get()

        Politician(id_site=4400,
                   party=p,
                   name="JOAO NUNES",
                   name_full="JOAO NUNES SILVA",
                   sex='m',
                   birth_date=datetime(year=2018, month=1, day=1)).save()
Beispiel #2
0
    def load_politicians(self):
        response = requests.get("{}/api/v2/deputados".format(self._host),
                                params={"itens": 1000})
        result = response.json()

        for data in result["dados"]:
            politician = self.get_by_external_id(data["id"], Politician.DEPUTY)
            if not politician:
                politician = Politician(role=Politician.DEPUTY)

            politician.picture = data["urlFoto"]
            politician.name = self._capitalize_name(data["nome"])
            politician.external_id = data["id"]
            politician.party = self._get_party_by_initial(data["siglaPartido"])
            politician.role_state = data["siglaUf"]

            politician.save()
Beispiel #3
0
    def load_politicians(self):
        response = requests.get("{}/dadosabertos/senador/lista/atual".format(
            self._host))
        result = xmltodict.parse(response.content.decode())

        politicians = result["ListaParlamentarEmExercicio"]["Parlamentares"]
        for data in politicians["Parlamentar"]:
            identity = data["IdentificacaoParlamentar"]

            politician = self.get_by_external_id(identity["CodigoParlamentar"],
                                                 Politician.SENATOR)
            if not politician:
                politician = Politician(role=Politician.SENATOR)

            politician.picture = identity["UrlFotoParlamentar"]
            politician.name = identity["NomeCompletoParlamentar"]
            politician.external_id = identity["CodigoParlamentar"]
            politician.party = self._get_party_by_initial(
                identity["SiglaPartidoParlamentar"])
            politician.role_state = identity["UfParlamentar"]

            politician.save()
Beispiel #4
0
def google_civic_get_or_create_politician(google_civic_candidate_campaign_entry):
    error_result = False
    ##########################
    # Does this politician exist locally?
    politician_on_stage = Politician()
    politician_on_stage_found = False
    first_name_guess = google_civic_candidate_campaign_entry.name.partition(' ')[0]
    last_name_guess = google_civic_candidate_campaign_entry.name.partition(' ')[-1]
    try:
        # print "We are searching based on full_name_google_civic"
        query1 = Politician.objects.all()
        query1 = query1.filter(full_name_google_civic=google_civic_candidate_campaign_entry.name)

        # Was at least one existing entry found based on the above criteria?
        if len(query1):
            politician_on_stage = query1[0]
            politician_on_stage_found = True
            if len(query1) > 1:
                # We have confusion, so skip processing this google_civic_candidate_campaign_entry
                print "More than one Politician found (query1)"
        else:
            print "No politician found based on full_name_google_civic: {name}".format(
                name=google_civic_candidate_campaign_entry.name)
    except Exception as e:
        handle_record_not_found_exception(e)

    if not politician_on_stage_found:
        # No entries were found, so we need to
        # a) Search more deeply
        # Searching based on full_name_assembled
        print "Searching against full_name_assembled: {name}".format(name=google_civic_candidate_campaign_entry.name)
        # TODO DALE 2015-05-02 With this code, if we had imported a "Betty T. Yee" from another non-google-civic
        #  source (where full_name_google_civic was empty), we would create a second Politician entry. Fix this.
        try:
            politician_query_full_name_assembled = Politician.objects.all()
            politician_query_full_name_assembled = politician_query_full_name_assembled.filter(
                full_name_assembled=google_civic_candidate_campaign_entry.name)

            if len(politician_query_full_name_assembled):
                politician_on_stage = politician_query_full_name_assembled[0]
                politician_on_stage_found = True
            else:
                print "No politician found based on full_name_assembled: {name}".format(
                    name=google_civic_candidate_campaign_entry.name)
        except Exception as e:
            handle_record_not_found_exception(e)

    if not politician_on_stage_found:
        # No entries were found, so we need to
        # a) Search more deeply
        print "first_name_guess: {first_name_guess}, last_name_guess: {last_name_guess}".format(
            first_name_guess=first_name_guess, last_name_guess=last_name_guess)
        # TODO DALE 2015-05-02 With this code, if we had imported a "Betty T. Yee" from another non-google-civic
        #  source (where full_name_google_civic was empty), we would create a second Politician entry. Fix this.
        try:
            politician_query_first_last_guess = Politician.objects.all()
            politician_query_first_last_guess = politician_query_first_last_guess.filter(first_name=first_name_guess)
            politician_query_first_last_guess = politician_query_first_last_guess.filter(last_name=last_name_guess)

            if len(politician_query_first_last_guess):
                politician_on_stage = politician_query_first_last_guess[0]
                politician_on_stage_found = True
            else:
                print "No politician found based on first_name_guess: {first_name} " \
                      "and last_name_guess: {last_name}".format(first_name=first_name_guess,
                                                                last_name=last_name_guess)
        except Exception as e:
            handle_record_not_found_exception(e)

    try:
        if politician_on_stage_found:
            # We found a match, and want to update the Politician data to match how Google Civic references the name
            # print "Store google_civic_candidate_campaign_entry.name in Politician.full_name_google_civic"
            politician_on_stage.full_name_google_civic = google_civic_candidate_campaign_entry.name
        else:
            # print "Create Politician entry: {name}".format(name=google_civic_candidate_campaign_entry.name)
            politician_on_stage = Politician(
                # Do not save first_name or last_name because middle initials will throw this off
                last_name=last_name_guess,
                first_name=first_name_guess,
                full_name_google_civic=google_civic_candidate_campaign_entry.name,
            )
        politician_on_stage.save()
    except Exception as e:
        handle_record_not_saved_exception(e)

    if error_result:
        print "There was an error trying to create a politician"
    # else:
        # print "It seems we have found a politician: {display_full_name}".format(
        # display_full_name=politician_on_stage.display_full_name())
        # print "It seems we have found a politician: "+str(politician_on_stage.display_full_name())
        # print "It seems we found or created a politician."

    results = {
        'error_result': error_result,
        'politician_on_stage': politician_on_stage,
    }
    return results
Beispiel #5
0
def transfer_theunitedstatesio_cached_data_to_wevote_tables():
    """
    In this method, we take the cached theunitedstatesio data and move it into the core We Vote data
    :return:
    """
    print "Running transfer_theunitedstatesio_cached_data_to_wevote_tables()"

    legislators_current_query = TheUnitedStatesIoLegislatorCurrent.objects.all()
    # Only retrieve entries that haven't been processed yet
    # legislators_current_query = legislators_current_query.filter(was_processed=False)

    for legislator_current_entry in legislators_current_query:
        print 'Transferring: ' + str(legislator_current_entry.id) + ':' \
              + legislator_current_entry.first_name + ' ' + legislator_current_entry.last_name
        politician_entry_found = False

        #########################
        # Search the Politician's table to see if we already have an entry for this person
        # Do we have a record of this politician based on id_bioguide?
        if legislator_current_entry.bioguide_id != "":
            try:
                # Try to find earlier version based on the bioguide identifier
                query1 = Politician.objects.all()
                query1 = query1.filter(id_bioguide__exact=legislator_current_entry.bioguide_id)

                # Was at least one existing entry found based on the above criteria?
                if len(query1):
                    politician_entry = query1[0]
                    politician_entry_found = True
            except Exception as e:
                handle_record_not_found_exception(e)

        if not politician_entry_found:
            # TheUnitedStatesIoLegislatorCurrent was not found based on bioguide id
            # ...so check to see if we have a record of this legislator based on govtrack id?
            if legislator_current_entry.govtrack_id != "":
                try:
                    query1 = Politician.objects.all()
                    query1 = query1.filter(id_govtrack__exact=legislator_current_entry.govtrack_id)

                    # Was at least one existing entry found based on the above criteria?
                    if len(query1):
                        politician_entry = query1[0]
                        print "FOUND"
                        politician_entry_found = True
                    else:
                        print "NOT FOUND"
                except Exception as e:
                    handle_record_not_found_exception(e)

        if not politician_entry_found:
            # TheUnitedStatesIoLegislatorCurrent was not found based on id_govtrack
            # ...so check to see if we have a record of this legislator based on full_name_google_civic
            if legislator_current_entry.first_name and legislator_current_entry.last_name:
                try:
                    full_name_assembled_guess = \
                        legislator_current_entry.first_name+" "+legislator_current_entry.last_name
                    print "Searching for existing full_name_google_civic: {full_name_assembled}".format(
                        full_name_assembled=full_name_assembled_guess)
                    query1 = Politician.objects.all()
                    query1 = query1.filter(full_name_google_civic=full_name_assembled_guess)

                    # Was at least one existing entry found based on the above criteria?
                    if len(query1):
                        politician_entry = query1[0]
                        print "FOUND"
                        politician_entry_found = True
                    else:
                        print "NOT FOUND"
                except Exception as e:
                    handle_record_not_found_exception(e)

        if not politician_entry_found:
            # TheUnitedStatesIoLegislatorCurrent was not found based on full_name_google_civic
            # ...so check to see if we have a record of this legislator based on full_name_assembled
            if legislator_current_entry.first_name and legislator_current_entry.last_name:
                try:
                    full_name_assembled_guess = \
                        legislator_current_entry.first_name+" "+legislator_current_entry.last_name
                    print "Searching for existing full_name_assembled: {full_name_assembled}".format(
                        full_name_assembled=full_name_assembled_guess)
                    query1 = Politician.objects.all()
                    query1 = query1.filter(full_name_assembled=full_name_assembled_guess)

                    # Was at least one existing entry found based on the above criteria?
                    if len(query1):
                        politician_entry = query1[0]
                        print "FOUND"
                        politician_entry_found = True
                    else:
                        print "NOT FOUND"
                except Exception as e:
                    handle_record_not_found_exception(e)

        if not politician_entry_found:
            # TheUnitedStatesIoLegislatorCurrent was not found
            # ...so create a new entry
            politician_entry = Politician(
                last_name=legislator_current_entry.last_name,
                first_name=legislator_current_entry.first_name,
                full_name_assembled=legislator_current_entry.first_name+" "+legislator_current_entry.last_name,
            )

        politician_entry.last_name = legislator_current_entry.last_name
        politician_entry.first_name = legislator_current_entry.first_name
        politician_entry.full_name_assembled = \
            legislator_current_entry.first_name+" "+legislator_current_entry.last_name
        politician_entry.birth_date = legislator_current_entry.birthday
        politician_entry.gender = legislator_current_entry.gender
        politician_entry.id_bioguide = legislator_current_entry.bioguide_id
        politician_entry.id_thomas = legislator_current_entry.thomas_id
        politician_entry.id_opensecrets = legislator_current_entry.opensecrets_id
        politician_entry.id_lis = legislator_current_entry.lis_id
        politician_entry.id_cspan = legislator_current_entry.cspan_id
        politician_entry.id_govtrack = legislator_current_entry.govtrack_id
        politician_entry.id_votesmart = legislator_current_entry.votesmart_id
        politician_entry.id_ballotpedia = legislator_current_entry.ballotpedia_id
        politician_entry.id_washington_post = legislator_current_entry.washington_post_id
        politician_entry.id_icpsr = legislator_current_entry.icpsr_id
        politician_entry.id_wikipedia = legislator_current_entry.wikipedia_id

        # OTHER FIELDS
        # "type",                   # row[4]
        politician_entry.state_code = legislator_current_entry.state                # "state",  # row[5]
        # "district",               # row[6]
        politician_entry.party = legislator_current_entry.party                     # "party",  # row[7]
        # "url",                    # row[8]
        # "address",                # row[9]
        # "phone",                  # row[10]
        # "contact_form",           # row[11]
        # "rss_url",                # row[12]
        # "twitter",                # row[13]
        # "facebook",               # row[14]
        # "facebook_id",            # row[15]
        # "youtube",                # row[16]
        # "youtube_id",             # row[17]

        # We use "try/exception" so we know when entry isn't saved due to unique requirement
        # This works! Bigger question -- how to deal with exceptions in general?
        try:
            politician_entry.save()
            # Mark the source entry as was_processed so we don't try to import the same data again
            # legislator_current_entry.save()
        except Exception as e:
            handle_exception(e)
Beispiel #6
0
def google_civic_get_or_create_politician(
        google_civic_candidate_campaign_entry):
    error_result = False
    ##########################
    # Does this politician exist locally?
    politician_on_stage_found = False
    first_name_guess = google_civic_candidate_campaign_entry.name.partition(
        ' ')[0]
    last_name_guess = google_civic_candidate_campaign_entry.name.partition(
        ' ')[-1]
    try:
        # print "We are searching based on full_name_google_civic"
        query1 = Politician.objects.all()
        query1 = query1.filter(
            full_name_google_civic=google_civic_candidate_campaign_entry.name)

        # Was at least one existing entry found based on the above criteria?
        if len(query1):
            politician_on_stage = query1[0]
            politician_on_stage_found = True
            if len(query1) > 1:
                # We have confusion, so skip processing this google_civic_candidate_campaign_entry
                print "More than one Politician found (query1)"
        else:
            print "No politician found based on full_name_google_civic: {name}".format(
                name=google_civic_candidate_campaign_entry.name)
    except Exception as e:
        handle_record_not_found_exception(e)

    if not politician_on_stage_found:
        # No entries were found, so we need to
        # a) Search more deeply
        # Searching based on full_name_assembled
        print "Searching against full_name_assembled: {name}".format(
            name=google_civic_candidate_campaign_entry.name)
        # TODO DALE 2015-05-02 With this code, if we had imported a "Betty T. Yee" from another non-google-civic
        #  source (where full_name_google_civic was empty), we would create a second Politician entry. Fix this.
        try:
            politician_query_full_name_assembled = Politician.objects.all()
            politician_query_full_name_assembled = politician_query_full_name_assembled.filter(
                full_name_assembled=google_civic_candidate_campaign_entry.name)

            if len(politician_query_full_name_assembled):
                politician_on_stage = politician_query_full_name_assembled[0]
                politician_on_stage_found = True
            else:
                print "No politician found based on full_name_assembled: {name}".format(
                    name=google_civic_candidate_campaign_entry.name)
        except Exception as e:
            handle_record_not_found_exception(e)

    if not politician_on_stage_found:
        # No entries were found, so we need to
        # a) Search more deeply
        print "first_name_guess: {first_name_guess}, last_name_guess: {last_name_guess}".format(
            first_name_guess=first_name_guess, last_name_guess=last_name_guess)
        # TODO DALE 2015-05-02 With this code, if we had imported a "Betty T. Yee" from another non-google-civic
        #  source (where full_name_google_civic was empty), we would create a second Politician entry. Fix this.
        try:
            politician_query_first_last_guess = Politician.objects.all()
            politician_query_first_last_guess = politician_query_first_last_guess.filter(
                first_name=first_name_guess)
            politician_query_first_last_guess = politician_query_first_last_guess.filter(
                last_name=last_name_guess)

            if len(politician_query_first_last_guess):
                politician_on_stage = politician_query_first_last_guess[0]
                politician_on_stage_found = True
            else:
                print "No politician found based on first_name_guess: {first_name} and last_name_guess: {last_name}".format(
                    first_name=first_name_guess, last_name=last_name_guess)
        except Exception as e:
            handle_record_not_found_exception(e)

    try:
        if politician_on_stage_found:
            # We found a match, and want to update the Politician data to match how Google Civic references the name
            # print "Store google_civic_candidate_campaign_entry.name in Politician.full_name_google_civic"
            politician_on_stage.full_name_google_civic = google_civic_candidate_campaign_entry.name
        else:
            # print "Create Politician entry: {name}".format(name=google_civic_candidate_campaign_entry.name)
            politician_on_stage = Politician(
                # Do not save first_name or last_name because middle initials will throw this off
                last_name=last_name_guess,
                first_name=first_name_guess,
                full_name_google_civic=google_civic_candidate_campaign_entry.
                name,
            )
        politician_on_stage.save()
    except Exception as e:
        handle_record_not_saved_exception(e)

    if error_result:
        print "There was an error trying to create a politician"
    # else:
    # print "It seems we have found a politician: {display_full_name}".format(display_full_name=politician_on_stage.display_full_name())
    # print "It seems we have found a politician: "+str(politician_on_stage.display_full_name())
    # print "It seems we found or created a politician."

    results = {
        'error_result': error_result,
        'politician_on_stage': politician_on_stage,
    }
    return results
Beispiel #7
0
def transfer_theunitedstatesio_cached_data_to_wevote_tables():
    """
    In this method, we take the cached theunitedstatesio data and move it into the core We Vote data
    :return:
    """
    print "Running transfer_theunitedstatesio_cached_data_to_wevote_tables()"

    legislators_current_query = TheUnitedStatesIoLegislatorCurrent.objects.all(
    )
    # Only retrieve entries that haven't been processed yet
    # legislators_current_query = legislators_current_query.filter(was_processed=False)

    for legislator_current_entry in legislators_current_query:
        print 'Transferring: ' + str(legislator_current_entry.id) + ':' \
              + legislator_current_entry.first_name + ' ' + legislator_current_entry.last_name
        politician_entry_found = False

        #########################
        # Search the Politician's table to see if we already have an entry for this person
        # Do we have a record of this politician based on id_bioguide?
        if legislator_current_entry.bioguide_id != "":
            try:
                # Try to find earlier version based on the bioguide identifier
                query1 = Politician.objects.all()
                query1 = query1.filter(
                    id_bioguide__exact=legislator_current_entry.bioguide_id)

                # Was at least one existing entry found based on the above criteria?
                if len(query1):
                    politician_entry = query1[0]
                    politician_entry_found = True
            except Exception as e:
                handle_record_not_found_exception(e)

        if not politician_entry_found:
            # TheUnitedStatesIoLegislatorCurrent was not found based on bioguide id
            # ...so check to see if we have a record of this legislator based on govtrack id?
            if legislator_current_entry.govtrack_id != "":
                try:
                    query1 = Politician.objects.all()
                    query1 = query1.filter(
                        id_govtrack__exact=legislator_current_entry.govtrack_id
                    )

                    # Was at least one existing entry found based on the above criteria?
                    if len(query1):
                        politician_entry = query1[0]
                        print "FOUND"
                        politician_entry_found = True
                    else:
                        print "NOT FOUND"
                except Exception as e:
                    handle_record_not_found_exception(e)

        if not politician_entry_found:
            # TheUnitedStatesIoLegislatorCurrent was not found based on id_govtrack
            # ...so check to see if we have a record of this legislator based on full_name_google_civic
            if legislator_current_entry.first_name and legislator_current_entry.last_name:
                try:
                    full_name_assembled_guess = legislator_current_entry.first_name + " " + legislator_current_entry.last_name
                    print "Searching for existing full_name_google_civic: {full_name_assembled}".format(
                        full_name_assembled=full_name_assembled_guess)
                    query1 = Politician.objects.all()
                    query1 = query1.filter(
                        full_name_google_civic=full_name_assembled_guess)

                    # Was at least one existing entry found based on the above criteria?
                    if len(query1):
                        politician_entry = query1[0]
                        print "FOUND"
                        politician_entry_found = True
                    else:
                        print "NOT FOUND"
                except Exception as e:
                    handle_record_not_found_exception(e)

        if not politician_entry_found:
            # TheUnitedStatesIoLegislatorCurrent was not found based on full_name_google_civic
            # ...so check to see if we have a record of this legislator based on full_name_assembled
            if legislator_current_entry.first_name and legislator_current_entry.last_name:
                try:
                    full_name_assembled_guess = legislator_current_entry.first_name + " " + legislator_current_entry.last_name
                    print "Searching for existing full_name_assembled: {full_name_assembled}".format(
                        full_name_assembled=full_name_assembled_guess)
                    query1 = Politician.objects.all()
                    query1 = query1.filter(
                        full_name_assembled=full_name_assembled_guess)

                    # Was at least one existing entry found based on the above criteria?
                    if len(query1):
                        politician_entry = query1[0]
                        print "FOUND"
                        politician_entry_found = True
                    else:
                        print "NOT FOUND"
                except Exception as e:
                    handle_record_not_found_exception(e)

        if not politician_entry_found:
            # TheUnitedStatesIoLegislatorCurrent was not found
            # ...so create a new entry
            politician_entry = Politician(
                last_name=legislator_current_entry.last_name,
                first_name=legislator_current_entry.first_name,
                full_name_assembled=legislator_current_entry.first_name + " " +
                legislator_current_entry.last_name,
            )

        politician_entry.last_name = legislator_current_entry.last_name
        politician_entry.first_name = legislator_current_entry.first_name
        politician_entry.full_name_assembled = \
            legislator_current_entry.first_name+" "+legislator_current_entry.last_name
        politician_entry.birth_date = legislator_current_entry.birthday
        politician_entry.gender = legislator_current_entry.gender
        politician_entry.id_bioguide = legislator_current_entry.bioguide_id
        politician_entry.id_thomas = legislator_current_entry.thomas_id
        politician_entry.id_opensecrets = legislator_current_entry.opensecrets_id
        politician_entry.id_lis = legislator_current_entry.lis_id
        politician_entry.id_cspan = legislator_current_entry.cspan_id
        politician_entry.id_govtrack = legislator_current_entry.govtrack_id
        politician_entry.id_votesmart = legislator_current_entry.votesmart_id
        politician_entry.id_ballotpedia = legislator_current_entry.ballotpedia_id
        politician_entry.id_washington_post = legislator_current_entry.washington_post_id
        politician_entry.id_icpsr = legislator_current_entry.icpsr_id
        politician_entry.id_wikipedia = legislator_current_entry.wikipedia_id

        # OTHER FIELDS
        # "type",                   # row[4]
        politician_entry.state_code = legislator_current_entry.state  # "state",  # row[5]
        # "district",               # row[6]
        politician_entry.party = legislator_current_entry.party  # "party",  # row[7]
        # "url",                    # row[8]
        # "address",                # row[9]
        # "phone",                  # row[10]
        # "contact_form",           # row[11]
        # "rss_url",                # row[12]
        # "twitter",                # row[13]
        # "facebook",               # row[14]
        # "facebook_id",            # row[15]
        # "youtube",                # row[16]
        # "youtube_id",             # row[17]

        # We use "try/exception" so we know when entry isn't saved due to unique requirement
        # This works! Bigger question -- how to deal with exceptions in general?
        try:
            politician_entry.save()
            # Mark the source entry as was_processed so we don't try to import the same data again
            # legislator_current_entry.save()
        except Exception as e:
            handle_exception(e)