def setUp(self):
Party(abbreviation='PT', name='PARTIDO DOS TRABALHADORES').save()
p = Party.objects.filter(abbreviation='PT').get()
Politician(id_site=4400,
party=p,
name="JOAO NUNES",
name_full="JOAO NUNES SILVA",
sex='m',
birth_date=datetime(year=2018, month=1, day=1)).save()
def load_politicians(self):
response = requests.get("{}/api/v2/deputados".format(self._host),
params={"itens": 1000})
result = response.json()
for data in result["dados"]:
politician = self.get_by_external_id(data["id"], Politician.DEPUTY)
if not politician:
politician = Politician(role=Politician.DEPUTY)
politician.picture = data["urlFoto"]
politician.name = self._capitalize_name(data["nome"])
politician.external_id = data["id"]
politician.party = self._get_party_by_initial(data["siglaPartido"])
politician.role_state = data["siglaUf"]
politician.save()
def load_politicians(self):
response = requests.get("{}/dadosabertos/senador/lista/atual".format(
self._host))
result = xmltodict.parse(response.content.decode())
politicians = result["ListaParlamentarEmExercicio"]["Parlamentares"]
for data in politicians["Parlamentar"]:
identity = data["IdentificacaoParlamentar"]
politician = self.get_by_external_id(identity["CodigoParlamentar"],
Politician.SENATOR)
if not politician:
politician = Politician(role=Politician.SENATOR)
politician.picture = identity["UrlFotoParlamentar"]
politician.name = identity["NomeCompletoParlamentar"]
politician.external_id = identity["CodigoParlamentar"]
politician.party = self._get_party_by_initial(
identity["SiglaPartidoParlamentar"])
politician.role_state = identity["UfParlamentar"]
politician.save()
def google_civic_get_or_create_politician(google_civic_candidate_campaign_entry):
error_result = False
##########################
# Does this politician exist locally?
politician_on_stage = Politician()
politician_on_stage_found = False
first_name_guess = google_civic_candidate_campaign_entry.name.partition(' ')[0]
last_name_guess = google_civic_candidate_campaign_entry.name.partition(' ')[-1]
try:
# print "We are searching based on full_name_google_civic"
query1 = Politician.objects.all()
query1 = query1.filter(full_name_google_civic=google_civic_candidate_campaign_entry.name)
# Was at least one existing entry found based on the above criteria?
if len(query1):
politician_on_stage = query1[0]
politician_on_stage_found = True
if len(query1) > 1:
# We have confusion, so skip processing this google_civic_candidate_campaign_entry
print "More than one Politician found (query1)"
else:
print "No politician found based on full_name_google_civic: {name}".format(
name=google_civic_candidate_campaign_entry.name)
except Exception as e:
handle_record_not_found_exception(e)
if not politician_on_stage_found:
# No entries were found, so we need to
# a) Search more deeply
# Searching based on full_name_assembled
print "Searching against full_name_assembled: {name}".format(name=google_civic_candidate_campaign_entry.name)
# TODO DALE 2015-05-02 With this code, if we had imported a "Betty T. Yee" from another non-google-civic
# source (where full_name_google_civic was empty), we would create a second Politician entry. Fix this.
try:
politician_query_full_name_assembled = Politician.objects.all()
politician_query_full_name_assembled = politician_query_full_name_assembled.filter(
full_name_assembled=google_civic_candidate_campaign_entry.name)
if len(politician_query_full_name_assembled):
politician_on_stage = politician_query_full_name_assembled[0]
politician_on_stage_found = True
else:
print "No politician found based on full_name_assembled: {name}".format(
name=google_civic_candidate_campaign_entry.name)
except Exception as e:
handle_record_not_found_exception(e)
if not politician_on_stage_found:
# No entries were found, so we need to
# a) Search more deeply
print "first_name_guess: {first_name_guess}, last_name_guess: {last_name_guess}".format(
first_name_guess=first_name_guess, last_name_guess=last_name_guess)
# TODO DALE 2015-05-02 With this code, if we had imported a "Betty T. Yee" from another non-google-civic
# source (where full_name_google_civic was empty), we would create a second Politician entry. Fix this.
try:
politician_query_first_last_guess = Politician.objects.all()
politician_query_first_last_guess = politician_query_first_last_guess.filter(first_name=first_name_guess)
politician_query_first_last_guess = politician_query_first_last_guess.filter(last_name=last_name_guess)
if len(politician_query_first_last_guess):
politician_on_stage = politician_query_first_last_guess[0]
politician_on_stage_found = True
else:
print "No politician found based on first_name_guess: {first_name} " \
"and last_name_guess: {last_name}".format(first_name=first_name_guess,
last_name=last_name_guess)
except Exception as e:
handle_record_not_found_exception(e)
try:
if politician_on_stage_found:
# We found a match, and want to update the Politician data to match how Google Civic references the name
# print "Store google_civic_candidate_campaign_entry.name in Politician.full_name_google_civic"
politician_on_stage.full_name_google_civic = google_civic_candidate_campaign_entry.name
else:
# print "Create Politician entry: {name}".format(name=google_civic_candidate_campaign_entry.name)
politician_on_stage = Politician(
# Do not save first_name or last_name because middle initials will throw this off
last_name=last_name_guess,
first_name=first_name_guess,
full_name_google_civic=google_civic_candidate_campaign_entry.name,
)
politician_on_stage.save()
except Exception as e:
handle_record_not_saved_exception(e)
if error_result:
print "There was an error trying to create a politician"
# else:
# print "It seems we have found a politician: {display_full_name}".format(
# display_full_name=politician_on_stage.display_full_name())
# print "It seems we have found a politician: "+str(politician_on_stage.display_full_name())
# print "It seems we found or created a politician."
results = {
'error_result': error_result,
'politician_on_stage': politician_on_stage,
}
return results
def transfer_theunitedstatesio_cached_data_to_wevote_tables():
"""
In this method, we take the cached theunitedstatesio data and move it into the core We Vote data
:return:
"""
print "Running transfer_theunitedstatesio_cached_data_to_wevote_tables()"
legislators_current_query = TheUnitedStatesIoLegislatorCurrent.objects.all()
# Only retrieve entries that haven't been processed yet
# legislators_current_query = legislators_current_query.filter(was_processed=False)
for legislator_current_entry in legislators_current_query:
print 'Transferring: ' + str(legislator_current_entry.id) + ':' \
+ legislator_current_entry.first_name + ' ' + legislator_current_entry.last_name
politician_entry_found = False
#########################
# Search the Politician's table to see if we already have an entry for this person
# Do we have a record of this politician based on id_bioguide?
if legislator_current_entry.bioguide_id != "":
try:
# Try to find earlier version based on the bioguide identifier
query1 = Politician.objects.all()
query1 = query1.filter(id_bioguide__exact=legislator_current_entry.bioguide_id)
# Was at least one existing entry found based on the above criteria?
if len(query1):
politician_entry = query1[0]
politician_entry_found = True
except Exception as e:
handle_record_not_found_exception(e)
if not politician_entry_found:
# TheUnitedStatesIoLegislatorCurrent was not found based on bioguide id
# ...so check to see if we have a record of this legislator based on govtrack id?
if legislator_current_entry.govtrack_id != "":
try:
query1 = Politician.objects.all()
query1 = query1.filter(id_govtrack__exact=legislator_current_entry.govtrack_id)
# Was at least one existing entry found based on the above criteria?
if len(query1):
politician_entry = query1[0]
print "FOUND"
politician_entry_found = True
else:
print "NOT FOUND"
except Exception as e:
handle_record_not_found_exception(e)
if not politician_entry_found:
# TheUnitedStatesIoLegislatorCurrent was not found based on id_govtrack
# ...so check to see if we have a record of this legislator based on full_name_google_civic
if legislator_current_entry.first_name and legislator_current_entry.last_name:
try:
full_name_assembled_guess = \
legislator_current_entry.first_name+" "+legislator_current_entry.last_name
print "Searching for existing full_name_google_civic: {full_name_assembled}".format(
full_name_assembled=full_name_assembled_guess)
query1 = Politician.objects.all()
query1 = query1.filter(full_name_google_civic=full_name_assembled_guess)
# Was at least one existing entry found based on the above criteria?
if len(query1):
politician_entry = query1[0]
print "FOUND"
politician_entry_found = True
else:
print "NOT FOUND"
except Exception as e:
handle_record_not_found_exception(e)
if not politician_entry_found:
# TheUnitedStatesIoLegislatorCurrent was not found based on full_name_google_civic
# ...so check to see if we have a record of this legislator based on full_name_assembled
if legislator_current_entry.first_name and legislator_current_entry.last_name:
try:
full_name_assembled_guess = \
legislator_current_entry.first_name+" "+legislator_current_entry.last_name
print "Searching for existing full_name_assembled: {full_name_assembled}".format(
full_name_assembled=full_name_assembled_guess)
query1 = Politician.objects.all()
query1 = query1.filter(full_name_assembled=full_name_assembled_guess)
# Was at least one existing entry found based on the above criteria?
if len(query1):
politician_entry = query1[0]
print "FOUND"
politician_entry_found = True
else:
print "NOT FOUND"
except Exception as e:
handle_record_not_found_exception(e)
if not politician_entry_found:
# TheUnitedStatesIoLegislatorCurrent was not found
# ...so create a new entry
politician_entry = Politician(
last_name=legislator_current_entry.last_name,
first_name=legislator_current_entry.first_name,
full_name_assembled=legislator_current_entry.first_name+" "+legislator_current_entry.last_name,
)
politician_entry.last_name = legislator_current_entry.last_name
politician_entry.first_name = legislator_current_entry.first_name
politician_entry.full_name_assembled = \
legislator_current_entry.first_name+" "+legislator_current_entry.last_name
politician_entry.birth_date = legislator_current_entry.birthday
politician_entry.gender = legislator_current_entry.gender
politician_entry.id_bioguide = legislator_current_entry.bioguide_id
politician_entry.id_thomas = legislator_current_entry.thomas_id
politician_entry.id_opensecrets = legislator_current_entry.opensecrets_id
politician_entry.id_lis = legislator_current_entry.lis_id
politician_entry.id_cspan = legislator_current_entry.cspan_id
politician_entry.id_govtrack = legislator_current_entry.govtrack_id
politician_entry.id_votesmart = legislator_current_entry.votesmart_id
politician_entry.id_ballotpedia = legislator_current_entry.ballotpedia_id
politician_entry.id_washington_post = legislator_current_entry.washington_post_id
politician_entry.id_icpsr = legislator_current_entry.icpsr_id
politician_entry.id_wikipedia = legislator_current_entry.wikipedia_id
# OTHER FIELDS
# "type", # row[4]
politician_entry.state_code = legislator_current_entry.state # "state", # row[5]
# "district", # row[6]
politician_entry.party = legislator_current_entry.party # "party", # row[7]
# "url", # row[8]
# "address", # row[9]
# "phone", # row[10]
# "contact_form", # row[11]
# "rss_url", # row[12]
# "twitter", # row[13]
# "facebook", # row[14]
# "facebook_id", # row[15]
# "youtube", # row[16]
# "youtube_id", # row[17]
# We use "try/exception" so we know when entry isn't saved due to unique requirement
# This works! Bigger question -- how to deal with exceptions in general?
try:
politician_entry.save()
# Mark the source entry as was_processed so we don't try to import the same data again
# legislator_current_entry.save()
except Exception as e:
handle_exception(e)
def google_civic_get_or_create_politician(
google_civic_candidate_campaign_entry):
error_result = False
##########################
# Does this politician exist locally?
politician_on_stage_found = False
first_name_guess = google_civic_candidate_campaign_entry.name.partition(
' ')[0]
last_name_guess = google_civic_candidate_campaign_entry.name.partition(
' ')[-1]
try:
# print "We are searching based on full_name_google_civic"
query1 = Politician.objects.all()
query1 = query1.filter(
full_name_google_civic=google_civic_candidate_campaign_entry.name)
# Was at least one existing entry found based on the above criteria?
if len(query1):
politician_on_stage = query1[0]
politician_on_stage_found = True
if len(query1) > 1:
# We have confusion, so skip processing this google_civic_candidate_campaign_entry
print "More than one Politician found (query1)"
else:
print "No politician found based on full_name_google_civic: {name}".format(
name=google_civic_candidate_campaign_entry.name)
except Exception as e:
handle_record_not_found_exception(e)
if not politician_on_stage_found:
# No entries were found, so we need to
# a) Search more deeply
# Searching based on full_name_assembled
print "Searching against full_name_assembled: {name}".format(
name=google_civic_candidate_campaign_entry.name)
# TODO DALE 2015-05-02 With this code, if we had imported a "Betty T. Yee" from another non-google-civic
# source (where full_name_google_civic was empty), we would create a second Politician entry. Fix this.
try:
politician_query_full_name_assembled = Politician.objects.all()
politician_query_full_name_assembled = politician_query_full_name_assembled.filter(
full_name_assembled=google_civic_candidate_campaign_entry.name)
if len(politician_query_full_name_assembled):
politician_on_stage = politician_query_full_name_assembled[0]
politician_on_stage_found = True
else:
print "No politician found based on full_name_assembled: {name}".format(
name=google_civic_candidate_campaign_entry.name)
except Exception as e:
handle_record_not_found_exception(e)
if not politician_on_stage_found:
# No entries were found, so we need to
# a) Search more deeply
print "first_name_guess: {first_name_guess}, last_name_guess: {last_name_guess}".format(
first_name_guess=first_name_guess, last_name_guess=last_name_guess)
# TODO DALE 2015-05-02 With this code, if we had imported a "Betty T. Yee" from another non-google-civic
# source (where full_name_google_civic was empty), we would create a second Politician entry. Fix this.
try:
politician_query_first_last_guess = Politician.objects.all()
politician_query_first_last_guess = politician_query_first_last_guess.filter(
first_name=first_name_guess)
politician_query_first_last_guess = politician_query_first_last_guess.filter(
last_name=last_name_guess)
if len(politician_query_first_last_guess):
politician_on_stage = politician_query_first_last_guess[0]
politician_on_stage_found = True
else:
print "No politician found based on first_name_guess: {first_name} and last_name_guess: {last_name}".format(
first_name=first_name_guess, last_name=last_name_guess)
except Exception as e:
handle_record_not_found_exception(e)
try:
if politician_on_stage_found:
# We found a match, and want to update the Politician data to match how Google Civic references the name
# print "Store google_civic_candidate_campaign_entry.name in Politician.full_name_google_civic"
politician_on_stage.full_name_google_civic = google_civic_candidate_campaign_entry.name
else:
# print "Create Politician entry: {name}".format(name=google_civic_candidate_campaign_entry.name)
politician_on_stage = Politician(
# Do not save first_name or last_name because middle initials will throw this off
last_name=last_name_guess,
first_name=first_name_guess,
full_name_google_civic=google_civic_candidate_campaign_entry.
name,
)
politician_on_stage.save()
except Exception as e:
handle_record_not_saved_exception(e)
if error_result:
print "There was an error trying to create a politician"
# else:
# print "It seems we have found a politician: {display_full_name}".format(display_full_name=politician_on_stage.display_full_name())
# print "It seems we have found a politician: "+str(politician_on_stage.display_full_name())
# print "It seems we found or created a politician."
results = {
'error_result': error_result,
'politician_on_stage': politician_on_stage,
}
return results
def transfer_theunitedstatesio_cached_data_to_wevote_tables():
"""
In this method, we take the cached theunitedstatesio data and move it into the core We Vote data
:return:
"""
print "Running transfer_theunitedstatesio_cached_data_to_wevote_tables()"
legislators_current_query = TheUnitedStatesIoLegislatorCurrent.objects.all(
)
# Only retrieve entries that haven't been processed yet
# legislators_current_query = legislators_current_query.filter(was_processed=False)
for legislator_current_entry in legislators_current_query:
print 'Transferring: ' + str(legislator_current_entry.id) + ':' \
+ legislator_current_entry.first_name + ' ' + legislator_current_entry.last_name
politician_entry_found = False
#########################
# Search the Politician's table to see if we already have an entry for this person
# Do we have a record of this politician based on id_bioguide?
if legislator_current_entry.bioguide_id != "":
try:
# Try to find earlier version based on the bioguide identifier
query1 = Politician.objects.all()
query1 = query1.filter(
id_bioguide__exact=legislator_current_entry.bioguide_id)
# Was at least one existing entry found based on the above criteria?
if len(query1):
politician_entry = query1[0]
politician_entry_found = True
except Exception as e:
handle_record_not_found_exception(e)
if not politician_entry_found:
# TheUnitedStatesIoLegislatorCurrent was not found based on bioguide id
# ...so check to see if we have a record of this legislator based on govtrack id?
if legislator_current_entry.govtrack_id != "":
try:
query1 = Politician.objects.all()
query1 = query1.filter(
id_govtrack__exact=legislator_current_entry.govtrack_id
)
# Was at least one existing entry found based on the above criteria?
if len(query1):
politician_entry = query1[0]
print "FOUND"
politician_entry_found = True
else:
print "NOT FOUND"
except Exception as e:
handle_record_not_found_exception(e)
if not politician_entry_found:
# TheUnitedStatesIoLegislatorCurrent was not found based on id_govtrack
# ...so check to see if we have a record of this legislator based on full_name_google_civic
if legislator_current_entry.first_name and legislator_current_entry.last_name:
try:
full_name_assembled_guess = legislator_current_entry.first_name + " " + legislator_current_entry.last_name
print "Searching for existing full_name_google_civic: {full_name_assembled}".format(
full_name_assembled=full_name_assembled_guess)
query1 = Politician.objects.all()
query1 = query1.filter(
full_name_google_civic=full_name_assembled_guess)
# Was at least one existing entry found based on the above criteria?
if len(query1):
politician_entry = query1[0]
print "FOUND"
politician_entry_found = True
else:
print "NOT FOUND"
except Exception as e:
handle_record_not_found_exception(e)
if not politician_entry_found:
# TheUnitedStatesIoLegislatorCurrent was not found based on full_name_google_civic
# ...so check to see if we have a record of this legislator based on full_name_assembled
if legislator_current_entry.first_name and legislator_current_entry.last_name:
try:
full_name_assembled_guess = legislator_current_entry.first_name + " " + legislator_current_entry.last_name
print "Searching for existing full_name_assembled: {full_name_assembled}".format(
full_name_assembled=full_name_assembled_guess)
query1 = Politician.objects.all()
query1 = query1.filter(
full_name_assembled=full_name_assembled_guess)
# Was at least one existing entry found based on the above criteria?
if len(query1):
politician_entry = query1[0]
print "FOUND"
politician_entry_found = True
else:
print "NOT FOUND"
except Exception as e:
handle_record_not_found_exception(e)
if not politician_entry_found:
# TheUnitedStatesIoLegislatorCurrent was not found
# ...so create a new entry
politician_entry = Politician(
last_name=legislator_current_entry.last_name,
first_name=legislator_current_entry.first_name,
full_name_assembled=legislator_current_entry.first_name + " " +
legislator_current_entry.last_name,
)
politician_entry.last_name = legislator_current_entry.last_name
politician_entry.first_name = legislator_current_entry.first_name
politician_entry.full_name_assembled = \
legislator_current_entry.first_name+" "+legislator_current_entry.last_name
politician_entry.birth_date = legislator_current_entry.birthday
politician_entry.gender = legislator_current_entry.gender
politician_entry.id_bioguide = legislator_current_entry.bioguide_id
politician_entry.id_thomas = legislator_current_entry.thomas_id
politician_entry.id_opensecrets = legislator_current_entry.opensecrets_id
politician_entry.id_lis = legislator_current_entry.lis_id
politician_entry.id_cspan = legislator_current_entry.cspan_id
politician_entry.id_govtrack = legislator_current_entry.govtrack_id
politician_entry.id_votesmart = legislator_current_entry.votesmart_id
politician_entry.id_ballotpedia = legislator_current_entry.ballotpedia_id
politician_entry.id_washington_post = legislator_current_entry.washington_post_id
politician_entry.id_icpsr = legislator_current_entry.icpsr_id
politician_entry.id_wikipedia = legislator_current_entry.wikipedia_id
# OTHER FIELDS
# "type", # row[4]
politician_entry.state_code = legislator_current_entry.state # "state", # row[5]
# "district", # row[6]
politician_entry.party = legislator_current_entry.party # "party", # row[7]
# "url", # row[8]
# "address", # row[9]
# "phone", # row[10]
# "contact_form", # row[11]
# "rss_url", # row[12]
# "twitter", # row[13]
# "facebook", # row[14]
# "facebook_id", # row[15]
# "youtube", # row[16]
# "youtube_id", # row[17]
# We use "try/exception" so we know when entry isn't saved due to unique requirement
# This works! Bigger question -- how to deal with exceptions in general?
try:
politician_entry.save()
# Mark the source entry as was_processed so we don't try to import the same data again
# legislator_current_entry.save()
except Exception as e:
handle_exception(e)
