def transfer_theunitedstatesio_cached_data_to_wevote_tables():
"""
In this method, we take the cached theunitedstatesio data and move it into the core We Vote data
:return:
"""
print "Running transfer_theunitedstatesio_cached_data_to_wevote_tables()"
legislators_current_query = TheUnitedStatesIoLegislatorCurrent.objects.all()
# Only retrieve entries that haven't been processed yet
# legislators_current_query = legislators_current_query.filter(was_processed=False)
for legislator_current_entry in legislators_current_query:
print 'Transferring: ' + str(legislator_current_entry.id) + ':' \
+ legislator_current_entry.first_name + ' ' + legislator_current_entry.last_name
politician_entry_found = False
#########################
# Search the Politician's table to see if we already have an entry for this person
# Do we have a record of this politician based on id_bioguide?
if legislator_current_entry.bioguide_id != "":
try:
# Try to find earlier version based on the bioguide identifier
query1 = Politician.objects.all()
query1 = query1.filter(id_bioguide__exact=legislator_current_entry.bioguide_id)
# Was at least one existing entry found based on the above criteria?
if len(query1):
politician_entry = query1[0]
politician_entry_found = True
except Exception as e:
handle_record_not_found_exception(e)
if not politician_entry_found:
# TheUnitedStatesIoLegislatorCurrent was not found based on bioguide id
# ...so check to see if we have a record of this legislator based on govtrack id?
if legislator_current_entry.govtrack_id != "":
try:
query1 = Politician.objects.all()
query1 = query1.filter(id_govtrack__exact=legislator_current_entry.govtrack_id)
# Was at least one existing entry found based on the above criteria?
if len(query1):
politician_entry = query1[0]
print "FOUND"
politician_entry_found = True
else:
print "NOT FOUND"
except Exception as e:
handle_record_not_found_exception(e)
if not politician_entry_found:
# TheUnitedStatesIoLegislatorCurrent was not found based on id_govtrack
# ...so check to see if we have a record of this legislator based on full_name_google_civic
if legislator_current_entry.first_name and legislator_current_entry.last_name:
try:
full_name_assembled_guess = \
legislator_current_entry.first_name+" "+legislator_current_entry.last_name
print "Searching for existing full_name_google_civic: {full_name_assembled}".format(
full_name_assembled=full_name_assembled_guess)
query1 = Politician.objects.all()
query1 = query1.filter(full_name_google_civic=full_name_assembled_guess)
# Was at least one existing entry found based on the above criteria?
if len(query1):
politician_entry = query1[0]
print "FOUND"
politician_entry_found = True
else:
print "NOT FOUND"
except Exception as e:
handle_record_not_found_exception(e)
if not politician_entry_found:
# TheUnitedStatesIoLegislatorCurrent was not found based on full_name_google_civic
# ...so check to see if we have a record of this legislator based on full_name_assembled
if legislator_current_entry.first_name and legislator_current_entry.last_name:
try:
full_name_assembled_guess = \
legislator_current_entry.first_name+" "+legislator_current_entry.last_name
print "Searching for existing full_name_assembled: {full_name_assembled}".format(
full_name_assembled=full_name_assembled_guess)
query1 = Politician.objects.all()
query1 = query1.filter(full_name_assembled=full_name_assembled_guess)
# Was at least one existing entry found based on the above criteria?
if len(query1):
politician_entry = query1[0]
print "FOUND"
politician_entry_found = True
else:
print "NOT FOUND"
except Exception as e:
handle_record_not_found_exception(e)
if not politician_entry_found:
# TheUnitedStatesIoLegislatorCurrent was not found
# ...so create a new entry
politician_entry = Politician(
last_name=legislator_current_entry.last_name,
first_name=legislator_current_entry.first_name,
full_name_assembled=legislator_current_entry.first_name+" "+legislator_current_entry.last_name,
)
politician_entry.last_name = legislator_current_entry.last_name
politician_entry.first_name = legislator_current_entry.first_name
politician_entry.full_name_assembled = \
legislator_current_entry.first_name+" "+legislator_current_entry.last_name
politician_entry.birth_date = legislator_current_entry.birthday
politician_entry.gender = legislator_current_entry.gender
politician_entry.id_bioguide = legislator_current_entry.bioguide_id
politician_entry.id_thomas = legislator_current_entry.thomas_id
politician_entry.id_opensecrets = legislator_current_entry.opensecrets_id
politician_entry.id_lis = legislator_current_entry.lis_id
politician_entry.id_cspan = legislator_current_entry.cspan_id
politician_entry.id_govtrack = legislator_current_entry.govtrack_id
politician_entry.id_votesmart = legislator_current_entry.votesmart_id
politician_entry.id_ballotpedia = legislator_current_entry.ballotpedia_id
politician_entry.id_washington_post = legislator_current_entry.washington_post_id
politician_entry.id_icpsr = legislator_current_entry.icpsr_id
politician_entry.id_wikipedia = legislator_current_entry.wikipedia_id
# OTHER FIELDS
# "type", # row[4]
politician_entry.state_code = legislator_current_entry.state # "state", # row[5]
# "district", # row[6]
politician_entry.party = legislator_current_entry.party # "party", # row[7]
# "url", # row[8]
# "address", # row[9]
# "phone", # row[10]
# "contact_form", # row[11]
# "rss_url", # row[12]
# "twitter", # row[13]
# "facebook", # row[14]
# "facebook_id", # row[15]
# "youtube", # row[16]
# "youtube_id", # row[17]
# We use "try/exception" so we know when entry isn't saved due to unique requirement
# This works! Bigger question -- how to deal with exceptions in general?
try:
politician_entry.save()
# Mark the source entry as was_processed so we don't try to import the same data again
# legislator_current_entry.save()
except Exception as e:
handle_exception(e)
def transfer_theunitedstatesio_cached_data_to_wevote_tables():
"""
In this method, we take the cached theunitedstatesio data and move it into the core We Vote data
:return:
"""
print "Running transfer_theunitedstatesio_cached_data_to_wevote_tables()"
legislators_current_query = TheUnitedStatesIoLegislatorCurrent.objects.all(
)
# Only retrieve entries that haven't been processed yet
# legislators_current_query = legislators_current_query.filter(was_processed=False)
for legislator_current_entry in legislators_current_query:
print 'Transferring: ' + str(legislator_current_entry.id) + ':' \
+ legislator_current_entry.first_name + ' ' + legislator_current_entry.last_name
politician_entry_found = False
#########################
# Search the Politician's table to see if we already have an entry for this person
# Do we have a record of this politician based on id_bioguide?
if legislator_current_entry.bioguide_id != "":
try:
# Try to find earlier version based on the bioguide identifier
query1 = Politician.objects.all()
query1 = query1.filter(
id_bioguide__exact=legislator_current_entry.bioguide_id)
# Was at least one existing entry found based on the above criteria?
if len(query1):
politician_entry = query1[0]
politician_entry_found = True
except Exception as e:
handle_record_not_found_exception(e)
if not politician_entry_found:
# TheUnitedStatesIoLegislatorCurrent was not found based on bioguide id
# ...so check to see if we have a record of this legislator based on govtrack id?
if legislator_current_entry.govtrack_id != "":
try:
query1 = Politician.objects.all()
query1 = query1.filter(
id_govtrack__exact=legislator_current_entry.govtrack_id
)
# Was at least one existing entry found based on the above criteria?
if len(query1):
politician_entry = query1[0]
print "FOUND"
politician_entry_found = True
else:
print "NOT FOUND"
except Exception as e:
handle_record_not_found_exception(e)
if not politician_entry_found:
# TheUnitedStatesIoLegislatorCurrent was not found based on id_govtrack
# ...so check to see if we have a record of this legislator based on full_name_google_civic
if legislator_current_entry.first_name and legislator_current_entry.last_name:
try:
full_name_assembled_guess = legislator_current_entry.first_name + " " + legislator_current_entry.last_name
print "Searching for existing full_name_google_civic: {full_name_assembled}".format(
full_name_assembled=full_name_assembled_guess)
query1 = Politician.objects.all()
query1 = query1.filter(
full_name_google_civic=full_name_assembled_guess)
# Was at least one existing entry found based on the above criteria?
if len(query1):
politician_entry = query1[0]
print "FOUND"
politician_entry_found = True
else:
print "NOT FOUND"
except Exception as e:
handle_record_not_found_exception(e)
if not politician_entry_found:
# TheUnitedStatesIoLegislatorCurrent was not found based on full_name_google_civic
# ...so check to see if we have a record of this legislator based on full_name_assembled
if legislator_current_entry.first_name and legislator_current_entry.last_name:
try:
full_name_assembled_guess = legislator_current_entry.first_name + " " + legislator_current_entry.last_name
print "Searching for existing full_name_assembled: {full_name_assembled}".format(
full_name_assembled=full_name_assembled_guess)
query1 = Politician.objects.all()
query1 = query1.filter(
full_name_assembled=full_name_assembled_guess)
# Was at least one existing entry found based on the above criteria?
if len(query1):
politician_entry = query1[0]
print "FOUND"
politician_entry_found = True
else:
print "NOT FOUND"
except Exception as e:
handle_record_not_found_exception(e)
if not politician_entry_found:
# TheUnitedStatesIoLegislatorCurrent was not found
# ...so create a new entry
politician_entry = Politician(
last_name=legislator_current_entry.last_name,
first_name=legislator_current_entry.first_name,
full_name_assembled=legislator_current_entry.first_name + " " +
legislator_current_entry.last_name,
)
politician_entry.last_name = legislator_current_entry.last_name
politician_entry.first_name = legislator_current_entry.first_name
politician_entry.full_name_assembled = \
legislator_current_entry.first_name+" "+legislator_current_entry.last_name
politician_entry.birth_date = legislator_current_entry.birthday
politician_entry.gender = legislator_current_entry.gender
politician_entry.id_bioguide = legislator_current_entry.bioguide_id
politician_entry.id_thomas = legislator_current_entry.thomas_id
politician_entry.id_opensecrets = legislator_current_entry.opensecrets_id
politician_entry.id_lis = legislator_current_entry.lis_id
politician_entry.id_cspan = legislator_current_entry.cspan_id
politician_entry.id_govtrack = legislator_current_entry.govtrack_id
politician_entry.id_votesmart = legislator_current_entry.votesmart_id
politician_entry.id_ballotpedia = legislator_current_entry.ballotpedia_id
politician_entry.id_washington_post = legislator_current_entry.washington_post_id
politician_entry.id_icpsr = legislator_current_entry.icpsr_id
politician_entry.id_wikipedia = legislator_current_entry.wikipedia_id
# OTHER FIELDS
# "type", # row[4]
politician_entry.state_code = legislator_current_entry.state # "state", # row[5]
# "district", # row[6]
politician_entry.party = legislator_current_entry.party # "party", # row[7]
# "url", # row[8]
# "address", # row[9]
# "phone", # row[10]
# "contact_form", # row[11]
# "rss_url", # row[12]
# "twitter", # row[13]
# "facebook", # row[14]
# "facebook_id", # row[15]
# "youtube", # row[16]
# "youtube_id", # row[17]
# We use "try/exception" so we know when entry isn't saved due to unique requirement
# This works! Bigger question -- how to deal with exceptions in general?
try:
politician_entry.save()
# Mark the source entry as was_processed so we don't try to import the same data again
# legislator_current_entry.save()
except Exception as e:
handle_exception(e)
