def load_politicians(self): response = requests.get("{}/api/v2/deputados".format(self._host), params={"itens": 1000}) result = response.json() for data in result["dados"]: politician = self.get_by_external_id(data["id"], Politician.DEPUTY) if not politician: politician = Politician(role=Politician.DEPUTY) politician.picture = data["urlFoto"] politician.name = self._capitalize_name(data["nome"]) politician.external_id = data["id"] politician.party = self._get_party_by_initial(data["siglaPartido"]) politician.role_state = data["siglaUf"] politician.save()
def load_politicians(self): response = requests.get("{}/dadosabertos/senador/lista/atual".format( self._host)) result = xmltodict.parse(response.content.decode()) politicians = result["ListaParlamentarEmExercicio"]["Parlamentares"] for data in politicians["Parlamentar"]: identity = data["IdentificacaoParlamentar"] politician = self.get_by_external_id(identity["CodigoParlamentar"], Politician.SENATOR) if not politician: politician = Politician(role=Politician.SENATOR) politician.picture = identity["UrlFotoParlamentar"] politician.name = identity["NomeCompletoParlamentar"] politician.external_id = identity["CodigoParlamentar"] politician.party = self._get_party_by_initial( identity["SiglaPartidoParlamentar"]) politician.role_state = identity["UfParlamentar"] politician.save()
def transfer_theunitedstatesio_cached_data_to_wevote_tables(): """ In this method, we take the cached theunitedstatesio data and move it into the core We Vote data :return: """ print "Running transfer_theunitedstatesio_cached_data_to_wevote_tables()" legislators_current_query = TheUnitedStatesIoLegislatorCurrent.objects.all() # Only retrieve entries that haven't been processed yet # legislators_current_query = legislators_current_query.filter(was_processed=False) for legislator_current_entry in legislators_current_query: print 'Transferring: ' + str(legislator_current_entry.id) + ':' \ + legislator_current_entry.first_name + ' ' + legislator_current_entry.last_name politician_entry_found = False ######################### # Search the Politician's table to see if we already have an entry for this person # Do we have a record of this politician based on id_bioguide? if legislator_current_entry.bioguide_id != "": try: # Try to find earlier version based on the bioguide identifier query1 = Politician.objects.all() query1 = query1.filter(id_bioguide__exact=legislator_current_entry.bioguide_id) # Was at least one existing entry found based on the above criteria? if len(query1): politician_entry = query1[0] politician_entry_found = True except Exception as e: handle_record_not_found_exception(e) if not politician_entry_found: # TheUnitedStatesIoLegislatorCurrent was not found based on bioguide id # ...so check to see if we have a record of this legislator based on govtrack id? if legislator_current_entry.govtrack_id != "": try: query1 = Politician.objects.all() query1 = query1.filter(id_govtrack__exact=legislator_current_entry.govtrack_id) # Was at least one existing entry found based on the above criteria? if len(query1): politician_entry = query1[0] print "FOUND" politician_entry_found = True else: print "NOT FOUND" except Exception as e: handle_record_not_found_exception(e) if not politician_entry_found: # TheUnitedStatesIoLegislatorCurrent was not found based on id_govtrack # ...so check to see if we have a record of this legislator based on full_name_google_civic if legislator_current_entry.first_name and legislator_current_entry.last_name: try: full_name_assembled_guess = \ legislator_current_entry.first_name+" "+legislator_current_entry.last_name print "Searching for existing full_name_google_civic: {full_name_assembled}".format( full_name_assembled=full_name_assembled_guess) query1 = Politician.objects.all() query1 = query1.filter(full_name_google_civic=full_name_assembled_guess) # Was at least one existing entry found based on the above criteria? if len(query1): politician_entry = query1[0] print "FOUND" politician_entry_found = True else: print "NOT FOUND" except Exception as e: handle_record_not_found_exception(e) if not politician_entry_found: # TheUnitedStatesIoLegislatorCurrent was not found based on full_name_google_civic # ...so check to see if we have a record of this legislator based on full_name_assembled if legislator_current_entry.first_name and legislator_current_entry.last_name: try: full_name_assembled_guess = \ legislator_current_entry.first_name+" "+legislator_current_entry.last_name print "Searching for existing full_name_assembled: {full_name_assembled}".format( full_name_assembled=full_name_assembled_guess) query1 = Politician.objects.all() query1 = query1.filter(full_name_assembled=full_name_assembled_guess) # Was at least one existing entry found based on the above criteria? if len(query1): politician_entry = query1[0] print "FOUND" politician_entry_found = True else: print "NOT FOUND" except Exception as e: handle_record_not_found_exception(e) if not politician_entry_found: # TheUnitedStatesIoLegislatorCurrent was not found # ...so create a new entry politician_entry = Politician( last_name=legislator_current_entry.last_name, first_name=legislator_current_entry.first_name, full_name_assembled=legislator_current_entry.first_name+" "+legislator_current_entry.last_name, ) politician_entry.last_name = legislator_current_entry.last_name politician_entry.first_name = legislator_current_entry.first_name politician_entry.full_name_assembled = \ legislator_current_entry.first_name+" "+legislator_current_entry.last_name politician_entry.birth_date = legislator_current_entry.birthday politician_entry.gender = legislator_current_entry.gender politician_entry.id_bioguide = legislator_current_entry.bioguide_id politician_entry.id_thomas = legislator_current_entry.thomas_id politician_entry.id_opensecrets = legislator_current_entry.opensecrets_id politician_entry.id_lis = legislator_current_entry.lis_id politician_entry.id_cspan = legislator_current_entry.cspan_id politician_entry.id_govtrack = legislator_current_entry.govtrack_id politician_entry.id_votesmart = legislator_current_entry.votesmart_id politician_entry.id_ballotpedia = legislator_current_entry.ballotpedia_id politician_entry.id_washington_post = legislator_current_entry.washington_post_id politician_entry.id_icpsr = legislator_current_entry.icpsr_id politician_entry.id_wikipedia = legislator_current_entry.wikipedia_id # OTHER FIELDS # "type", # row[4] politician_entry.state_code = legislator_current_entry.state # "state", # row[5] # "district", # row[6] politician_entry.party = legislator_current_entry.party # "party", # row[7] # "url", # row[8] # "address", # row[9] # "phone", # row[10] # "contact_form", # row[11] # "rss_url", # row[12] # "twitter", # row[13] # "facebook", # row[14] # "facebook_id", # row[15] # "youtube", # row[16] # "youtube_id", # row[17] # We use "try/exception" so we know when entry isn't saved due to unique requirement # This works! Bigger question -- how to deal with exceptions in general? try: politician_entry.save() # Mark the source entry as was_processed so we don't try to import the same data again # legislator_current_entry.save() except Exception as e: handle_exception(e)
def transfer_theunitedstatesio_cached_data_to_wevote_tables(): """ In this method, we take the cached theunitedstatesio data and move it into the core We Vote data :return: """ print "Running transfer_theunitedstatesio_cached_data_to_wevote_tables()" legislators_current_query = TheUnitedStatesIoLegislatorCurrent.objects.all( ) # Only retrieve entries that haven't been processed yet # legislators_current_query = legislators_current_query.filter(was_processed=False) for legislator_current_entry in legislators_current_query: print 'Transferring: ' + str(legislator_current_entry.id) + ':' \ + legislator_current_entry.first_name + ' ' + legislator_current_entry.last_name politician_entry_found = False ######################### # Search the Politician's table to see if we already have an entry for this person # Do we have a record of this politician based on id_bioguide? if legislator_current_entry.bioguide_id != "": try: # Try to find earlier version based on the bioguide identifier query1 = Politician.objects.all() query1 = query1.filter( id_bioguide__exact=legislator_current_entry.bioguide_id) # Was at least one existing entry found based on the above criteria? if len(query1): politician_entry = query1[0] politician_entry_found = True except Exception as e: handle_record_not_found_exception(e) if not politician_entry_found: # TheUnitedStatesIoLegislatorCurrent was not found based on bioguide id # ...so check to see if we have a record of this legislator based on govtrack id? if legislator_current_entry.govtrack_id != "": try: query1 = Politician.objects.all() query1 = query1.filter( id_govtrack__exact=legislator_current_entry.govtrack_id ) # Was at least one existing entry found based on the above criteria? if len(query1): politician_entry = query1[0] print "FOUND" politician_entry_found = True else: print "NOT FOUND" except Exception as e: handle_record_not_found_exception(e) if not politician_entry_found: # TheUnitedStatesIoLegislatorCurrent was not found based on id_govtrack # ...so check to see if we have a record of this legislator based on full_name_google_civic if legislator_current_entry.first_name and legislator_current_entry.last_name: try: full_name_assembled_guess = legislator_current_entry.first_name + " " + legislator_current_entry.last_name print "Searching for existing full_name_google_civic: {full_name_assembled}".format( full_name_assembled=full_name_assembled_guess) query1 = Politician.objects.all() query1 = query1.filter( full_name_google_civic=full_name_assembled_guess) # Was at least one existing entry found based on the above criteria? if len(query1): politician_entry = query1[0] print "FOUND" politician_entry_found = True else: print "NOT FOUND" except Exception as e: handle_record_not_found_exception(e) if not politician_entry_found: # TheUnitedStatesIoLegislatorCurrent was not found based on full_name_google_civic # ...so check to see if we have a record of this legislator based on full_name_assembled if legislator_current_entry.first_name and legislator_current_entry.last_name: try: full_name_assembled_guess = legislator_current_entry.first_name + " " + legislator_current_entry.last_name print "Searching for existing full_name_assembled: {full_name_assembled}".format( full_name_assembled=full_name_assembled_guess) query1 = Politician.objects.all() query1 = query1.filter( full_name_assembled=full_name_assembled_guess) # Was at least one existing entry found based on the above criteria? if len(query1): politician_entry = query1[0] print "FOUND" politician_entry_found = True else: print "NOT FOUND" except Exception as e: handle_record_not_found_exception(e) if not politician_entry_found: # TheUnitedStatesIoLegislatorCurrent was not found # ...so create a new entry politician_entry = Politician( last_name=legislator_current_entry.last_name, first_name=legislator_current_entry.first_name, full_name_assembled=legislator_current_entry.first_name + " " + legislator_current_entry.last_name, ) politician_entry.last_name = legislator_current_entry.last_name politician_entry.first_name = legislator_current_entry.first_name politician_entry.full_name_assembled = \ legislator_current_entry.first_name+" "+legislator_current_entry.last_name politician_entry.birth_date = legislator_current_entry.birthday politician_entry.gender = legislator_current_entry.gender politician_entry.id_bioguide = legislator_current_entry.bioguide_id politician_entry.id_thomas = legislator_current_entry.thomas_id politician_entry.id_opensecrets = legislator_current_entry.opensecrets_id politician_entry.id_lis = legislator_current_entry.lis_id politician_entry.id_cspan = legislator_current_entry.cspan_id politician_entry.id_govtrack = legislator_current_entry.govtrack_id politician_entry.id_votesmart = legislator_current_entry.votesmart_id politician_entry.id_ballotpedia = legislator_current_entry.ballotpedia_id politician_entry.id_washington_post = legislator_current_entry.washington_post_id politician_entry.id_icpsr = legislator_current_entry.icpsr_id politician_entry.id_wikipedia = legislator_current_entry.wikipedia_id # OTHER FIELDS # "type", # row[4] politician_entry.state_code = legislator_current_entry.state # "state", # row[5] # "district", # row[6] politician_entry.party = legislator_current_entry.party # "party", # row[7] # "url", # row[8] # "address", # row[9] # "phone", # row[10] # "contact_form", # row[11] # "rss_url", # row[12] # "twitter", # row[13] # "facebook", # row[14] # "facebook_id", # row[15] # "youtube", # row[16] # "youtube_id", # row[17] # We use "try/exception" so we know when entry isn't saved due to unique requirement # This works! Bigger question -- how to deal with exceptions in general? try: politician_entry.save() # Mark the source entry as was_processed so we don't try to import the same data again # legislator_current_entry.save() except Exception as e: handle_exception(e)