def handle(self, **options): from slumber.exceptions import HttpClientError, HttpServerError from candidates.election_specific import PARTY_DATA, shorten_post_label from candidates.models import PopItPerson from candidates.popit import create_popit_api_object api = create_popit_api_object() csv_filename = join(dirname(__file__), '..', '..', 'data', 'candidates.csv') with open(csv_filename) as f: all_data = csv.DictReader(f) for candidate in all_data: vi_person_id = candidate['Distrito'] + candidate[ 'Numero Lista'] + candidate['Posicion'] + candidate[ 'Cargo'] + candidate['Nombre Lista'] election_data, post_data = get_post_data( api, candidate['Cargo'], candidate['Distrito']) if (election_data == False): print("Skipping: " + candidate['Cargo'] + ", " + candidate['Distrito'] + ", " + candidate['Nombre']) continue name = candidate['Nombre'] birth_date = None gender = None image_url = None person = get_existing_popit_person(vi_person_id) if person: print("Found an existing person:", person.get_absolute_url()) else: print("No existing person, creating a new one:", name) person = PopItPerson() # Now update fields from the imported data: person.name = name.split(",")[1] + " " + name.split(",")[0] person.gender = gender if birth_date: person.birth_date = str(birth_date) else: person.birth_date = None standing_in_election = { 'post_id': post_data['id'], 'name': shorten_post_label(post_data['label']), 'party_list_position': candidate['Posicion'], } if 'area' in post_data: standing_in_election['mapit_url'] = post_data['area'][ 'identifier'] person.standing_in = {election_data.slug: standing_in_election} party_id = get_party_id(candidate["Partido"]) person.party_memberships = { election_data.slug: { 'id': party_id, 'name': PARTY_DATA.party_id_to_name[party_id], } } person.set_identifier('import-id', vi_person_id) change_metadata = get_change_metadata( None, 'Imported candidate from CSV', ) person.record_version(change_metadata) try: person.save_to_popit(api) except HttpClientError as hce: print("Got an HttpClientError:", hce.content) raise except HttpServerError as hse: print("The server error content was:", hse.content) raise
def handle(self, username=None, **options): from slumber.exceptions import HttpClientError from candidates.popit import create_popit_api_object from candidates.election_specific import PARTY_DATA, shorten_post_label from candidates.models import PopItPerson if username is None: message = "You must supply the name of a user to be associated with the image uploads." raise CommandError(message) try: user = User.objects.get(username=username) except User.DoesNotExist: message = "No user with the username '{0}' could be found" raise CommandError(message.format(username)) api = create_popit_api_object() json_filename = join( dirname(__file__), '..', '..','data', 'candidates.json' ) with open(json_filename) as f: all_data = json.load(f) # This map is needed to get getting YNR election data from # the election ID used in the JSON file. json_election_id_to_name = { e['pk']: e['fields']['name'] for e in all_data if e['model'] == 'elections.election' } person_dict = { e['pk']: e['fields'] for e in all_data if e['model'] == 'popolo.person' } candidate_list = [ dict(person_id=e['pk'], election_id=e['fields']['election']) for e in all_data if e['model'] == 'elections.candidate' ] for candidate in candidate_list: vi_person_id = candidate['person_id'] person_data = person_dict[vi_person_id] election_data, post_data = get_post_data( api, candidate['election_id'], json_election_id_to_name ) birth_date = None if person_data['birth_date']: birth_date = str(dateutil.parser.parse( person_data['birth_date'], dayfirst=True ).date()) name = person_data['name'] gender = person_data['gender'] image_url = person_data['image'] person = get_existing_popit_person(vi_person_id) if person: print("Found an existing person:", person.get_absolute_url()) else: print("No existing person, creating a new one:", name) person = PopItPerson() # Now update fields from the imported data: person.name = name person.gender = gender if birth_date: person.birth_date = str(birth_date) else: person.birth_date = None standing_in_election = { 'post_id': post_data['id'], 'name': shorten_post_label(post_data['label']), } if 'area' in post_data: standing_in_election['mapit_url'] = post_data['area']['identifier'] person.standing_in = { election_data.slug: standing_in_election } person.party_memberships = { election_data.slug: { 'id': UNKNOWN_PARTY_ID, 'name': PARTY_DATA.party_id_to_name[UNKNOWN_PARTY_ID], } } person.set_identifier('import-id', vi_person_id) change_metadata = get_change_metadata( None, 'Imported candidate from JSON', ) person.record_version(change_metadata) try: person.save_to_popit(api) if image_url: enqueue_image(person, user, image_url) except HttpClientError as hce: print("Got an HttpClientError:", hce.content) raise
def handle(self, username=None, **options): election_data = { 'prv-2015': 'listedescandidatsauxelectionslegislativeslisteprovincialeanptic.csv', 'nat-2015': 'listedescandidatsauxelectionslegislativesanptic.csv' } field_map = { 'prv-2015': { 'region': 1, 'party': 4, 'list_order': 5, 'first_name': 7, 'last_name': 6, 'gender': 8, 'birth_date': 9, 'party_short': 3 }, 'nat-2015': { 'region': 0, 'party': 2, 'list_order': 3, 'first_name': 5, 'last_name': 4, 'gender': 6, 'birth_date': 7, 'party_short': 2 } } api = create_popit_api_object() party_id_missing = {} party_name_to_id = {} for party_id, party_name in PARTY_DATA.party_id_to_name.items(): party_name_to_id[party_name] = party_id for election_id, filename in election_data.items(): csv_filename = join( dirname(__file__), '..', '..', 'data', filename ) fields = field_map[election_id] with codecs.open(csv_filename, 'r', encoding='windows-1252') as f: initial = True for candidate in unicode_csv_reader(f): # skip header line if initial: initial = False continue region = candidate[fields['region']] party = candidate[fields['party']] party_list_order = candidate[fields['list_order']] first_name = string.capwords(candidate[fields['first_name']]) last_name = string.capwords(candidate[fields['last_name']]) gender = candidate[fields['gender']] birth_date = None if candidate[fields['birth_date']] is not None: birth_date = str(dateutil.parser.parse( candidate[fields['birth_date']], dayfirst=True ).date()) name = first_name + ' ' + last_name id = '-'.join([ re.sub('[^\w]*', '', re.sub(r' ', '-', strip_accents(name.lower()))), re.sub('[^\w]*', '', candidate[fields['party_short']].lower()), birth_date ]) # national candidate if region == 'PAYS': region = 'Burkina Faso' election_data, post_data = get_post_data( api, election_id, region ) # debug # tmp = '%s %s %s (%s) - %s (%s)' % ( id, first_name, last_name, party, region, post_data['label'] ) # print tmp person = get_existing_popit_person(id) if person: # print "Found an existing person:", person.get_absolute_url() pass else: print "No existing person, creating a new one:", name person = PopItPerson() person.set_identifier('import-id', id) person.family_name = last_name person.given_name = first_name person.name = name person.gender = gender if birth_date: person.birth_date = str(birth_date) else: person.birth_date = None standing_in_election = { 'post_id': post_data['id'], 'name': AREA_POST_DATA.shorten_post_label( election_data.slug, post_data['label'], ), 'party_list_position': party_list_order, } if 'area' in post_data: standing_in_election['mapit_url'] = post_data['area']['identifier'] person.standing_in = { election_data.slug: standing_in_election } change_metadata = get_change_metadata( None, 'Imported candidate from CSV', ) party_comp = re.sub(' +', ' ', party) party_id = UNKNOWN_PARTY_ID if party_comp in party_name_to_id.keys(): party_id = party_name_to_id[party_comp] party = party_comp else: party_id = party_name_to_id['Unknown Party'] party = 'Unknown Party' if party_id == UNKNOWN_PARTY_ID and party_comp not in party_id_missing.keys(): party_id_missing[party_comp] = 1 person.party_memberships = { election_data.slug: { 'id': party_id, 'name': party, 'imported_name': party_comp } } person.record_version(change_metadata) try: person.save_to_popit(api) except HttpClientError as hce: print "Got an HttpClientError:", hce.content raise if len(party_id_missing) > 0: print "Unmatched party names:" for name in party_id_missing.keys(): print name
def handle(self, username=None, **options): from slumber.exceptions import HttpClientError from candidates.election_specific import PARTY_DATA, shorten_post_label from candidates.models import PopItPerson from candidates.popit import create_popit_api_object election_data = { 'prv-2015': 'listedescandidatsauxelectionslegislativeslisteprovincialeanptic.csv', 'nat-2015': 'listedescandidatsauxelectionslegislativesanptic.csv' } field_map = { 'prv-2015': { 'region': 1, 'party': 4, 'list_order': 5, 'first_name': 7, 'last_name': 6, 'gender': 8, 'birth_date': 9, 'party_short': 3 }, 'nat-2015': { 'region': 0, 'party': 2, 'list_order': 3, 'first_name': 5, 'last_name': 4, 'gender': 6, 'birth_date': 7, 'party_short': 2 } } api = create_popit_api_object() party_id_missing = {} party_name_to_id = {} for party_id, party_name in PARTY_DATA.party_id_to_name.items(): party_name_to_id[party_name] = party_id for election_id, filename in election_data.items(): csv_filename = join(dirname(__file__), '..', '..', 'data', filename) fields = field_map[election_id] with codecs.open(csv_filename, 'r', encoding='windows-1252') as f: initial = True for candidate in unicode_csv_reader(f): # skip header line if initial: initial = False continue region = candidate[fields['region']] party = candidate[fields['party']] party_list_order = candidate[fields['list_order']] first_name = string.capwords( candidate[fields['first_name']]) last_name = string.capwords(candidate[fields['last_name']]) gender = candidate[fields['gender']] birth_date = None if candidate[fields['birth_date']] is not None: birth_date = str( dateutil.parser.parse( candidate[fields['birth_date']], dayfirst=True).date()) name = first_name + ' ' + last_name id = '-'.join([ re.sub('[^\w]*', '', re.sub(r' ', '-', strip_accents(name.lower()))), re.sub('[^\w]*', '', candidate[fields['party_short']].lower()), birth_date ]) # national candidate if region == 'PAYS': region = 'Burkina Faso' election_data, post_data = get_post_data( api, election_id, region) # debug # tmp = '%s %s %s (%s) - %s (%s)' % ( id, first_name, last_name, party, region, post_data['label'] ) # print(tmp) person = get_existing_popit_person(id) if person: # print("Found an existing person:", person.get_absolute_url()) pass else: print("No existing person, creating a new one:", name) person = PopItPerson() person.set_identifier('import-id', id) person.family_name = last_name person.given_name = first_name person.name = name person.gender = gender if birth_date: person.birth_date = str(birth_date) else: person.birth_date = None standing_in_election = { 'post_id': post_data['id'], 'name': shorten_post_label(post_data['label']), 'party_list_position': party_list_order, } if 'area' in post_data: standing_in_election['mapit_url'] = post_data['area'][ 'identifier'] person.standing_in = { election_data.slug: standing_in_election } change_metadata = get_change_metadata( None, 'Imported candidate from CSV', ) party_comp = re.sub(' +', ' ', party) party_id = UNKNOWN_PARTY_ID if party_comp in party_name_to_id.keys(): party_id = party_name_to_id[party_comp] party = party_comp else: party_id = party_name_to_id['Unknown Party'] party = 'Unknown Party' if party_id == UNKNOWN_PARTY_ID and party_comp not in party_id_missing.keys( ): party_id_missing[party_comp] = 1 person.party_memberships = { election_data.slug: { 'id': party_id, 'name': party, 'imported_name': party_comp } } person.record_version(change_metadata) try: person.save_to_popit(api) except HttpClientError as hce: print("Got an HttpClientError:", hce.content) raise if len(party_id_missing) > 0: print("Unmatched party names:") for name in party_id_missing.keys(): print(name)
def handle(self, **options): from slumber.exceptions import HttpClientError, HttpServerError from candidates.election_specific import PARTY_DATA, shorten_post_label from candidates.models import PopItPerson from candidates.popit import create_popit_api_object api = create_popit_api_object() csv_filename = join( dirname(__file__), '..', '..','data', 'candidates.csv' ) with open(csv_filename) as f: all_data = csv.DictReader(f) for candidate in all_data: vi_person_id = candidate['Distrito']+candidate['Numero Lista']+candidate['Posicion']+candidate['Cargo']+candidate['Nombre Lista'] election_data, post_data = get_post_data( api, candidate['Cargo'], candidate['Distrito'] ) if (election_data == False): print("Skipping: "+ candidate['Cargo'] +", " + candidate['Distrito']+", " + candidate['Nombre']) continue; name = candidate['Nombre'] birth_date = None gender = None image_url = None person = get_existing_popit_person(vi_person_id) if person: print("Found an existing person:", person.get_absolute_url()) else: print("No existing person, creating a new one:", name) person = PopItPerson() # Now update fields from the imported data: person.name = name.split(",")[1] + " " + name.split(",")[0] person.gender = gender if birth_date: person.birth_date = str(birth_date) else: person.birth_date = None standing_in_election = { 'post_id': post_data['id'], 'name': shorten_post_label(post_data['label']), 'party_list_position': candidate['Posicion'], } if 'area' in post_data: standing_in_election['mapit_url'] = post_data['area']['identifier'] person.standing_in = { election_data.slug: standing_in_election } party_id = get_party_id(candidate["Partido"]); person.party_memberships = { election_data.slug: { 'id': party_id, 'name': PARTY_DATA.party_id_to_name[party_id], } } person.set_identifier('import-id', vi_person_id) change_metadata = get_change_metadata( None, 'Imported candidate from CSV', ) person.record_version(change_metadata) try: person.save_to_popit(api) except HttpClientError as hce: print("Got an HttpClientError:", hce.content) raise except HttpServerError as hse: print("The server error content was:", hse.content) raise
def handle(self, username=None, **options): from slumber.exceptions import HttpClientError from candidates.popit import create_popit_api_object from candidates.election_specific import PARTY_DATA, shorten_post_label from candidates.models import PopItPerson if username is None: message = "You must supply the name of a user to be associated with the image uploads." raise CommandError(message) try: user = User.objects.get(username=username) except User.DoesNotExist: message = "No user with the username '{0}' could be found" raise CommandError(message.format(username)) api = create_popit_api_object() json_filename = join(dirname(__file__), '..', '..', 'data', 'candidates.json') with open(json_filename) as f: all_data = json.load(f) # This map is needed to get getting YNR election data from # the election ID used in the JSON file. json_election_id_to_name = { e['pk']: e['fields']['name'] for e in all_data if e['model'] == 'elections.election' } person_dict = { e['pk']: e['fields'] for e in all_data if e['model'] == 'popolo.person' } candidate_list = [ dict(person_id=e['pk'], election_id=e['fields']['election']) for e in all_data if e['model'] == 'elections.candidate' ] for candidate in candidate_list: vi_person_id = candidate['person_id'] person_data = person_dict[vi_person_id] election_data, post_data = get_post_data(api, candidate['election_id'], json_election_id_to_name) birth_date = None if person_data['birth_date']: birth_date = str( dateutil.parser.parse(person_data['birth_date'], dayfirst=True).date()) name = person_data['name'] gender = person_data['gender'] image_url = person_data['image'] person = get_existing_popit_person(vi_person_id) if person: print("Found an existing person:", person.get_absolute_url()) else: print("No existing person, creating a new one:", name) person = PopItPerson() # Now update fields from the imported data: person.name = name person.gender = gender if birth_date: person.birth_date = str(birth_date) else: person.birth_date = None standing_in_election = { 'post_id': post_data['id'], 'name': shorten_post_label(post_data['label']), } if 'area' in post_data: standing_in_election['mapit_url'] = post_data['area'][ 'identifier'] person.standing_in = {election_data.slug: standing_in_election} person.party_memberships = { election_data.slug: { 'id': UNKNOWN_PARTY_ID, 'name': PARTY_DATA.party_id_to_name[UNKNOWN_PARTY_ID], } } person.set_identifier('import-id', vi_person_id) change_metadata = get_change_metadata( None, 'Imported candidate from JSON', ) person.record_version(change_metadata) try: person.save_to_popit(api) if image_url: enqueue_image(person, user, image_url) except HttpClientError as hce: print("Got an HttpClientError:", hce.content) raise