def handle_mapit_type(self, election, election_data, mapit_type, **options): post_label_format = _('{post_role} for {area_name}') if options['post_label']: post_label_format = options['post_label'] mapit_tuple = (mapit_type, election_data['mapit_generation']) for id, area in MAPIT_DATA.areas_by_id[mapit_tuple].items(): post_id = AREA_POST_DATA.get_post_id( election, mapit_type, id ) role = election_data['for_post_role'] area_mapit_url = settings.MAPIT_BASE_URL + 'area/' + str(area['id']) post_data = { 'role': role, 'id': post_id, 'label': post_label_format.format( post_role=role, area_name=area['name'] ), 'area': { 'name': area['name'], 'id': 'mapit:' + str(area['id']), 'identifier': area_mapit_url, }, 'organization_id': election_data['organization_id'], } create_or_update(self.api.posts, post_data)
def get_post_data(api, json_election_id, json_election_id_to_name): json_election_name = json_election_id_to_name[json_election_id] ynr_election_id = { u'Pre-candidatos a Presidente': 'presidentes-argentina-paso-2015', u'Pre-candidatos a Gobernador de Buenos Aires': 'gobernadores-argentina-paso-2015', u'Pre-candidatos a Gobernador de Tucumán': 'gobernadores-argentina-paso-2015', u'Pre-candidatos a Gobernador de Entre Ríos': 'gobernadores-argentina-paso-2015', u'Pre-Candidatos a Gobernador de San Juan': 'gobernadores-argentina-paso-2015', }[json_election_name] ynr_election_data = settings.ELECTIONS[ynr_election_id] ynr_election_data['id'] = ynr_election_id province = None m = re.search(r'a Gobernador de (?P<province>.*)', json_election_name) if m: province = m.group('province') mapit_areas_by_name = MAPIT_DATA.areas_by_name[('PRV', 1)] mapit_area = mapit_areas_by_name[strip_accents(province).upper()] post_id = AREA_POST_DATA.get_post_id(ynr_election_id, mapit_area['type'], mapit_area['id']) else: # It must be the presidential election: post_id = 'presidente' post_data = get_post_cached(api, post_id)['result'] return ynr_election_data, post_data
def get_post_data(api, origin_post,origin_district): from candidates.cache import get_post_cached from candidates.election_specific import AREA_DATA, AREA_POST_DATA if ("SUPLENTE" in origin_post): return False, False; ynr_election_id = { 'DIPUTADO NACIONAL TITULAR': 'diputados-argentina-paso-2015', 'SENADOR NACIONAL TITULAR': 'senadores-argentina-paso-2015', 'PARLAMENTARIO MERCOSUR DISTRITO REGIONAL TITULAR': 'parlamentarios-mercosur-regional-paso-2015', 'PARLAMENTARIO MERCOSUR DISTRITO NACIONAL TITULAR': 'parlamentarios-mercosur-unico-paso-2015' }[origin_post] ynr_election_data = Election.objects.get_by_slug(ynr_election_id) province = None if origin_district == "PARLAMENTARIO MERCOSUR DISTRITO NACIONAL(1)": post_id = 'pmeu' else: areas_by_name = AREA_DATA.areas_by_name[('PRV', '1')] area = areas_by_name[origin_district] post_id = AREA_POST_DATA.get_post_id( ynr_election_id, area['type'], area['id'] ) post_data = get_post_cached(api, post_id)['result'] return ynr_election_data, post_data
def get_post_data(api, origin_post, origin_district): from candidates.cache import get_post_cached from candidates.election_specific import AREA_DATA, AREA_POST_DATA if ("SUPLENTE" in origin_post): return False, False ynr_election_id = { 'DIPUTADO NACIONAL TITULAR': 'diputados-argentina-paso-2015', 'SENADOR NACIONAL TITULAR': 'senadores-argentina-paso-2015', 'PARLAMENTARIO MERCOSUR DISTRITO REGIONAL TITULAR': 'parlamentarios-mercosur-regional-paso-2015', 'PARLAMENTARIO MERCOSUR DISTRITO NACIONAL TITULAR': 'parlamentarios-mercosur-unico-paso-2015' }[origin_post] ynr_election_data = Election.objects.get_by_slug(ynr_election_id) province = None if origin_district == "PARLAMENTARIO MERCOSUR DISTRITO NACIONAL(1)": post_id = 'pmeu' else: areas_by_name = AREA_DATA.areas_by_name[('PRV', '1')] area = areas_by_name[origin_district] post_id = AREA_POST_DATA.get_post_id(ynr_election_id, area['type'], area['id']) post_data = get_post_cached(api, post_id)['result'] return ynr_election_data, post_data
def get_post_data(api, json_election_id, json_election_id_to_name): from candidates.cache import get_post_cached from candidates.election_specific import AREA_DATA, AREA_POST_DATA json_election_name = json_election_id_to_name[json_election_id] ynr_election_id = { 'Pre-candidatos a Presidente': 'presidentes-argentina-paso-2015', 'Pre-candidatos a Gobernador de Buenos Aires': 'gobernadores-argentina-paso-2015', 'Pre-candidatos a Gobernador de Tucumán': 'gobernadores-argentina-paso-2015', 'Pre-candidatos a Gobernador de Entre Ríos': 'gobernadores-argentina-paso-2015', 'Pre-Candidatos a Gobernador de San Juan': 'gobernadores-argentina-paso-2015', }[json_election_name] ynr_election_data = Election.objects.get_by_slug(ynr_election_id) province = None m = re.search(r'a Gobernador de (?P<province>.*)', json_election_name) if m: province = m.group('province') areas_by_name = AREA_DATA.areas_by_name[('PRV', '1')] area = areas_by_name[strip_accents(province).upper()] post_id = AREA_POST_DATA.get_post_id( ynr_election_id, area['type'], area['id'] ) else: # It must be the presidential election: post_id = 'presidente' post_data = get_post_cached(api, post_id)['result'] return ynr_election_data, post_data
def handle(self, **options): repo_root = realpath(join(dirname(__file__), '..', '..', '..')) output_filename = join( repo_root, 'elections', settings.ELECTION_APP, 'static', 'js', 'post-to-party-set.js', ) with open(output_filename, 'w') as f: f.write('var postIDToPartySet = ') mapping = { post['id']: AREA_POST_DATA.post_id_to_party_set(post['id']) for election, election_data in settings.ELECTIONS_CURRENT for post in get_all_posts(election_data['for_post_role']) } unknown_post_ids = [ k for k, v in mapping.items() if v is None ] f.write(json.dumps(mapping)) f.write(';\n') if unknown_post_ids: print "Warning: no party set could be found for these post IDs:" print unknown_post_ids
def all_posts_in_all_elections(): """A generator function to yield data on every post for every election The purpose of this generator function is to iterate over every post that should exist and yield the post ID along with the election and area data it should be associated with. It might not be obvious that posts can appear in multiple elections; this means that you might get the same post ID yielded more than once, but with different election and area data.""" for election_data in Election.objects.all(): for area_type in election_data.area_types.all(): area_tuple = (area_type.name, election_data.area_generation) for area_id, area in AREA_DATA.areas_by_id[area_tuple].items(): post_id = AREA_POST_DATA.get_post_id( election_data.slug, area_type, area_id ) yield { 'election': election_data.slug, 'election_data': election_data, 'area_type': area_type, 'area_id': area_id, 'area': area, 'post_id': post_id, }
def get_post_data(api, json_election_id, json_election_id_to_name): from candidates.cache import get_post_cached from candidates.election_specific import AREA_DATA, AREA_POST_DATA json_election_name = json_election_id_to_name[json_election_id] ynr_election_id = { 'Pre-candidatos a Presidente': 'presidentes-argentina-paso-2015', 'Pre-candidatos a Gobernador de Buenos Aires': 'gobernadores-argentina-paso-2015', 'Pre-candidatos a Gobernador de Tucumán': 'gobernadores-argentina-paso-2015', 'Pre-candidatos a Gobernador de Entre Ríos': 'gobernadores-argentina-paso-2015', 'Pre-Candidatos a Gobernador de San Juan': 'gobernadores-argentina-paso-2015', }[json_election_name] ynr_election_data = Election.objects.get_by_slug(ynr_election_id) province = None m = re.search(r'a Gobernador de (?P<province>.*)', json_election_name) if m: province = m.group('province') areas_by_name = AREA_DATA.areas_by_name[('PRV', '1')] area = areas_by_name[strip_accents(province).upper()] post_id = AREA_POST_DATA.get_post_id(ynr_election_id, area['type'], area['id']) else: # It must be the presidential election: post_id = 'presidente' post_data = get_post_cached(api, post_id)['result'] return ynr_election_data, post_data
def get_post_data(api, json_election_id, json_election_id_to_name): json_election_name = json_election_id_to_name[json_election_id] ynr_election_id = { u'Pre-candidatos a Presidente': 'presidentes-argentina-paso-2015', u'Pre-candidatos a Gobernador de Buenos Aires': 'gobernadores-argentina-paso-2015', u'Pre-candidatos a Gobernador de Tucumán': 'gobernadores-argentina-paso-2015', u'Pre-candidatos a Gobernador de Entre Ríos': 'gobernadores-argentina-paso-2015', u'Pre-Candidatos a Gobernador de San Juan': 'gobernadores-argentina-paso-2015', }[json_election_name] ynr_election_data = settings.ELECTIONS[ynr_election_id] ynr_election_data['id'] = ynr_election_id province = None m = re.search(r'a Gobernador de (?P<province>.*)', json_election_name) if m: province = m.group('province') mapit_areas_by_name = MAPIT_DATA.areas_by_name[('PRV', 1)] mapit_area = mapit_areas_by_name[strip_accents(province).upper()] post_id = AREA_POST_DATA.get_post_id( ynr_election_id, mapit_area['type'], mapit_area['id'] ) else: # It must be the presidential election: post_id = 'presidente' post_data = get_post_cached(api, post_id)['result'] return ynr_election_data, post_data
def get_post_data(api, election_id, province): ynr_election_data = Election.objects.get_by_slug(election_id) area_key = (ynr_election_data.area_types.first().name, ynr_election_data.area_generation) areas_by_name = AREA_DATA.areas_by_name[area_key] if province != 'Burkina Faso': province = strip_accents(province).upper() area = areas_by_name[province] post_id = AREA_POST_DATA.get_post_id( election_id, area['type'], area['id'] ) post_data = get_post_cached(api, post_id)['result'] return ynr_election_data, post_data
def get_post_data(api, election_id, province): from candidates.cache import get_post_cached from candidates.election_specific import AREA_DATA, AREA_POST_DATA ynr_election_data = Election.objects.get_by_slug(election_id) area_key = (ynr_election_data.area_types.first().name, ynr_election_data.area_generation) areas_by_name = AREA_DATA.areas_by_name[area_key] if province != 'Burkina Faso': province = strip_accents(province).upper() area = areas_by_name[province] post_id = AREA_POST_DATA.get_post_id(election_id, area['type'], area['id']) post_data = get_post_cached(api, post_id)['result'] return ynr_election_data, post_data
def handle_mapit_type(self, election, election_data, mapit_type): mapit_tuple = (mapit_type, election_data['mapit_generation']) for id, area in MAPIT_DATA.areas_by_id[mapit_tuple].items(): post_id = AREA_POST_DATA.get_post_id( election, mapit_type, id ) role = election_data['for_post_role'] area_mapit_url = settings.MAPIT_BASE_URL + 'area/' + str(area['id']) post_data = { 'role': role, 'id': post_id, 'label': role + u' por ' + area['name'], 'area': { 'name': area['name'], 'id': 'mapit:' + str(area['id']), 'identifier': area_mapit_url, }, 'organization_id': election_data['organization_id'], } create_or_update(self.api.posts, post_data)
def handle(self, **options): repo_root = realpath(join(dirname(__file__), '..', '..', '..')) output_filename = join( repo_root, 'elections', settings.ELECTION_APP, 'static', 'js', 'post-to-party-set.js', ) with open(output_filename, 'w') as f: f.write('var postIDToPartySet = ') mapping = { post['id']: AREA_POST_DATA.post_id_to_party_set(post['id']) for election, election_data in settings.ELECTIONS_CURRENT for post in get_all_posts(election_data['for_post_role']) } unknown_post_ids = [k for k, v in mapping.items() if v is None] f.write(json.dumps(mapping)) f.write(';\n') if unknown_post_ids: print "Warning: no party set could be found for these post IDs:" print unknown_post_ids
def handle(self, **options): api = create_popit_api_object() csv_filename = join( dirname(__file__), '..', '..','data', 'candidates.csv' ) with open(csv_filename) as f: all_data = csv.DictReader(f) for candidate in all_data: vi_person_id = candidate['Distrito']+candidate['Numero Lista']+candidate['Posicion']+candidate['Cargo']+candidate['Nombre Lista'] election_data, post_data = get_post_data( api, candidate['Cargo'], candidate['Distrito'] ) if (election_data == False): print "Skipping: "+ candidate['Cargo'] +", " + candidate['Distrito']+", " + candidate['Nombre'] continue; name = candidate['Nombre'] birth_date = None gender = None image_url = None person = get_existing_popit_person(vi_person_id) if person: print "Found an existing person:", person.get_absolute_url() else: print "No existing person, creating a new one:", name person = PopItPerson() # Now update fields from the imported data: person.name = name.split(",")[1] + " " + name.split(",")[0] person.gender = gender if birth_date: person.birth_date = str(birth_date) else: person.birth_date = None standing_in_election = { 'post_id': post_data['id'], 'name': AREA_POST_DATA.shorten_post_label( election_data.slug, post_data['label'], ), 'party_list_position': candidate['Posicion'], } if 'area' in post_data: standing_in_election['mapit_url'] = post_data['area']['identifier'] person.standing_in = { election_data.slug: standing_in_election } party_id = get_party_id(candidate["Partido"]); person.party_memberships = { election_data.slug: { 'id': party_id, 'name': PARTY_DATA.party_id_to_name[party_id], } } person.set_identifier('import-id', vi_person_id) change_metadata = get_change_metadata( None, 'Imported candidate from CSV', ) person.record_version(change_metadata) try: person.save_to_popit(api) except HttpClientError as hce: print "Got an HttpClientError:", hce.content raise except HttpServerError as hse: print "The server error content was:", hse.content raise
def handle(self, username=None, **options): election_data = { 'prv-2015': 'listedescandidatsauxelectionslegislativeslisteprovincialeanptic.csv', 'nat-2015': 'listedescandidatsauxelectionslegislativesanptic.csv' } field_map = { 'prv-2015': { 'region': 1, 'party': 4, 'list_order': 5, 'first_name': 7, 'last_name': 6, 'gender': 8, 'birth_date': 9, 'party_short': 3 }, 'nat-2015': { 'region': 0, 'party': 2, 'list_order': 3, 'first_name': 5, 'last_name': 4, 'gender': 6, 'birth_date': 7, 'party_short': 2 } } api = create_popit_api_object() party_id_missing = {} party_name_to_id = {} for party_id, party_name in PARTY_DATA.party_id_to_name.items(): party_name_to_id[party_name] = party_id for election_id, filename in election_data.items(): csv_filename = join( dirname(__file__), '..', '..', 'data', filename ) fields = field_map[election_id] with codecs.open(csv_filename, 'r', encoding='windows-1252') as f: initial = True for candidate in unicode_csv_reader(f): # skip header line if initial: initial = False continue region = candidate[fields['region']] party = candidate[fields['party']] party_list_order = candidate[fields['list_order']] first_name = string.capwords(candidate[fields['first_name']]) last_name = string.capwords(candidate[fields['last_name']]) gender = candidate[fields['gender']] birth_date = None if candidate[fields['birth_date']] is not None: birth_date = str(dateutil.parser.parse( candidate[fields['birth_date']], dayfirst=True ).date()) name = first_name + ' ' + last_name id = '-'.join([ re.sub('[^\w]*', '', re.sub(r' ', '-', strip_accents(name.lower()))), re.sub('[^\w]*', '', candidate[fields['party_short']].lower()), birth_date ]) # national candidate if region == 'PAYS': region = 'Burkina Faso' election_data, post_data = get_post_data( api, election_id, region ) # debug # tmp = '%s %s %s (%s) - %s (%s)' % ( id, first_name, last_name, party, region, post_data['label'] ) # print tmp person = get_existing_popit_person(id) if person: # print "Found an existing person:", person.get_absolute_url() pass else: print "No existing person, creating a new one:", name person = PopItPerson() person.set_identifier('import-id', id) person.family_name = last_name person.given_name = first_name person.name = name person.gender = gender if birth_date: person.birth_date = str(birth_date) else: person.birth_date = None standing_in_election = { 'post_id': post_data['id'], 'name': AREA_POST_DATA.shorten_post_label( election_data.slug, post_data['label'], ), 'party_list_position': party_list_order, } if 'area' in post_data: standing_in_election['mapit_url'] = post_data['area']['identifier'] person.standing_in = { election_data.slug: standing_in_election } change_metadata = get_change_metadata( None, 'Imported candidate from CSV', ) party_comp = re.sub(' +', ' ', party) party_id = UNKNOWN_PARTY_ID if party_comp in party_name_to_id.keys(): party_id = party_name_to_id[party_comp] party = party_comp else: party_id = party_name_to_id['Unknown Party'] party = 'Unknown Party' if party_id == UNKNOWN_PARTY_ID and party_comp not in party_id_missing.keys(): party_id_missing[party_comp] = 1 person.party_memberships = { election_data.slug: { 'id': party_id, 'name': party, 'imported_name': party_comp } } person.record_version(change_metadata) try: person.save_to_popit(api) except HttpClientError as hce: print "Got an HttpClientError:", hce.content raise if len(party_id_missing) > 0: print "Unmatched party names:" for name in party_id_missing.keys(): print name
def handle(self, username=None, **options): if username is None: message = "You must supply the name of a user to be associated with the image uploads." raise CommandError(message) try: user = User.objects.get(username=username) except User.DoesNotExist: message = "No user with the username '{0}' could be found" raise CommandError(message.format(username)) api = create_popit_api_object() json_filename = join(dirname(__file__), '..', '..', 'data', 'candidates.json') with open(json_filename) as f: all_data = json.load(f) # This map is needed to get getting YNR election data from # the election ID used in the JSON file. json_election_id_to_name = { e['pk']: e['fields']['name'] for e in all_data if e['model'] == 'elections.election' } person_dict = { e['pk']: e['fields'] for e in all_data if e['model'] == 'popolo.person' } candidate_list = [ dict(person_id=e['pk'], election_id=e['fields']['election']) for e in all_data if e['model'] == 'elections.candidate' ] for candidate in candidate_list: vi_person_id = candidate['person_id'] person_data = person_dict[vi_person_id] election_data, post_data = get_post_data(api, candidate['election_id'], json_election_id_to_name) birth_date = None if person_data['birth_date']: birth_date = str( dateutil.parser.parse(person_data['birth_date'], dayfirst=True).date()) name = person_data['name'] gender = person_data['gender'] image_url = person_data['image'] person = get_existing_popit_person(vi_person_id) if person: print "Found an existing person:", person.get_absolute_url() else: print "No existing person, creating a new one:", name person = PopItPerson() # Now update fields from the imported data: person.name = name person.gender = gender if birth_date: person.birth_date = str(birth_date) else: person.birth_date = None standing_in_election = { 'post_id': post_data['id'], 'name': AREA_POST_DATA.shorten_post_label( election_data['id'], post_data['label'], ), } if 'area' in post_data: standing_in_election['mapit_url'] = post_data['area'][ 'identifier'] person.standing_in = {election_data['id']: standing_in_election} person.party_memberships = { election_data['id']: { 'id': UNKNOWN_PARTY_ID, 'name': PARTY_DATA.party_id_to_name[UNKNOWN_PARTY_ID], } } person.set_identifier('import-id', vi_person_id) change_metadata = get_change_metadata( None, 'Imported candidate from JSON', ) person.record_version(change_metadata) try: person.save_to_popit(api) if image_url: enqueue_image(person, user, image_url) except HttpClientError as hce: print "Got an HttpClientError:", hce.content raise
def handle(self, username=None, **options): if username is None: message = "You must supply the name of a user to be associated with the image uploads." raise CommandError(message) try: user = User.objects.get(username=username) except User.DoesNotExist: message = "No user with the username '{0}' could be found" raise CommandError(message.format(username)) api = create_popit_api_object() json_filename = join( dirname(__file__), '..', '..','data', 'candidates.json' ) with open(json_filename) as f: all_data = json.load(f) # This map is needed to get getting YNR election data from # the election ID used in the JSON file. json_election_id_to_name = { e['pk']: e['fields']['name'] for e in all_data if e['model'] == 'elections.election' } person_dict = { e['pk']: e['fields'] for e in all_data if e['model'] == 'popolo.person' } candidate_list = [ dict(person_id=e['pk'], election_id=e['fields']['election']) for e in all_data if e['model'] == 'elections.candidate' ] for candidate in candidate_list: vi_person_id = candidate['person_id'] person_data = person_dict[vi_person_id] election_data, post_data = get_post_data( api, candidate['election_id'], json_election_id_to_name ) birth_date = None if person_data['birth_date']: birth_date = str(dateutil.parser.parse( person_data['birth_date'], dayfirst=True ).date()) name = person_data['name'] gender = person_data['gender'] image_url = person_data['image'] person = get_existing_popit_person(vi_person_id) if person: print "Found an existing person:", person.get_absolute_url() else: print "No existing person, creating a new one:", name person = PopItPerson() # Now update fields from the imported data: person.name = name person.gender = gender if birth_date: person.birth_date = str(birth_date) else: person.birth_date = None standing_in_election = { 'post_id': post_data['id'], 'name': AREA_POST_DATA.shorten_post_label( election_data['id'], post_data['label'], ), } if 'area' in post_data: standing_in_election['mapit_url'] = post_data['area']['identifier'] person.standing_in = { election_data['id']: standing_in_election } person.party_memberships = { election_data['id']: { 'id': UNKNOWN_PARTY_ID, 'name': PARTY_DATA.party_id_to_name[UNKNOWN_PARTY_ID], } } person.set_identifier('import-id', vi_person_id) change_metadata = get_change_metadata( None, 'Imported candidate from JSON', ) person.record_version(change_metadata) try: person.save_to_popit(api) if image_url: enqueue_image(person, user, image_url) except HttpClientError as hce: print "Got an HttpClientError:", hce.content raise