def get_post_data(api, json_election_id, json_election_id_to_name):
from candidates.cache import get_post_cached
from candidates.election_specific import AREA_DATA, AREA_POST_DATA
json_election_name = json_election_id_to_name[json_election_id]
ynr_election_id = {
'Pre-candidatos a Presidente':
'presidentes-argentina-paso-2015',
'Pre-candidatos a Gobernador de Buenos Aires':
'gobernadores-argentina-paso-2015',
'Pre-candidatos a Gobernador de Tucumán':
'gobernadores-argentina-paso-2015',
'Pre-candidatos a Gobernador de Entre Ríos':
'gobernadores-argentina-paso-2015',
'Pre-Candidatos a Gobernador de San Juan':
'gobernadores-argentina-paso-2015',
}[json_election_name]
ynr_election_data = Election.objects.get_by_slug(ynr_election_id)
province = None
m = re.search(r'a Gobernador de (?P<province>.*)', json_election_name)
if m:
province = m.group('province')
areas_by_name = AREA_DATA.areas_by_name[('PRV', '1')]
area = areas_by_name[strip_accents(province).upper()]
post_id = AREA_POST_DATA.get_post_id(
ynr_election_id, area['type'], area['id']
)
else:
# It must be the presidential election:
post_id = 'presidente'
post_data = get_post_cached(api, post_id)['result']
return ynr_election_data, post_data
def get_post_data(api, json_election_id, json_election_id_to_name):
from candidates.cache import get_post_cached
from candidates.election_specific import AREA_DATA, AREA_POST_DATA
json_election_name = json_election_id_to_name[json_election_id]
ynr_election_id = {
'Pre-candidatos a Presidente':
'presidentes-argentina-paso-2015',
'Pre-candidatos a Gobernador de Buenos Aires':
'gobernadores-argentina-paso-2015',
'Pre-candidatos a Gobernador de Tucumán':
'gobernadores-argentina-paso-2015',
'Pre-candidatos a Gobernador de Entre Ríos':
'gobernadores-argentina-paso-2015',
'Pre-Candidatos a Gobernador de San Juan':
'gobernadores-argentina-paso-2015',
}[json_election_name]
ynr_election_data = Election.objects.get_by_slug(ynr_election_id)
province = None
m = re.search(r'a Gobernador de (?P<province>.*)', json_election_name)
if m:
province = m.group('province')
areas_by_name = AREA_DATA.areas_by_name[('PRV', '1')]
area = areas_by_name[strip_accents(province).upper()]
post_id = AREA_POST_DATA.get_post_id(ynr_election_id, area['type'],
area['id'])
else:
# It must be the presidential election:
post_id = 'presidente'
post_data = get_post_cached(api, post_id)['result']
return ynr_election_data, post_data
def get_post_data(api, json_election_id, json_election_id_to_name):
json_election_name = json_election_id_to_name[json_election_id]
ynr_election_id = {
u'Pre-candidatos a Presidente':
'presidentes-argentina-paso-2015',
u'Pre-candidatos a Gobernador de Buenos Aires':
'gobernadores-argentina-paso-2015',
u'Pre-candidatos a Gobernador de Tucumán':
'gobernadores-argentina-paso-2015',
u'Pre-candidatos a Gobernador de Entre Ríos':
'gobernadores-argentina-paso-2015',
u'Pre-Candidatos a Gobernador de San Juan':
'gobernadores-argentina-paso-2015',
}[json_election_name]
ynr_election_data = settings.ELECTIONS[ynr_election_id]
ynr_election_data['id'] = ynr_election_id
province = None
m = re.search(r'a Gobernador de (?P<province>.*)', json_election_name)
if m:
province = m.group('province')
mapit_areas_by_name = MAPIT_DATA.areas_by_name[('PRV', 1)]
mapit_area = mapit_areas_by_name[strip_accents(province).upper()]
post_id = AREA_POST_DATA.get_post_id(
ynr_election_id, mapit_area['type'], mapit_area['id']
)
else:
# It must be the presidential election:
post_id = 'presidente'
post_data = get_post_cached(api, post_id)['result']
return ynr_election_data, post_data
def get_post_data(api, json_election_id, json_election_id_to_name):
json_election_name = json_election_id_to_name[json_election_id]
ynr_election_id = {
u'Pre-candidatos a Presidente':
'presidentes-argentina-paso-2015',
u'Pre-candidatos a Gobernador de Buenos Aires':
'gobernadores-argentina-paso-2015',
u'Pre-candidatos a Gobernador de Tucumán':
'gobernadores-argentina-paso-2015',
u'Pre-candidatos a Gobernador de Entre Ríos':
'gobernadores-argentina-paso-2015',
u'Pre-Candidatos a Gobernador de San Juan':
'gobernadores-argentina-paso-2015',
}[json_election_name]
ynr_election_data = settings.ELECTIONS[ynr_election_id]
ynr_election_data['id'] = ynr_election_id
province = None
m = re.search(r'a Gobernador de (?P<province>.*)', json_election_name)
if m:
province = m.group('province')
mapit_areas_by_name = MAPIT_DATA.areas_by_name[('PRV', 1)]
mapit_area = mapit_areas_by_name[strip_accents(province).upper()]
post_id = AREA_POST_DATA.get_post_id(ynr_election_id,
mapit_area['type'],
mapit_area['id'])
else:
# It must be the presidential election:
post_id = 'presidente'
post_data = get_post_cached(api, post_id)['result']
return ynr_election_data, post_data
def get_post_data(api, election_id, province):
ynr_election_data = Election.objects.get_by_slug(election_id)
area_key = (ynr_election_data.area_types.first().name,
ynr_election_data.area_generation)
areas_by_name = AREA_DATA.areas_by_name[area_key]
if province != 'Burkina Faso':
province = strip_accents(province).upper()
area = areas_by_name[province]
post_id = AREA_POST_DATA.get_post_id(
election_id, area['type'], area['id']
)
post_data = get_post_cached(api, post_id)['result']
return ynr_election_data, post_data
def get_post_data(api, election_id, province):
from candidates.cache import get_post_cached
from candidates.election_specific import AREA_DATA, AREA_POST_DATA
ynr_election_data = Election.objects.get_by_slug(election_id)
area_key = (ynr_election_data.area_types.first().name,
ynr_election_data.area_generation)
areas_by_name = AREA_DATA.areas_by_name[area_key]
if province != 'Burkina Faso':
province = strip_accents(province).upper()
area = areas_by_name[province]
post_id = AREA_POST_DATA.get_post_id(election_id, area['type'], area['id'])
post_data = get_post_cached(api, post_id)['result']
return ynr_election_data, post_data
def get_electionleaflets_url(mapit_area_id, constituency_name):
"""Generate an electionleaflets.org URL from a constituency name
>>> get_electionleaflets_url(u"66115", u"Ynys M\u00F4n")
u'http://electionleaflets.org/constituencies/66115/ynys_mon/'
>>> get_electionleaflets_url(u"66056", u"Ashton-under-Lyne")
u'http://electionleaflets.org/constituencies/66056/ashton_under_lyne/'
>>> get_electionleaflets_url(u"14403", u"Ayr, Carrick and Cumnock")
u'http://electionleaflets.org/constituencies/14403/ayr_carrick_and_cumnock/'
"""
result = strip_accents(constituency_name)
result = result.lower()
result = re.sub(r'[^a-z]+', ' ', result)
result = re.sub(r'\s+', ' ', result).strip()
slug = result.replace(' ', '_')
url_format = u'http://electionleaflets.org/constituencies/{area_id}/{slug}/'
return url_format.format(area_id=mapit_area_id, slug=slug)
def get_electionleaflets_url(mapit_area_id, constituency_name):
"""Generate an electionleaflets.org URL from a constituency name
>>> (get_electionleaflets_url("66115", "Ynys M\u00F4n") ==
... 'http://electionleaflets.org/constituencies/66115/ynys_mon/')
True
>>> (get_electionleaflets_url("66056", "Ashton-under-Lyne") ==
... 'http://electionleaflets.org/constituencies/66056/ashton_under_lyne/')
True
>>> (get_electionleaflets_url("14403", "Ayr, Carrick and Cumnock") ==
... 'http://electionleaflets.org/constituencies/14403/ayr_carrick_and_cumnock/')
True
"""
result = strip_accents(constituency_name)
result = result.lower()
result = re.sub(r"[^a-z]+", " ", result)
result = re.sub(r"\s+", " ", result).strip()
slug = result.replace(" ", "_")
url_format = "http://electionleaflets.org/constituencies/{area_id}/{slug}/"
return url_format.format(area_id=mapit_area_id, slug=slug)
def handle(self, username=None, **options):
election_data = {
'prv-2015': 'listedescandidatsauxelectionslegislativeslisteprovincialeanptic.csv',
'nat-2015': 'listedescandidatsauxelectionslegislativesanptic.csv'
}
field_map = {
'prv-2015': {
'region': 1,
'party': 4,
'list_order': 5,
'first_name': 7,
'last_name': 6,
'gender': 8,
'birth_date': 9,
'party_short': 3
},
'nat-2015': {
'region': 0,
'party': 2,
'list_order': 3,
'first_name': 5,
'last_name': 4,
'gender': 6,
'birth_date': 7,
'party_short': 2
}
}
api = create_popit_api_object()
party_id_missing = {}
party_name_to_id = {}
for party_id, party_name in PARTY_DATA.party_id_to_name.items():
party_name_to_id[party_name] = party_id
for election_id, filename in election_data.items():
csv_filename = join(
dirname(__file__), '..', '..', 'data', filename
)
fields = field_map[election_id]
with codecs.open(csv_filename, 'r', encoding='windows-1252') as f:
initial = True
for candidate in unicode_csv_reader(f):
# skip header line
if initial:
initial = False
continue
region = candidate[fields['region']]
party = candidate[fields['party']]
party_list_order = candidate[fields['list_order']]
first_name = string.capwords(candidate[fields['first_name']])
last_name = string.capwords(candidate[fields['last_name']])
gender = candidate[fields['gender']]
birth_date = None
if candidate[fields['birth_date']] is not None:
birth_date = str(dateutil.parser.parse(
candidate[fields['birth_date']], dayfirst=True
).date())
name = first_name + ' ' + last_name
id = '-'.join([
re.sub('[^\w]*', '', re.sub(r' ', '-', strip_accents(name.lower()))),
re.sub('[^\w]*', '', candidate[fields['party_short']].lower()),
birth_date
])
# national candidate
if region == 'PAYS':
region = 'Burkina Faso'
election_data, post_data = get_post_data(
api, election_id, region
)
# debug
# tmp = '%s %s %s (%s) - %s (%s)' % ( id, first_name, last_name, party, region, post_data['label'] )
# print tmp
person = get_existing_popit_person(id)
if person:
# print "Found an existing person:", person.get_absolute_url()
pass
else:
print "No existing person, creating a new one:", name
person = PopItPerson()
person.set_identifier('import-id', id)
person.family_name = last_name
person.given_name = first_name
person.name = name
person.gender = gender
if birth_date:
person.birth_date = str(birth_date)
else:
person.birth_date = None
standing_in_election = {
'post_id': post_data['id'],
'name': AREA_POST_DATA.shorten_post_label(
election_data.slug,
post_data['label'],
),
'party_list_position': party_list_order,
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area']['identifier']
person.standing_in = {
election_data.slug: standing_in_election
}
change_metadata = get_change_metadata(
None,
'Imported candidate from CSV',
)
party_comp = re.sub(' +', ' ', party)
party_id = UNKNOWN_PARTY_ID
if party_comp in party_name_to_id.keys():
party_id = party_name_to_id[party_comp]
party = party_comp
else:
party_id = party_name_to_id['Unknown Party']
party = 'Unknown Party'
if party_id == UNKNOWN_PARTY_ID and party_comp not in party_id_missing.keys():
party_id_missing[party_comp] = 1
person.party_memberships = {
election_data.slug: {
'id': party_id,
'name': party,
'imported_name': party_comp
}
}
person.record_version(change_metadata)
try:
person.save_to_popit(api)
except HttpClientError as hce:
print "Got an HttpClientError:", hce.content
raise
if len(party_id_missing) > 0:
print "Unmatched party names:"
for name in party_id_missing.keys():
print name
def handle(self, username=None, **options):
from slumber.exceptions import HttpClientError
from candidates.election_specific import PARTY_DATA, shorten_post_label
from candidates.models import PopItPerson
from candidates.popit import create_popit_api_object
election_data = {
'prv-2015':
'listedescandidatsauxelectionslegislativeslisteprovincialeanptic.csv',
'nat-2015': 'listedescandidatsauxelectionslegislativesanptic.csv'
}
field_map = {
'prv-2015': {
'region': 1,
'party': 4,
'list_order': 5,
'first_name': 7,
'last_name': 6,
'gender': 8,
'birth_date': 9,
'party_short': 3
},
'nat-2015': {
'region': 0,
'party': 2,
'list_order': 3,
'first_name': 5,
'last_name': 4,
'gender': 6,
'birth_date': 7,
'party_short': 2
}
}
api = create_popit_api_object()
party_id_missing = {}
party_name_to_id = {}
for party_id, party_name in PARTY_DATA.party_id_to_name.items():
party_name_to_id[party_name] = party_id
for election_id, filename in election_data.items():
csv_filename = join(dirname(__file__), '..', '..', 'data',
filename)
fields = field_map[election_id]
with codecs.open(csv_filename, 'r', encoding='windows-1252') as f:
initial = True
for candidate in unicode_csv_reader(f):
# skip header line
if initial:
initial = False
continue
region = candidate[fields['region']]
party = candidate[fields['party']]
party_list_order = candidate[fields['list_order']]
first_name = string.capwords(
candidate[fields['first_name']])
last_name = string.capwords(candidate[fields['last_name']])
gender = candidate[fields['gender']]
birth_date = None
if candidate[fields['birth_date']] is not None:
birth_date = str(
dateutil.parser.parse(
candidate[fields['birth_date']],
dayfirst=True).date())
name = first_name + ' ' + last_name
id = '-'.join([
re.sub('[^\w]*', '',
re.sub(r' ', '-', strip_accents(name.lower()))),
re.sub('[^\w]*', '',
candidate[fields['party_short']].lower()),
birth_date
])
# national candidate
if region == 'PAYS':
region = 'Burkina Faso'
election_data, post_data = get_post_data(
api, election_id, region)
# debug
# tmp = '%s %s %s (%s) - %s (%s)' % ( id, first_name, last_name, party, region, post_data['label'] )
# print(tmp)
person = get_existing_popit_person(id)
if person:
# print("Found an existing person:", person.get_absolute_url())
pass
else:
print("No existing person, creating a new one:", name)
person = PopItPerson()
person.set_identifier('import-id', id)
person.family_name = last_name
person.given_name = first_name
person.name = name
person.gender = gender
if birth_date:
person.birth_date = str(birth_date)
else:
person.birth_date = None
standing_in_election = {
'post_id': post_data['id'],
'name': shorten_post_label(post_data['label']),
'party_list_position': party_list_order,
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area'][
'identifier']
person.standing_in = {
election_data.slug: standing_in_election
}
change_metadata = get_change_metadata(
None,
'Imported candidate from CSV',
)
party_comp = re.sub(' +', ' ', party)
party_id = UNKNOWN_PARTY_ID
if party_comp in party_name_to_id.keys():
party_id = party_name_to_id[party_comp]
party = party_comp
else:
party_id = party_name_to_id['Unknown Party']
party = 'Unknown Party'
if party_id == UNKNOWN_PARTY_ID and party_comp not in party_id_missing.keys(
):
party_id_missing[party_comp] = 1
person.party_memberships = {
election_data.slug: {
'id': party_id,
'name': party,
'imported_name': party_comp
}
}
person.record_version(change_metadata)
try:
person.save_to_popit(api)
except HttpClientError as hce:
print("Got an HttpClientError:", hce.content)
raise
if len(party_id_missing) > 0:
print("Unmatched party names:")
for name in party_id_missing.keys():
print(name)
