def handle_mapit_type(self, election, election_data, mapit_type, **options):
post_label_format = _('{post_role} for {area_name}')
if options['post_label']:
post_label_format = options['post_label']
mapit_tuple = (mapit_type, election_data['mapit_generation'])
for id, area in MAPIT_DATA.areas_by_id[mapit_tuple].items():
post_id = AREA_POST_DATA.get_post_id(
election, mapit_type, id
)
role = election_data['for_post_role']
area_mapit_url = settings.MAPIT_BASE_URL + 'area/' + str(area['id'])
post_data = {
'role': role,
'id': post_id,
'label': post_label_format.format(
post_role=role, area_name=area['name']
),
'area': {
'name': area['name'],
'id': 'mapit:' + str(area['id']),
'identifier': area_mapit_url,
},
'organization_id': election_data['organization_id'],
}
create_or_update(self.api.posts, post_data)
def get_post_data(api, json_election_id, json_election_id_to_name):
json_election_name = json_election_id_to_name[json_election_id]
ynr_election_id = {
u'Pre-candidatos a Presidente':
'presidentes-argentina-paso-2015',
u'Pre-candidatos a Gobernador de Buenos Aires':
'gobernadores-argentina-paso-2015',
u'Pre-candidatos a Gobernador de Tucumán':
'gobernadores-argentina-paso-2015',
u'Pre-candidatos a Gobernador de Entre Ríos':
'gobernadores-argentina-paso-2015',
u'Pre-Candidatos a Gobernador de San Juan':
'gobernadores-argentina-paso-2015',
}[json_election_name]
ynr_election_data = settings.ELECTIONS[ynr_election_id]
ynr_election_data['id'] = ynr_election_id
province = None
m = re.search(r'a Gobernador de (?P<province>.*)', json_election_name)
if m:
province = m.group('province')
mapit_areas_by_name = MAPIT_DATA.areas_by_name[('PRV', 1)]
mapit_area = mapit_areas_by_name[strip_accents(province).upper()]
post_id = AREA_POST_DATA.get_post_id(ynr_election_id,
mapit_area['type'],
mapit_area['id'])
else:
# It must be the presidential election:
post_id = 'presidente'
post_data = get_post_cached(api, post_id)['result']
return ynr_election_data, post_data
def get_post_data(api, origin_post,origin_district):
from candidates.cache import get_post_cached
from candidates.election_specific import AREA_DATA, AREA_POST_DATA
if ("SUPLENTE" in origin_post):
return False, False;
ynr_election_id = {
'DIPUTADO NACIONAL TITULAR':
'diputados-argentina-paso-2015',
'SENADOR NACIONAL TITULAR':
'senadores-argentina-paso-2015',
'PARLAMENTARIO MERCOSUR DISTRITO REGIONAL TITULAR':
'parlamentarios-mercosur-regional-paso-2015',
'PARLAMENTARIO MERCOSUR DISTRITO NACIONAL TITULAR':
'parlamentarios-mercosur-unico-paso-2015'
}[origin_post]
ynr_election_data = Election.objects.get_by_slug(ynr_election_id)
province = None
if origin_district == "PARLAMENTARIO MERCOSUR DISTRITO NACIONAL(1)":
post_id = 'pmeu'
else:
areas_by_name = AREA_DATA.areas_by_name[('PRV', '1')]
area = areas_by_name[origin_district]
post_id = AREA_POST_DATA.get_post_id(
ynr_election_id, area['type'], area['id']
)
post_data = get_post_cached(api, post_id)['result']
return ynr_election_data, post_data
def get_post_data(api, origin_post, origin_district):
from candidates.cache import get_post_cached
from candidates.election_specific import AREA_DATA, AREA_POST_DATA
if ("SUPLENTE" in origin_post):
return False, False
ynr_election_id = {
'DIPUTADO NACIONAL TITULAR':
'diputados-argentina-paso-2015',
'SENADOR NACIONAL TITULAR':
'senadores-argentina-paso-2015',
'PARLAMENTARIO MERCOSUR DISTRITO REGIONAL TITULAR':
'parlamentarios-mercosur-regional-paso-2015',
'PARLAMENTARIO MERCOSUR DISTRITO NACIONAL TITULAR':
'parlamentarios-mercosur-unico-paso-2015'
}[origin_post]
ynr_election_data = Election.objects.get_by_slug(ynr_election_id)
province = None
if origin_district == "PARLAMENTARIO MERCOSUR DISTRITO NACIONAL(1)":
post_id = 'pmeu'
else:
areas_by_name = AREA_DATA.areas_by_name[('PRV', '1')]
area = areas_by_name[origin_district]
post_id = AREA_POST_DATA.get_post_id(ynr_election_id, area['type'],
area['id'])
post_data = get_post_cached(api, post_id)['result']
return ynr_election_data, post_data
def get_post_data(api, json_election_id, json_election_id_to_name):
from candidates.cache import get_post_cached
from candidates.election_specific import AREA_DATA, AREA_POST_DATA
json_election_name = json_election_id_to_name[json_election_id]
ynr_election_id = {
'Pre-candidatos a Presidente':
'presidentes-argentina-paso-2015',
'Pre-candidatos a Gobernador de Buenos Aires':
'gobernadores-argentina-paso-2015',
'Pre-candidatos a Gobernador de Tucumán':
'gobernadores-argentina-paso-2015',
'Pre-candidatos a Gobernador de Entre Ríos':
'gobernadores-argentina-paso-2015',
'Pre-Candidatos a Gobernador de San Juan':
'gobernadores-argentina-paso-2015',
}[json_election_name]
ynr_election_data = Election.objects.get_by_slug(ynr_election_id)
province = None
m = re.search(r'a Gobernador de (?P<province>.*)', json_election_name)
if m:
province = m.group('province')
areas_by_name = AREA_DATA.areas_by_name[('PRV', '1')]
area = areas_by_name[strip_accents(province).upper()]
post_id = AREA_POST_DATA.get_post_id(
ynr_election_id, area['type'], area['id']
)
else:
# It must be the presidential election:
post_id = 'presidente'
post_data = get_post_cached(api, post_id)['result']
return ynr_election_data, post_data
def handle(self, **options):
repo_root = realpath(join(dirname(__file__), '..', '..', '..'))
output_filename = join(
repo_root,
'elections',
settings.ELECTION_APP,
'static',
'js',
'post-to-party-set.js',
)
with open(output_filename, 'w') as f:
f.write('var postIDToPartySet = ')
mapping = {
post['id']: AREA_POST_DATA.post_id_to_party_set(post['id'])
for election, election_data in settings.ELECTIONS_CURRENT
for post in get_all_posts(election_data['for_post_role'])
}
unknown_post_ids = [
k for k, v in mapping.items()
if v is None
]
f.write(json.dumps(mapping))
f.write(';\n')
if unknown_post_ids:
print "Warning: no party set could be found for these post IDs:"
print unknown_post_ids
def all_posts_in_all_elections():
"""A generator function to yield data on every post for every election
The purpose of this generator function is to iterate over every
post that should exist and yield the post ID along with the
election and area data it should be associated with.
It might not be obvious that posts can appear in multiple
elections; this means that you might get the same post ID yielded
more than once, but with different election and area data."""
for election_data in Election.objects.all():
for area_type in election_data.area_types.all():
area_tuple = (area_type.name, election_data.area_generation)
for area_id, area in AREA_DATA.areas_by_id[area_tuple].items():
post_id = AREA_POST_DATA.get_post_id(
election_data.slug, area_type, area_id
)
yield {
'election': election_data.slug,
'election_data': election_data,
'area_type': area_type,
'area_id': area_id,
'area': area,
'post_id': post_id,
}
def get_post_data(api, json_election_id, json_election_id_to_name):
from candidates.cache import get_post_cached
from candidates.election_specific import AREA_DATA, AREA_POST_DATA
json_election_name = json_election_id_to_name[json_election_id]
ynr_election_id = {
'Pre-candidatos a Presidente':
'presidentes-argentina-paso-2015',
'Pre-candidatos a Gobernador de Buenos Aires':
'gobernadores-argentina-paso-2015',
'Pre-candidatos a Gobernador de Tucumán':
'gobernadores-argentina-paso-2015',
'Pre-candidatos a Gobernador de Entre Ríos':
'gobernadores-argentina-paso-2015',
'Pre-Candidatos a Gobernador de San Juan':
'gobernadores-argentina-paso-2015',
}[json_election_name]
ynr_election_data = Election.objects.get_by_slug(ynr_election_id)
province = None
m = re.search(r'a Gobernador de (?P<province>.*)', json_election_name)
if m:
province = m.group('province')
areas_by_name = AREA_DATA.areas_by_name[('PRV', '1')]
area = areas_by_name[strip_accents(province).upper()]
post_id = AREA_POST_DATA.get_post_id(ynr_election_id, area['type'],
area['id'])
else:
# It must be the presidential election:
post_id = 'presidente'
post_data = get_post_cached(api, post_id)['result']
return ynr_election_data, post_data
def get_post_data(api, json_election_id, json_election_id_to_name):
json_election_name = json_election_id_to_name[json_election_id]
ynr_election_id = {
u'Pre-candidatos a Presidente':
'presidentes-argentina-paso-2015',
u'Pre-candidatos a Gobernador de Buenos Aires':
'gobernadores-argentina-paso-2015',
u'Pre-candidatos a Gobernador de Tucumán':
'gobernadores-argentina-paso-2015',
u'Pre-candidatos a Gobernador de Entre Ríos':
'gobernadores-argentina-paso-2015',
u'Pre-Candidatos a Gobernador de San Juan':
'gobernadores-argentina-paso-2015',
}[json_election_name]
ynr_election_data = settings.ELECTIONS[ynr_election_id]
ynr_election_data['id'] = ynr_election_id
province = None
m = re.search(r'a Gobernador de (?P<province>.*)', json_election_name)
if m:
province = m.group('province')
mapit_areas_by_name = MAPIT_DATA.areas_by_name[('PRV', 1)]
mapit_area = mapit_areas_by_name[strip_accents(province).upper()]
post_id = AREA_POST_DATA.get_post_id(
ynr_election_id, mapit_area['type'], mapit_area['id']
)
else:
# It must be the presidential election:
post_id = 'presidente'
post_data = get_post_cached(api, post_id)['result']
return ynr_election_data, post_data
def get_post_data(api, election_id, province):
ynr_election_data = Election.objects.get_by_slug(election_id)
area_key = (ynr_election_data.area_types.first().name,
ynr_election_data.area_generation)
areas_by_name = AREA_DATA.areas_by_name[area_key]
if province != 'Burkina Faso':
province = strip_accents(province).upper()
area = areas_by_name[province]
post_id = AREA_POST_DATA.get_post_id(
election_id, area['type'], area['id']
)
post_data = get_post_cached(api, post_id)['result']
return ynr_election_data, post_data
def get_post_data(api, election_id, province):
from candidates.cache import get_post_cached
from candidates.election_specific import AREA_DATA, AREA_POST_DATA
ynr_election_data = Election.objects.get_by_slug(election_id)
area_key = (ynr_election_data.area_types.first().name,
ynr_election_data.area_generation)
areas_by_name = AREA_DATA.areas_by_name[area_key]
if province != 'Burkina Faso':
province = strip_accents(province).upper()
area = areas_by_name[province]
post_id = AREA_POST_DATA.get_post_id(election_id, area['type'], area['id'])
post_data = get_post_cached(api, post_id)['result']
return ynr_election_data, post_data
def handle_mapit_type(self, election, election_data, mapit_type):
mapit_tuple = (mapit_type, election_data['mapit_generation'])
for id, area in MAPIT_DATA.areas_by_id[mapit_tuple].items():
post_id = AREA_POST_DATA.get_post_id(
election, mapit_type, id
)
role = election_data['for_post_role']
area_mapit_url = settings.MAPIT_BASE_URL + 'area/' + str(area['id'])
post_data = {
'role': role,
'id': post_id,
'label': role + u' por ' + area['name'],
'area': {
'name': area['name'],
'id': 'mapit:' + str(area['id']),
'identifier': area_mapit_url,
},
'organization_id': election_data['organization_id'],
}
create_or_update(self.api.posts, post_data)
def handle(self, **options):
repo_root = realpath(join(dirname(__file__), '..', '..', '..'))
output_filename = join(
repo_root,
'elections',
settings.ELECTION_APP,
'static',
'js',
'post-to-party-set.js',
)
with open(output_filename, 'w') as f:
f.write('var postIDToPartySet = ')
mapping = {
post['id']: AREA_POST_DATA.post_id_to_party_set(post['id'])
for election, election_data in settings.ELECTIONS_CURRENT
for post in get_all_posts(election_data['for_post_role'])
}
unknown_post_ids = [k for k, v in mapping.items() if v is None]
f.write(json.dumps(mapping))
f.write(';\n')
if unknown_post_ids:
print "Warning: no party set could be found for these post IDs:"
print unknown_post_ids
def handle(self, **options):
api = create_popit_api_object()
csv_filename = join(
dirname(__file__), '..', '..','data', 'candidates.csv'
)
with open(csv_filename) as f:
all_data = csv.DictReader(f)
for candidate in all_data:
vi_person_id = candidate['Distrito']+candidate['Numero Lista']+candidate['Posicion']+candidate['Cargo']+candidate['Nombre Lista']
election_data, post_data = get_post_data(
api, candidate['Cargo'], candidate['Distrito']
)
if (election_data == False):
print "Skipping: "+ candidate['Cargo'] +", " + candidate['Distrito']+", " + candidate['Nombre']
continue;
name = candidate['Nombre']
birth_date = None
gender = None
image_url = None
person = get_existing_popit_person(vi_person_id)
if person:
print "Found an existing person:", person.get_absolute_url()
else:
print "No existing person, creating a new one:", name
person = PopItPerson()
# Now update fields from the imported data:
person.name = name.split(",")[1] + " " + name.split(",")[0]
person.gender = gender
if birth_date:
person.birth_date = str(birth_date)
else:
person.birth_date = None
standing_in_election = {
'post_id': post_data['id'],
'name': AREA_POST_DATA.shorten_post_label(
election_data.slug,
post_data['label'],
),
'party_list_position': candidate['Posicion'],
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area']['identifier']
person.standing_in = {
election_data.slug: standing_in_election
}
party_id = get_party_id(candidate["Partido"]);
person.party_memberships = {
election_data.slug: {
'id': party_id,
'name': PARTY_DATA.party_id_to_name[party_id],
}
}
person.set_identifier('import-id', vi_person_id)
change_metadata = get_change_metadata(
None,
'Imported candidate from CSV',
)
person.record_version(change_metadata)
try:
person.save_to_popit(api)
except HttpClientError as hce:
print "Got an HttpClientError:", hce.content
raise
except HttpServerError as hse:
print "The server error content was:", hse.content
raise
def handle(self, username=None, **options):
election_data = {
'prv-2015': 'listedescandidatsauxelectionslegislativeslisteprovincialeanptic.csv',
'nat-2015': 'listedescandidatsauxelectionslegislativesanptic.csv'
}
field_map = {
'prv-2015': {
'region': 1,
'party': 4,
'list_order': 5,
'first_name': 7,
'last_name': 6,
'gender': 8,
'birth_date': 9,
'party_short': 3
},
'nat-2015': {
'region': 0,
'party': 2,
'list_order': 3,
'first_name': 5,
'last_name': 4,
'gender': 6,
'birth_date': 7,
'party_short': 2
}
}
api = create_popit_api_object()
party_id_missing = {}
party_name_to_id = {}
for party_id, party_name in PARTY_DATA.party_id_to_name.items():
party_name_to_id[party_name] = party_id
for election_id, filename in election_data.items():
csv_filename = join(
dirname(__file__), '..', '..', 'data', filename
)
fields = field_map[election_id]
with codecs.open(csv_filename, 'r', encoding='windows-1252') as f:
initial = True
for candidate in unicode_csv_reader(f):
# skip header line
if initial:
initial = False
continue
region = candidate[fields['region']]
party = candidate[fields['party']]
party_list_order = candidate[fields['list_order']]
first_name = string.capwords(candidate[fields['first_name']])
last_name = string.capwords(candidate[fields['last_name']])
gender = candidate[fields['gender']]
birth_date = None
if candidate[fields['birth_date']] is not None:
birth_date = str(dateutil.parser.parse(
candidate[fields['birth_date']], dayfirst=True
).date())
name = first_name + ' ' + last_name
id = '-'.join([
re.sub('[^\w]*', '', re.sub(r' ', '-', strip_accents(name.lower()))),
re.sub('[^\w]*', '', candidate[fields['party_short']].lower()),
birth_date
])
# national candidate
if region == 'PAYS':
region = 'Burkina Faso'
election_data, post_data = get_post_data(
api, election_id, region
)
# debug
# tmp = '%s %s %s (%s) - %s (%s)' % ( id, first_name, last_name, party, region, post_data['label'] )
# print tmp
person = get_existing_popit_person(id)
if person:
# print "Found an existing person:", person.get_absolute_url()
pass
else:
print "No existing person, creating a new one:", name
person = PopItPerson()
person.set_identifier('import-id', id)
person.family_name = last_name
person.given_name = first_name
person.name = name
person.gender = gender
if birth_date:
person.birth_date = str(birth_date)
else:
person.birth_date = None
standing_in_election = {
'post_id': post_data['id'],
'name': AREA_POST_DATA.shorten_post_label(
election_data.slug,
post_data['label'],
),
'party_list_position': party_list_order,
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area']['identifier']
person.standing_in = {
election_data.slug: standing_in_election
}
change_metadata = get_change_metadata(
None,
'Imported candidate from CSV',
)
party_comp = re.sub(' +', ' ', party)
party_id = UNKNOWN_PARTY_ID
if party_comp in party_name_to_id.keys():
party_id = party_name_to_id[party_comp]
party = party_comp
else:
party_id = party_name_to_id['Unknown Party']
party = 'Unknown Party'
if party_id == UNKNOWN_PARTY_ID and party_comp not in party_id_missing.keys():
party_id_missing[party_comp] = 1
person.party_memberships = {
election_data.slug: {
'id': party_id,
'name': party,
'imported_name': party_comp
}
}
person.record_version(change_metadata)
try:
person.save_to_popit(api)
except HttpClientError as hce:
print "Got an HttpClientError:", hce.content
raise
if len(party_id_missing) > 0:
print "Unmatched party names:"
for name in party_id_missing.keys():
print name
def handle(self, username=None, **options):
if username is None:
message = "You must supply the name of a user to be associated with the image uploads."
raise CommandError(message)
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
message = "No user with the username '{0}' could be found"
raise CommandError(message.format(username))
api = create_popit_api_object()
json_filename = join(dirname(__file__), '..', '..', 'data',
'candidates.json')
with open(json_filename) as f:
all_data = json.load(f)
# This map is needed to get getting YNR election data from
# the election ID used in the JSON file.
json_election_id_to_name = {
e['pk']: e['fields']['name']
for e in all_data if e['model'] == 'elections.election'
}
person_dict = {
e['pk']: e['fields']
for e in all_data if e['model'] == 'popolo.person'
}
candidate_list = [
dict(person_id=e['pk'], election_id=e['fields']['election'])
for e in all_data if e['model'] == 'elections.candidate'
]
for candidate in candidate_list:
vi_person_id = candidate['person_id']
person_data = person_dict[vi_person_id]
election_data, post_data = get_post_data(api,
candidate['election_id'],
json_election_id_to_name)
birth_date = None
if person_data['birth_date']:
birth_date = str(
dateutil.parser.parse(person_data['birth_date'],
dayfirst=True).date())
name = person_data['name']
gender = person_data['gender']
image_url = person_data['image']
person = get_existing_popit_person(vi_person_id)
if person:
print "Found an existing person:", person.get_absolute_url()
else:
print "No existing person, creating a new one:", name
person = PopItPerson()
# Now update fields from the imported data:
person.name = name
person.gender = gender
if birth_date:
person.birth_date = str(birth_date)
else:
person.birth_date = None
standing_in_election = {
'post_id':
post_data['id'],
'name':
AREA_POST_DATA.shorten_post_label(
election_data['id'],
post_data['label'],
),
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area'][
'identifier']
person.standing_in = {election_data['id']: standing_in_election}
person.party_memberships = {
election_data['id']: {
'id': UNKNOWN_PARTY_ID,
'name': PARTY_DATA.party_id_to_name[UNKNOWN_PARTY_ID],
}
}
person.set_identifier('import-id', vi_person_id)
change_metadata = get_change_metadata(
None,
'Imported candidate from JSON',
)
person.record_version(change_metadata)
try:
person.save_to_popit(api)
if image_url:
enqueue_image(person, user, image_url)
except HttpClientError as hce:
print "Got an HttpClientError:", hce.content
raise
def handle(self, username=None, **options):
if username is None:
message = "You must supply the name of a user to be associated with the image uploads."
raise CommandError(message)
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
message = "No user with the username '{0}' could be found"
raise CommandError(message.format(username))
api = create_popit_api_object()
json_filename = join(
dirname(__file__), '..', '..','data', 'candidates.json'
)
with open(json_filename) as f:
all_data = json.load(f)
# This map is needed to get getting YNR election data from
# the election ID used in the JSON file.
json_election_id_to_name = {
e['pk']: e['fields']['name']
for e in all_data if e['model'] == 'elections.election'
}
person_dict = {
e['pk']: e['fields']
for e in all_data if e['model'] == 'popolo.person'
}
candidate_list = [
dict(person_id=e['pk'], election_id=e['fields']['election'])
for e in all_data if e['model'] == 'elections.candidate'
]
for candidate in candidate_list:
vi_person_id = candidate['person_id']
person_data = person_dict[vi_person_id]
election_data, post_data = get_post_data(
api, candidate['election_id'], json_election_id_to_name
)
birth_date = None
if person_data['birth_date']:
birth_date = str(dateutil.parser.parse(
person_data['birth_date'], dayfirst=True
).date())
name = person_data['name']
gender = person_data['gender']
image_url = person_data['image']
person = get_existing_popit_person(vi_person_id)
if person:
print "Found an existing person:", person.get_absolute_url()
else:
print "No existing person, creating a new one:", name
person = PopItPerson()
# Now update fields from the imported data:
person.name = name
person.gender = gender
if birth_date:
person.birth_date = str(birth_date)
else:
person.birth_date = None
standing_in_election = {
'post_id': post_data['id'],
'name': AREA_POST_DATA.shorten_post_label(
election_data['id'],
post_data['label'],
),
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area']['identifier']
person.standing_in = {
election_data['id']: standing_in_election
}
person.party_memberships = {
election_data['id']: {
'id': UNKNOWN_PARTY_ID,
'name': PARTY_DATA.party_id_to_name[UNKNOWN_PARTY_ID],
}
}
person.set_identifier('import-id', vi_person_id)
change_metadata = get_change_metadata(
None,
'Imported candidate from JSON',
)
person.record_version(change_metadata)
try:
person.save_to_popit(api)
if image_url:
enqueue_image(person, user, image_url)
except HttpClientError as hce:
print "Got an HttpClientError:", hce.content
raise
