def handle(self, **options):
from slumber.exceptions import HttpClientError, HttpServerError
from candidates.election_specific import PARTY_DATA, shorten_post_label
from candidates.models import PopItPerson
from candidates.popit import create_popit_api_object
api = create_popit_api_object()
csv_filename = join(dirname(__file__), '..', '..', 'data',
'candidates.csv')
with open(csv_filename) as f:
all_data = csv.DictReader(f)
for candidate in all_data:
vi_person_id = candidate['Distrito'] + candidate[
'Numero Lista'] + candidate['Posicion'] + candidate[
'Cargo'] + candidate['Nombre Lista']
election_data, post_data = get_post_data(
api, candidate['Cargo'], candidate['Distrito'])
if (election_data == False):
print("Skipping: " + candidate['Cargo'] + ", " +
candidate['Distrito'] + ", " + candidate['Nombre'])
continue
name = candidate['Nombre']
birth_date = None
gender = None
image_url = None
person = get_existing_popit_person(vi_person_id)
if person:
print("Found an existing person:",
person.get_absolute_url())
else:
print("No existing person, creating a new one:", name)
person = PopItPerson()
# Now update fields from the imported data:
person.name = name.split(",")[1] + " " + name.split(",")[0]
person.gender = gender
if birth_date:
person.birth_date = str(birth_date)
else:
person.birth_date = None
standing_in_election = {
'post_id': post_data['id'],
'name': shorten_post_label(post_data['label']),
'party_list_position': candidate['Posicion'],
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area'][
'identifier']
person.standing_in = {election_data.slug: standing_in_election}
party_id = get_party_id(candidate["Partido"])
person.party_memberships = {
election_data.slug: {
'id': party_id,
'name': PARTY_DATA.party_id_to_name[party_id],
}
}
person.set_identifier('import-id', vi_person_id)
change_metadata = get_change_metadata(
None,
'Imported candidate from CSV',
)
person.record_version(change_metadata)
try:
person.save_to_popit(api)
except HttpClientError as hce:
print("Got an HttpClientError:", hce.content)
raise
except HttpServerError as hse:
print("The server error content was:", hse.content)
raise
def handle(self, username=None, **options):
from slumber.exceptions import HttpClientError
from candidates.popit import create_popit_api_object
from candidates.election_specific import PARTY_DATA, shorten_post_label
from candidates.models import PopItPerson
if username is None:
message = "You must supply the name of a user to be associated with the image uploads."
raise CommandError(message)
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
message = "No user with the username '{0}' could be found"
raise CommandError(message.format(username))
api = create_popit_api_object()
json_filename = join(
dirname(__file__), '..', '..','data', 'candidates.json'
)
with open(json_filename) as f:
all_data = json.load(f)
# This map is needed to get getting YNR election data from
# the election ID used in the JSON file.
json_election_id_to_name = {
e['pk']: e['fields']['name']
for e in all_data if e['model'] == 'elections.election'
}
person_dict = {
e['pk']: e['fields']
for e in all_data if e['model'] == 'popolo.person'
}
candidate_list = [
dict(person_id=e['pk'], election_id=e['fields']['election'])
for e in all_data if e['model'] == 'elections.candidate'
]
for candidate in candidate_list:
vi_person_id = candidate['person_id']
person_data = person_dict[vi_person_id]
election_data, post_data = get_post_data(
api, candidate['election_id'], json_election_id_to_name
)
birth_date = None
if person_data['birth_date']:
birth_date = str(dateutil.parser.parse(
person_data['birth_date'], dayfirst=True
).date())
name = person_data['name']
gender = person_data['gender']
image_url = person_data['image']
person = get_existing_popit_person(vi_person_id)
if person:
print("Found an existing person:", person.get_absolute_url())
else:
print("No existing person, creating a new one:", name)
person = PopItPerson()
# Now update fields from the imported data:
person.name = name
person.gender = gender
if birth_date:
person.birth_date = str(birth_date)
else:
person.birth_date = None
standing_in_election = {
'post_id': post_data['id'],
'name': shorten_post_label(post_data['label']),
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area']['identifier']
person.standing_in = {
election_data.slug: standing_in_election
}
person.party_memberships = {
election_data.slug: {
'id': UNKNOWN_PARTY_ID,
'name': PARTY_DATA.party_id_to_name[UNKNOWN_PARTY_ID],
}
}
person.set_identifier('import-id', vi_person_id)
change_metadata = get_change_metadata(
None,
'Imported candidate from JSON',
)
person.record_version(change_metadata)
try:
person.save_to_popit(api)
if image_url:
enqueue_image(person, user, image_url)
except HttpClientError as hce:
print("Got an HttpClientError:", hce.content)
raise
def handle(self, username=None, **options):
election_data = {
'prv-2015': 'listedescandidatsauxelectionslegislativeslisteprovincialeanptic.csv',
'nat-2015': 'listedescandidatsauxelectionslegislativesanptic.csv'
}
field_map = {
'prv-2015': {
'region': 1,
'party': 4,
'list_order': 5,
'first_name': 7,
'last_name': 6,
'gender': 8,
'birth_date': 9,
'party_short': 3
},
'nat-2015': {
'region': 0,
'party': 2,
'list_order': 3,
'first_name': 5,
'last_name': 4,
'gender': 6,
'birth_date': 7,
'party_short': 2
}
}
api = create_popit_api_object()
party_id_missing = {}
party_name_to_id = {}
for party_id, party_name in PARTY_DATA.party_id_to_name.items():
party_name_to_id[party_name] = party_id
for election_id, filename in election_data.items():
csv_filename = join(
dirname(__file__), '..', '..', 'data', filename
)
fields = field_map[election_id]
with codecs.open(csv_filename, 'r', encoding='windows-1252') as f:
initial = True
for candidate in unicode_csv_reader(f):
# skip header line
if initial:
initial = False
continue
region = candidate[fields['region']]
party = candidate[fields['party']]
party_list_order = candidate[fields['list_order']]
first_name = string.capwords(candidate[fields['first_name']])
last_name = string.capwords(candidate[fields['last_name']])
gender = candidate[fields['gender']]
birth_date = None
if candidate[fields['birth_date']] is not None:
birth_date = str(dateutil.parser.parse(
candidate[fields['birth_date']], dayfirst=True
).date())
name = first_name + ' ' + last_name
id = '-'.join([
re.sub('[^\w]*', '', re.sub(r' ', '-', strip_accents(name.lower()))),
re.sub('[^\w]*', '', candidate[fields['party_short']].lower()),
birth_date
])
# national candidate
if region == 'PAYS':
region = 'Burkina Faso'
election_data, post_data = get_post_data(
api, election_id, region
)
# debug
# tmp = '%s %s %s (%s) - %s (%s)' % ( id, first_name, last_name, party, region, post_data['label'] )
# print tmp
person = get_existing_popit_person(id)
if person:
# print "Found an existing person:", person.get_absolute_url()
pass
else:
print "No existing person, creating a new one:", name
person = PopItPerson()
person.set_identifier('import-id', id)
person.family_name = last_name
person.given_name = first_name
person.name = name
person.gender = gender
if birth_date:
person.birth_date = str(birth_date)
else:
person.birth_date = None
standing_in_election = {
'post_id': post_data['id'],
'name': AREA_POST_DATA.shorten_post_label(
election_data.slug,
post_data['label'],
),
'party_list_position': party_list_order,
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area']['identifier']
person.standing_in = {
election_data.slug: standing_in_election
}
change_metadata = get_change_metadata(
None,
'Imported candidate from CSV',
)
party_comp = re.sub(' +', ' ', party)
party_id = UNKNOWN_PARTY_ID
if party_comp in party_name_to_id.keys():
party_id = party_name_to_id[party_comp]
party = party_comp
else:
party_id = party_name_to_id['Unknown Party']
party = 'Unknown Party'
if party_id == UNKNOWN_PARTY_ID and party_comp not in party_id_missing.keys():
party_id_missing[party_comp] = 1
person.party_memberships = {
election_data.slug: {
'id': party_id,
'name': party,
'imported_name': party_comp
}
}
person.record_version(change_metadata)
try:
person.save_to_popit(api)
except HttpClientError as hce:
print "Got an HttpClientError:", hce.content
raise
if len(party_id_missing) > 0:
print "Unmatched party names:"
for name in party_id_missing.keys():
print name
def handle(self, username=None, **options):
from slumber.exceptions import HttpClientError
from candidates.election_specific import PARTY_DATA, shorten_post_label
from candidates.models import PopItPerson
from candidates.popit import create_popit_api_object
election_data = {
'prv-2015':
'listedescandidatsauxelectionslegislativeslisteprovincialeanptic.csv',
'nat-2015': 'listedescandidatsauxelectionslegislativesanptic.csv'
}
field_map = {
'prv-2015': {
'region': 1,
'party': 4,
'list_order': 5,
'first_name': 7,
'last_name': 6,
'gender': 8,
'birth_date': 9,
'party_short': 3
},
'nat-2015': {
'region': 0,
'party': 2,
'list_order': 3,
'first_name': 5,
'last_name': 4,
'gender': 6,
'birth_date': 7,
'party_short': 2
}
}
api = create_popit_api_object()
party_id_missing = {}
party_name_to_id = {}
for party_id, party_name in PARTY_DATA.party_id_to_name.items():
party_name_to_id[party_name] = party_id
for election_id, filename in election_data.items():
csv_filename = join(dirname(__file__), '..', '..', 'data',
filename)
fields = field_map[election_id]
with codecs.open(csv_filename, 'r', encoding='windows-1252') as f:
initial = True
for candidate in unicode_csv_reader(f):
# skip header line
if initial:
initial = False
continue
region = candidate[fields['region']]
party = candidate[fields['party']]
party_list_order = candidate[fields['list_order']]
first_name = string.capwords(
candidate[fields['first_name']])
last_name = string.capwords(candidate[fields['last_name']])
gender = candidate[fields['gender']]
birth_date = None
if candidate[fields['birth_date']] is not None:
birth_date = str(
dateutil.parser.parse(
candidate[fields['birth_date']],
dayfirst=True).date())
name = first_name + ' ' + last_name
id = '-'.join([
re.sub('[^\w]*', '',
re.sub(r' ', '-', strip_accents(name.lower()))),
re.sub('[^\w]*', '',
candidate[fields['party_short']].lower()),
birth_date
])
# national candidate
if region == 'PAYS':
region = 'Burkina Faso'
election_data, post_data = get_post_data(
api, election_id, region)
# debug
# tmp = '%s %s %s (%s) - %s (%s)' % ( id, first_name, last_name, party, region, post_data['label'] )
# print(tmp)
person = get_existing_popit_person(id)
if person:
# print("Found an existing person:", person.get_absolute_url())
pass
else:
print("No existing person, creating a new one:", name)
person = PopItPerson()
person.set_identifier('import-id', id)
person.family_name = last_name
person.given_name = first_name
person.name = name
person.gender = gender
if birth_date:
person.birth_date = str(birth_date)
else:
person.birth_date = None
standing_in_election = {
'post_id': post_data['id'],
'name': shorten_post_label(post_data['label']),
'party_list_position': party_list_order,
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area'][
'identifier']
person.standing_in = {
election_data.slug: standing_in_election
}
change_metadata = get_change_metadata(
None,
'Imported candidate from CSV',
)
party_comp = re.sub(' +', ' ', party)
party_id = UNKNOWN_PARTY_ID
if party_comp in party_name_to_id.keys():
party_id = party_name_to_id[party_comp]
party = party_comp
else:
party_id = party_name_to_id['Unknown Party']
party = 'Unknown Party'
if party_id == UNKNOWN_PARTY_ID and party_comp not in party_id_missing.keys(
):
party_id_missing[party_comp] = 1
person.party_memberships = {
election_data.slug: {
'id': party_id,
'name': party,
'imported_name': party_comp
}
}
person.record_version(change_metadata)
try:
person.save_to_popit(api)
except HttpClientError as hce:
print("Got an HttpClientError:", hce.content)
raise
if len(party_id_missing) > 0:
print("Unmatched party names:")
for name in party_id_missing.keys():
print(name)
def handle(self, **options):
from slumber.exceptions import HttpClientError, HttpServerError
from candidates.election_specific import PARTY_DATA, shorten_post_label
from candidates.models import PopItPerson
from candidates.popit import create_popit_api_object
api = create_popit_api_object()
csv_filename = join(
dirname(__file__), '..', '..','data', 'candidates.csv'
)
with open(csv_filename) as f:
all_data = csv.DictReader(f)
for candidate in all_data:
vi_person_id = candidate['Distrito']+candidate['Numero Lista']+candidate['Posicion']+candidate['Cargo']+candidate['Nombre Lista']
election_data, post_data = get_post_data(
api, candidate['Cargo'], candidate['Distrito']
)
if (election_data == False):
print("Skipping: "+ candidate['Cargo'] +", " + candidate['Distrito']+", " + candidate['Nombre'])
continue;
name = candidate['Nombre']
birth_date = None
gender = None
image_url = None
person = get_existing_popit_person(vi_person_id)
if person:
print("Found an existing person:", person.get_absolute_url())
else:
print("No existing person, creating a new one:", name)
person = PopItPerson()
# Now update fields from the imported data:
person.name = name.split(",")[1] + " " + name.split(",")[0]
person.gender = gender
if birth_date:
person.birth_date = str(birth_date)
else:
person.birth_date = None
standing_in_election = {
'post_id': post_data['id'],
'name': shorten_post_label(post_data['label']),
'party_list_position': candidate['Posicion'],
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area']['identifier']
person.standing_in = {
election_data.slug: standing_in_election
}
party_id = get_party_id(candidate["Partido"]);
person.party_memberships = {
election_data.slug: {
'id': party_id,
'name': PARTY_DATA.party_id_to_name[party_id],
}
}
person.set_identifier('import-id', vi_person_id)
change_metadata = get_change_metadata(
None,
'Imported candidate from CSV',
)
person.record_version(change_metadata)
try:
person.save_to_popit(api)
except HttpClientError as hce:
print("Got an HttpClientError:", hce.content)
raise
except HttpServerError as hse:
print("The server error content was:", hse.content)
raise
def handle(self, username=None, **options):
from slumber.exceptions import HttpClientError
from candidates.popit import create_popit_api_object
from candidates.election_specific import PARTY_DATA, shorten_post_label
from candidates.models import PopItPerson
if username is None:
message = "You must supply the name of a user to be associated with the image uploads."
raise CommandError(message)
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
message = "No user with the username '{0}' could be found"
raise CommandError(message.format(username))
api = create_popit_api_object()
json_filename = join(dirname(__file__), '..', '..', 'data',
'candidates.json')
with open(json_filename) as f:
all_data = json.load(f)
# This map is needed to get getting YNR election data from
# the election ID used in the JSON file.
json_election_id_to_name = {
e['pk']: e['fields']['name']
for e in all_data if e['model'] == 'elections.election'
}
person_dict = {
e['pk']: e['fields']
for e in all_data if e['model'] == 'popolo.person'
}
candidate_list = [
dict(person_id=e['pk'], election_id=e['fields']['election'])
for e in all_data if e['model'] == 'elections.candidate'
]
for candidate in candidate_list:
vi_person_id = candidate['person_id']
person_data = person_dict[vi_person_id]
election_data, post_data = get_post_data(api,
candidate['election_id'],
json_election_id_to_name)
birth_date = None
if person_data['birth_date']:
birth_date = str(
dateutil.parser.parse(person_data['birth_date'],
dayfirst=True).date())
name = person_data['name']
gender = person_data['gender']
image_url = person_data['image']
person = get_existing_popit_person(vi_person_id)
if person:
print("Found an existing person:", person.get_absolute_url())
else:
print("No existing person, creating a new one:", name)
person = PopItPerson()
# Now update fields from the imported data:
person.name = name
person.gender = gender
if birth_date:
person.birth_date = str(birth_date)
else:
person.birth_date = None
standing_in_election = {
'post_id': post_data['id'],
'name': shorten_post_label(post_data['label']),
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area'][
'identifier']
person.standing_in = {election_data.slug: standing_in_election}
person.party_memberships = {
election_data.slug: {
'id': UNKNOWN_PARTY_ID,
'name': PARTY_DATA.party_id_to_name[UNKNOWN_PARTY_ID],
}
}
person.set_identifier('import-id', vi_person_id)
change_metadata = get_change_metadata(
None,
'Imported candidate from JSON',
)
person.record_version(change_metadata)
try:
person.save_to_popit(api)
if image_url:
enqueue_image(person, user, image_url)
except HttpClientError as hce:
print("Got an HttpClientError:", hce.content)
raise