def handle_noargs(self, **options):
api_key = hmac.new(settings.IEBC_API_SECRET,
"appid=%s" % (settings.IEBC_API_ID,),
hashlib.sha256).hexdigest()
token_data = get_data(make_api_token_url(settings.IEBC_API_ID, api_key))
token = token_data['token']
def url(path, query_filter=None):
"""A closure to avoid repeating parameters"""
return make_api_url(path, settings.IEBC_API_SECRET, token, query_filter)
cache_directory = os.path.join(data_directory, 'api-cache-2013-02-14')
mkdir_p(cache_directory)
for area_type in 'county', 'constituency', 'ward':
cache_filename = os.path.join(cache_directory, area_type + '.json')
area_type_data = get_data_with_cache(cache_filename, url('/%s/' % (area_type)))
areas = area_type_data['region']['locations']
for i, area in enumerate(areas):
# Get the candidates for that area:
code = area['code']
candidates_cache_filename = os.path.join(cache_directory, 'candidates-for-' + area_type + '-' + code + '.json')
candidate_data = get_data_with_cache(candidates_cache_filename, url('/candidate/', query_filter='%s=%s' % (area_type, code)))
for race in candidate_data['candidates']:
place_name = race['race']
candidates = race['candidates']
if len(candidates) == 0:
continue
distinct_contest_types = set(c['contest_type'] for c in race['candidates'])
if len(distinct_contest_types) != 1:
print "Multiple contest types found for %s: %s" % (place_name, distinct_contest_types)
contest_type = iter(distinct_contest_types).next()
place_kind, session, title, race_type = known_race_type_mapping[contest_type]
if options['place'] and options['place'].lower() != place_name.lower():
continue
remove_duplicate_candidates_for_place(place_name,
place_kind,
session,
title,
race_type,
candidates,
**options)
def handle_noargs(self, **options):
api_key = hmac.new(settings.IEBC_API_SECRET,
"appid=%s" % (settings.IEBC_API_ID,),
hashlib.sha256).hexdigest()
token_data = get_data(make_api_token_url(settings.IEBC_API_ID, api_key))
token = token_data['token']
def url(path, query_filter=None):
"""A closure to avoid repeating parameters"""
return make_api_url(path, settings.IEBC_API_SECRET, token, query_filter)
# To get all the candidates, we iterate over each county,
# constituency and ward, and request the candidates for each.
cache_directory = os.path.join(data_directory, 'api-cache-2013-02-08')
mkdir_p(cache_directory)
parties_cache_filename = os.path.join(cache_directory, 'parties')
party_data = get_data_with_cache(parties_cache_filename, url('/party/'), only_from_cache=True)
for party in party_data['parties']:
party_api_code = party['code']
try:
party_organisation = get_matching_party(party['name'], create=False, **options)
except CommandError, ce:
print >> sys.stderr, "Not setting the API code: %s" % (ce,)
party_organisation.external_id = party_api_code
maybe_save(party_organisation, **options)
def handle_noargs(self, **options):
api_key = hmac.new(settings.IEBC_API_SECRET,
"appid=%s" % (settings.IEBC_API_ID, ),
hashlib.sha256).hexdigest()
token_data = get_data(make_api_token_url(settings.IEBC_API_ID,
api_key))
token = token_data['token']
def url(path, query_filter=None):
"""A closure to avoid repeating parameters"""
return make_api_url(path, settings.IEBC_API_SECRET, token,
query_filter)
# To get all the candidates, we iterate over each county,
# constituency and ward, and request the candidates for each.
cache_directory = os.path.join(data_directory, 'api-cache-2013-02-08')
mkdir_p(cache_directory)
parties_cache_filename = os.path.join(cache_directory, 'parties')
party_data = get_data_with_cache(parties_cache_filename,
url('/party/'),
only_from_cache=True)
for party in party_data['parties']:
party_api_code = party['code']
try:
party_organisation = get_matching_party(party['name'],
create=False,
**options)
except CommandError, ce:
print >> sys.stderr, "Not setting the API code: %s" % (ce, )
party_organisation.external_id = party_api_code
maybe_save(party_organisation, **options)
def handle_noargs(self, **options):
api_key = hmac.new(settings.IEBC_API_SECRET,
"appid=%s" % (settings.IEBC_API_ID, ),
hashlib.sha256).hexdigest()
token_data = get_data(make_api_token_url(settings.IEBC_API_ID,
api_key))
token = token_data['token']
def url(path, query_filter=None):
"""A closure to avoid repeating parameters"""
return make_api_url(path, settings.IEBC_API_SECRET, token,
query_filter)
# To get all the candidates, we iterate over each county,
# constituency and ward, and request the candidates for each.
cache_directory = os.path.join(data_directory, 'api-cache-2013-02-08')
mkdir_p(cache_directory)
# ------------------------------------------------------------------------
print "Getting (incorrect) party names mapping for matching..."
parties_cache_filename = os.path.join(cache_directory, 'parties')
party_data = get_data_with_cache(parties_cache_filename,
url('/party/'))
party_names_api = set(d['name'].strip().encode('utf-8')
for d in party_data['parties'])
party_names_db = set(
o.name.strip().encode('utf-8')
for o in Organisation.objects.filter(kind__slug='party'))
with open(os.path.join(data_directory, 'party-names.csv'), 'w') as fp:
writer = csv.writer(fp)
for t in itertools.izip_longest(party_names_api, party_names_db):
writer.writerow(t)
# ------------------------------------------------------------------------
print "Getting (incorrect) ward names mapping for matching..."
ward_data = get_data(url('/ward/'))
wards_from_api = sorted(ward['name'].encode('utf-8')
for ward in ward_data['region']['locations'])
wards_from_db = sorted(
p.name.encode('utf-8')
for p in Place.objects.filter(kind__slug='ward'))
with open(os.path.join(data_directory, 'wards-names.csv'), 'w') as fp:
writer = csv.writer(fp)
for t in itertools.izip_longest(wards_from_api, wards_from_db):
writer.writerow(t)
# ------------------------------------------------------------------------
headings = [
'Same/Different', 'API Name', 'API Party', 'API Place',
'API Candidate Code', 'Mz Legal Name', 'Mz Other Names', 'Mz URL',
'Mz Parties Ever', 'Mz Aspirant Ever', 'Mz Politician Ever',
'Mz ID'
]
with open(os.path.join(sys.path[0], 'names-to-check.csv'), 'w') as fp:
writer = csv.DictWriter(fp, headings)
writer.writerow(dict((h, h) for h in headings))
for area_type in 'county', 'constituency', 'ward':
cache_filename = os.path.join(cache_directory, area_type)
area_type_data = get_data_with_cache(cache_filename,
url('/%s/' % (area_type)))
areas = area_type_data['region']['locations']
for i, area in enumerate(areas):
# Get the candidates for that area:
code = area['code']
candidates_cache_filename = os.path.join(
cache_directory,
'candidates-for-' + area_type + '-' + code)
candidate_data = get_data_with_cache(
candidates_cache_filename,
url('/candidate/',
query_filter='%s=%s' % (area_type, code)))
for race in candidate_data['candidates']:
full_race_name = race['race']
race_type, place_name = parse_race_name(full_race_name)
place_kind, session, title = known_race_type_mapping[
race_type]
candidates = race['candidates']
for candidate in candidates:
first_names = candidate['other_name'] or ''
surname = candidate['surname'] or ''
person = get_person_from_names(
first_names, surname)
if person:
print "Got person match to:", person
row = {}
row['Same/Different'] = ''
row['API Name'] = first_names + ' ' + surname
party_data = candidate['party']
row['API Party'] = party_data[
'name'] if 'name' in party_data else ''
row['API Place'] = '%s (%s)' % (place_name,
area_type)
row['API Candidate Code'] = candidate['code']
row['Mz Legal Name'] = person.legal_name
row['Mz Other Names'] = person.other_names
row['Mz URL'] = 'http://info.mzalendo.com' + person.get_absolute_url(
)
row['Mz Parties Ever'] = ', '.join(
o.name for o in person.parties_ever())
for heading, positions in (
('Mz Aspirant Ever',
person.aspirant_positions_ever()),
('Mz Politician Ever',
person.politician_positions_ever())):
row[heading] = ', '.join(
'%s at %s' % (p.title.name, p.place)
for p in positions)
row['Mz ID'] = person.id
for key, value in row.items():
row[key] = unicode(value).encode('utf-8')
writer.writerow(row)
def handle_noargs(self, **options):
api_key = hmac.new(settings.IEBC_API_SECRET,
"appid=%s" % (settings.IEBC_API_ID,),
hashlib.sha256).hexdigest()
token_data = get_data(make_api_token_url(settings.IEBC_API_ID, api_key))
token = token_data['token']
def url(path, query_filter=None):
"""A closure to avoid repeating parameters"""
return make_api_url(path, settings.IEBC_API_SECRET, token, query_filter)
# To get all the candidates, we iterate over each county,
# constituency and ward, and request the candidates for each.
cache_directory = os.path.join(data_directory, 'api-cache-2013-02-08')
mkdir_p(cache_directory)
# ------------------------------------------------------------------------
print "Getting (incorrect) party names mapping for matching..."
parties_cache_filename = os.path.join(cache_directory, 'parties')
party_data = get_data_with_cache(parties_cache_filename, url('/party/'))
party_names_api = set(d['name'].strip().encode('utf-8') for d in party_data['parties'])
party_names_db = set(o.name.strip().encode('utf-8') for o in Organisation.objects.filter(kind__slug='party'))
with open(os.path.join(data_directory, 'party-names.csv'), 'w') as fp:
writer = csv.writer(fp)
for t in itertools.izip_longest(party_names_api, party_names_db):
writer.writerow(t)
# ------------------------------------------------------------------------
print "Getting (incorrect) ward names mapping for matching..."
ward_data = get_data(url('/ward/'))
wards_from_api = sorted(ward['name'].encode('utf-8') for ward in ward_data['region']['locations'])
wards_from_db = sorted(p.name.encode('utf-8') for p in Place.objects.filter(kind__slug='ward'))
with open(os.path.join(data_directory, 'wards-names.csv'), 'w') as fp:
writer = csv.writer(fp)
for t in itertools.izip_longest(wards_from_api, wards_from_db):
writer.writerow(t)
# ------------------------------------------------------------------------
headings = ['Same/Different',
'API Name',
'API Party',
'API Place',
'API Candidate Code',
'Mz Legal Name',
'Mz Other Names',
'Mz URL',
'Mz Parties Ever',
'Mz Aspirant Ever',
'Mz Politician Ever',
'Mz ID']
with open(os.path.join(sys.path[0], 'names-to-check.csv'), 'w') as fp:
writer = csv.DictWriter(fp, headings)
writer.writerow(dict((h, h) for h in headings))
for area_type in 'county', 'constituency', 'ward':
cache_filename = os.path.join(cache_directory, area_type)
area_type_data = get_data_with_cache(cache_filename, url('/%s/' % (area_type)))
areas = area_type_data['region']['locations']
for i, area in enumerate(areas):
# Get the candidates for that area:
code = area['code']
candidates_cache_filename = os.path.join(cache_directory, 'candidates-for-' + area_type + '-' + code)
candidate_data = get_data_with_cache(candidates_cache_filename, url('/candidate/', query_filter='%s=%s' % (area_type, code)))
for race in candidate_data['candidates']:
full_race_name = race['race']
race_type, place_name = parse_race_name(full_race_name)
place_kind, session, title = known_race_type_mapping[race_type]
candidates = race['candidates']
for candidate in candidates:
first_names = candidate['other_name'] or ''
surname = candidate['surname'] or ''
person = get_person_from_names(first_names, surname)
if person:
print "Got person match to:", person
row = {}
row['Same/Different'] = ''
row['API Name'] = first_names + ' ' + surname
party_data = candidate['party']
row['API Party'] = party_data['name'] if 'name' in party_data else ''
row['API Place'] = '%s (%s)' % (place_name, area_type)
row['API Candidate Code'] = candidate['code']
row['Mz Legal Name'] = person.legal_name
row['Mz Other Names'] = person.other_names
row['Mz URL'] = 'http://info.mzalendo.com' + person.get_absolute_url()
row['Mz Parties Ever'] = ', '.join(o.name for o in person.parties_ever())
for heading, positions in (('Mz Aspirant Ever', person.aspirant_positions_ever()),
('Mz Politician Ever', person.politician_positions_ever())):
row[heading] = ', '.join('%s at %s' % (p.title.name, p.place) for p in positions)
row['Mz ID'] = person.id
for key, value in row.items():
row[key] = unicode(value).encode('utf-8')
writer.writerow(row)
class Command(NoArgsCommand):
help = 'Update the database with aspirants from the IEBC website'
option_list = NoArgsCommand.option_list + (make_option(
'--commit',
action='store_true',
dest='commit',
help='Actually update the database'), )
def handle_noargs(self, **options):
api_key = hmac.new(settings.IEBC_API_SECRET,
"appid=%s" % (settings.IEBC_API_ID, ),
hashlib.sha256).hexdigest()
token_data = get_data(make_api_token_url(settings.IEBC_API_ID,
api_key))
token = token_data['token']
def url(path, query_filter=None):
"""A closure to avoid repeating parameters"""
return make_api_url(path, settings.IEBC_API_SECRET, token,
query_filter)
# To get all the candidates, we iterate over each county,
# constituency and ward, and request the candidates for each.
cache_directory = os.path.join(data_directory, 'api-cache-2013-02-08')
mkdir_p(cache_directory)
parties_cache_filename = os.path.join(cache_directory, 'parties')
party_data = get_data_with_cache(parties_cache_filename,
url('/party/'),
only_from_cache=True)
for party in party_data['parties']:
party_api_code = party['code']
try:
party_organisation = get_matching_party(party['name'],
create=False,
**options)
except CommandError, ce:
print >> sys.stderr, "Not setting the API code: %s" % (ce, )
party_organisation.external_id = party_api_code
maybe_save(party_organisation, **options)
print "########################################################################"
for area_type in 'county', 'constituency', 'ward':
cache_filename = os.path.join(cache_directory, area_type)
area_type_data = get_data_with_cache(cache_filename,
url('/%s/' % (area_type)),
only_from_cache=True)
areas = area_type_data['region']['locations']
for i, area in enumerate(areas):
# Get the candidates for that area:
code = area['code']
candidates_cache_filename = os.path.join(
cache_directory,
'candidates-for-' + area_type + '-' + code)
candidate_data = get_data_with_cache(candidates_cache_filename,
url('/candidate/',
query_filter='%s=%s' %
(area_type, code)),
only_from_cache=True)
# print "got candidate_data:", candidate_data
for race in candidate_data['candidates']:
full_race_name = race['race']
race_type, place_name = parse_race_name(full_race_name)
place_kind, session, title = known_race_type_mapping[
race_type]
place = get_matching_place(place_name, place_kind, session)
place.external_id = code
maybe_save(place, **options)
candidates = race['candidates']
for candidate in candidates:
first_names = candidate['other_name'] or ''
surname = candidate['surname'] or ''
person = get_person_from_names(first_names, surname)
if not person:
raise Exception, "Failed to find the person from '%s' '%s'" % (
first_names, surname)
aspirant_position_properties = {
'organisation':
Organisation.objects.get(name='REPUBLIC OF KENYA'),
'place':
place,
'person':
person,
'title':
title,
'category':
'political'
}
positions = Position.objects.filter(
**aspirant_position_properties).currently_active()
if positions:
if len(positions) > 1:
print >> sys.stderr, "There were multiple (%d) matches for: %s" % (
len(positions),
aspirant_position_properties,
)
for position in positions:
position.external_id = candidate['code']
maybe_save(position, **options)
else:
message = "There were no matches for %s" % (
aspirant_position_properties, )
# raise Exception, message
print >> sys.stderr, message
def handle_noargs(self, **options):
api_key = hmac.new(settings.IEBC_API_SECRET,
"appid=%s" % (settings.IEBC_API_ID,),
hashlib.sha256).hexdigest()
token_data = get_data(make_api_token_url(settings.IEBC_API_ID, api_key))
token = token_data['token']
def url(path, query_filter=None):
"""A closure to avoid repeating parameters"""
return make_api_url(path, settings.IEBC_API_SECRET, token, query_filter)
aspirants_to_remove = set(Position.objects.all().aspirant_positions().exclude(title__slug__iexact='aspirant-president').currently_active())
# To get all the candidates, we iterate over each county,
# constituency and ward, and request the candidates for each.
cache_directory = os.path.join(data_directory, 'api-cache-2013-03-01')
mkdir_p(cache_directory)
same_person_checker = SamePersonChecker(os.path.join(data_directory,
'names-manually-checked.csv'))
failed = False
for area_type in 'county', 'constituency', 'ward':
cache_filename = os.path.join(cache_directory, area_type + '.json')
area_type_data = get_data_with_cache(cache_filename, url('/%s/' % (area_type)))
areas = area_type_data['region']['locations']
area_name_to_codes = defaultdict(list)
# Unfortunately areas with the same name appear multiple
# times in these results:
for area in areas:
area_name_to_codes[area['name']].append(area)
for area_name, areas in area_name_to_codes.items():
all_candidates = defaultdict(list)
for area in areas:
place_code = area['code']
candidates_cache_filename = os.path.join(cache_directory, 'candidates-for-' + area_type + '-' + place_code + '.json')
candidate_data = get_data_with_cache(candidates_cache_filename, url('/candidate/', query_filter='%s=%s' % (area_type, place_code)))
races = candidate_data['candidates']
for race in races:
candidates = race['candidates']
if not candidates:
continue
contest_type = get_contest_type(candidates)
all_candidates[contest_type] += race['candidates']
for contest_type, candidates in all_candidates.items():
place_kind, session, title, race_type = known_race_type_mapping[contest_type]
succeeded = update_candidates_for_place(area_name,
place_kind,
session,
title,
race_type,
candidates,
same_person_checker,
**options)
if not succeeded:
failed = True
# Try to update the picture for each candidate:
for candidate in candidates:
update_picture_for_candidate(candidate, cache_directory, **options)
if failed:
print "Failed: you need to update", same_person_checker.csv_filename
def handle_noargs(self, **options):
api_key = hmac.new(settings.IEBC_API_SECRET,
"appid=%s" % (settings.IEBC_API_ID, ),
hashlib.sha256).hexdigest()
token_data = get_data(make_api_token_url(settings.IEBC_API_ID,
api_key))
token = token_data['token']
def url(path, query_filter=None):
"""A closure to avoid repeating parameters"""
return make_api_url(path, settings.IEBC_API_SECRET, token,
query_filter)
aspirants_to_remove = set(
Position.objects.all().aspirant_positions().exclude(
title__slug__iexact='aspirant-president').currently_active())
# To get all the candidates, we iterate over each county,
# constituency and ward, and request the candidates for each.
cache_directory = os.path.join(data_directory, 'api-cache-2013-03-01')
mkdir_p(cache_directory)
same_person_checker = SamePersonChecker(
os.path.join(data_directory, 'names-manually-checked.csv'))
failed = False
for area_type in 'county', 'constituency', 'ward':
cache_filename = os.path.join(cache_directory, area_type + '.json')
area_type_data = get_data_with_cache(cache_filename,
url('/%s/' % (area_type)))
areas = area_type_data['region']['locations']
area_name_to_codes = defaultdict(list)
# Unfortunately areas with the same name appear multiple
# times in these results:
for area in areas:
area_name_to_codes[area['name']].append(area)
for area_name, areas in area_name_to_codes.items():
all_candidates = defaultdict(list)
for area in areas:
place_code = area['code']
candidates_cache_filename = os.path.join(
cache_directory, 'candidates-for-' + area_type + '-' +
place_code + '.json')
candidate_data = get_data_with_cache(
candidates_cache_filename,
url('/candidate/',
query_filter='%s=%s' % (area_type, place_code)))
races = candidate_data['candidates']
for race in races:
candidates = race['candidates']
if not candidates:
continue
contest_type = get_contest_type(candidates)
all_candidates[contest_type] += race['candidates']
for contest_type, candidates in all_candidates.items():
place_kind, session, title, race_type = known_race_type_mapping[
contest_type]
succeeded = update_candidates_for_place(
area_name, place_kind, session, title, race_type,
candidates, same_person_checker, **options)
if not succeeded:
failed = True
# Try to update the picture for each candidate:
for candidate in candidates:
update_picture_for_candidate(candidate,
cache_directory,
**options)
if failed:
print "Failed: you need to update", same_person_checker.csv_filename