def handle_noargs(self, **options):


        api_key = hmac.new(settings.IEBC_API_SECRET,
                           "appid=%s" % (settings.IEBC_API_ID,),
                           hashlib.sha256).hexdigest()

        token_data = get_data(make_api_token_url(settings.IEBC_API_ID, api_key))
        token = token_data['token']

        def url(path, query_filter=None):
            """A closure to avoid repeating parameters"""
            return make_api_url(path, settings.IEBC_API_SECRET, token, query_filter)

        cache_directory = os.path.join(data_directory, 'api-cache-2013-02-14')

        mkdir_p(cache_directory)

        for area_type in 'county', 'constituency', 'ward':
            cache_filename = os.path.join(cache_directory, area_type + '.json')
            area_type_data = get_data_with_cache(cache_filename, url('/%s/' % (area_type)))
            areas = area_type_data['region']['locations']
            for i, area in enumerate(areas):
                # Get the candidates for that area:
                code = area['code']
                candidates_cache_filename = os.path.join(cache_directory, 'candidates-for-' + area_type + '-' + code + '.json')
                candidate_data = get_data_with_cache(candidates_cache_filename, url('/candidate/', query_filter='%s=%s' % (area_type, code)))
                for race in candidate_data['candidates']:
                    place_name = race['race']
                    candidates = race['candidates']
                    if len(candidates) == 0:
                        continue
                    distinct_contest_types = set(c['contest_type'] for c in race['candidates'])
                    if len(distinct_contest_types) != 1:
                        print "Multiple contest types found for %s: %s" % (place_name, distinct_contest_types)
                    contest_type = iter(distinct_contest_types).next()
                    place_kind, session, title, race_type = known_race_type_mapping[contest_type]

                    if options['place'] and options['place'].lower() != place_name.lower():
                        continue

                    remove_duplicate_candidates_for_place(place_name,
                                                          place_kind,
                                                          session,
                                                          title,
                                                          race_type,
                                                          candidates,
                                                          **options)
    def handle_noargs(self, **options):

        api_key = hmac.new(settings.IEBC_API_SECRET,
                           "appid=%s" % (settings.IEBC_API_ID,),
                           hashlib.sha256).hexdigest()

        token_data = get_data(make_api_token_url(settings.IEBC_API_ID, api_key))
        token = token_data['token']

        def url(path, query_filter=None):
            """A closure to avoid repeating parameters"""
            return make_api_url(path, settings.IEBC_API_SECRET, token, query_filter)

        # To get all the candidates, we iterate over each county,
        # constituency and ward, and request the candidates for each.

        cache_directory = os.path.join(data_directory, 'api-cache-2013-02-08')

        mkdir_p(cache_directory)

        parties_cache_filename = os.path.join(cache_directory, 'parties')
        party_data = get_data_with_cache(parties_cache_filename, url('/party/'), only_from_cache=True)
        for party in party_data['parties']:
            party_api_code = party['code']
            try:
                party_organisation = get_matching_party(party['name'], create=False, **options)
            except CommandError, ce:
                print >> sys.stderr, "Not setting the API code: %s" % (ce,)
            party_organisation.external_id = party_api_code
            maybe_save(party_organisation, **options)
Exemple #3
0
    def handle_noargs(self, **options):

        api_key = hmac.new(settings.IEBC_API_SECRET,
                           "appid=%s" % (settings.IEBC_API_ID, ),
                           hashlib.sha256).hexdigest()

        token_data = get_data(make_api_token_url(settings.IEBC_API_ID,
                                                 api_key))
        token = token_data['token']

        def url(path, query_filter=None):
            """A closure to avoid repeating parameters"""
            return make_api_url(path, settings.IEBC_API_SECRET, token,
                                query_filter)

        # To get all the candidates, we iterate over each county,
        # constituency and ward, and request the candidates for each.

        cache_directory = os.path.join(data_directory, 'api-cache-2013-02-08')

        mkdir_p(cache_directory)

        parties_cache_filename = os.path.join(cache_directory, 'parties')
        party_data = get_data_with_cache(parties_cache_filename,
                                         url('/party/'),
                                         only_from_cache=True)
        for party in party_data['parties']:
            party_api_code = party['code']
            try:
                party_organisation = get_matching_party(party['name'],
                                                        create=False,
                                                        **options)
            except CommandError, ce:
                print >> sys.stderr, "Not setting the API code: %s" % (ce, )
            party_organisation.external_id = party_api_code
            maybe_save(party_organisation, **options)
Exemple #4
0
    def handle_noargs(self, **options):

        api_key = hmac.new(settings.IEBC_API_SECRET,
                           "appid=%s" % (settings.IEBC_API_ID, ),
                           hashlib.sha256).hexdigest()

        token_data = get_data(make_api_token_url(settings.IEBC_API_ID,
                                                 api_key))
        token = token_data['token']

        def url(path, query_filter=None):
            """A closure to avoid repeating parameters"""
            return make_api_url(path, settings.IEBC_API_SECRET, token,
                                query_filter)

        # To get all the candidates, we iterate over each county,
        # constituency and ward, and request the candidates for each.

        cache_directory = os.path.join(data_directory, 'api-cache-2013-02-08')

        mkdir_p(cache_directory)

        # ------------------------------------------------------------------------

        print "Getting (incorrect) party names mapping for matching..."

        parties_cache_filename = os.path.join(cache_directory, 'parties')
        party_data = get_data_with_cache(parties_cache_filename,
                                         url('/party/'))

        party_names_api = set(d['name'].strip().encode('utf-8')
                              for d in party_data['parties'])
        party_names_db = set(
            o.name.strip().encode('utf-8')
            for o in Organisation.objects.filter(kind__slug='party'))

        with open(os.path.join(data_directory, 'party-names.csv'), 'w') as fp:
            writer = csv.writer(fp)
            for t in itertools.izip_longest(party_names_api, party_names_db):
                writer.writerow(t)

        # ------------------------------------------------------------------------

        print "Getting (incorrect) ward names mapping for matching..."

        ward_data = get_data(url('/ward/'))

        wards_from_api = sorted(ward['name'].encode('utf-8')
                                for ward in ward_data['region']['locations'])
        wards_from_db = sorted(
            p.name.encode('utf-8')
            for p in Place.objects.filter(kind__slug='ward'))

        with open(os.path.join(data_directory, 'wards-names.csv'), 'w') as fp:
            writer = csv.writer(fp)
            for t in itertools.izip_longest(wards_from_api, wards_from_db):
                writer.writerow(t)

        # ------------------------------------------------------------------------

        headings = [
            'Same/Different', 'API Name', 'API Party', 'API Place',
            'API Candidate Code', 'Mz Legal Name', 'Mz Other Names', 'Mz URL',
            'Mz Parties Ever', 'Mz Aspirant Ever', 'Mz Politician Ever',
            'Mz ID'
        ]

        with open(os.path.join(sys.path[0], 'names-to-check.csv'), 'w') as fp:

            writer = csv.DictWriter(fp, headings)

            writer.writerow(dict((h, h) for h in headings))

            for area_type in 'county', 'constituency', 'ward':
                cache_filename = os.path.join(cache_directory, area_type)
                area_type_data = get_data_with_cache(cache_filename,
                                                     url('/%s/' % (area_type)))
                areas = area_type_data['region']['locations']
                for i, area in enumerate(areas):
                    # Get the candidates for that area:
                    code = area['code']
                    candidates_cache_filename = os.path.join(
                        cache_directory,
                        'candidates-for-' + area_type + '-' + code)
                    candidate_data = get_data_with_cache(
                        candidates_cache_filename,
                        url('/candidate/',
                            query_filter='%s=%s' % (area_type, code)))
                    for race in candidate_data['candidates']:
                        full_race_name = race['race']
                        race_type, place_name = parse_race_name(full_race_name)
                        place_kind, session, title = known_race_type_mapping[
                            race_type]
                        candidates = race['candidates']
                        for candidate in candidates:
                            first_names = candidate['other_name'] or ''
                            surname = candidate['surname'] or ''
                            person = get_person_from_names(
                                first_names, surname)
                            if person:
                                print "Got person match to:", person
                                row = {}
                                row['Same/Different'] = ''
                                row['API Name'] = first_names + ' ' + surname
                                party_data = candidate['party']
                                row['API Party'] = party_data[
                                    'name'] if 'name' in party_data else ''
                                row['API Place'] = '%s (%s)' % (place_name,
                                                                area_type)
                                row['API Candidate Code'] = candidate['code']
                                row['Mz Legal Name'] = person.legal_name
                                row['Mz Other Names'] = person.other_names
                                row['Mz URL'] = 'http://info.mzalendo.com' + person.get_absolute_url(
                                )
                                row['Mz Parties Ever'] = ', '.join(
                                    o.name for o in person.parties_ever())
                                for heading, positions in (
                                    ('Mz Aspirant Ever',
                                     person.aspirant_positions_ever()),
                                    ('Mz Politician Ever',
                                     person.politician_positions_ever())):
                                    row[heading] = ', '.join(
                                        '%s at %s' % (p.title.name, p.place)
                                        for p in positions)
                                row['Mz ID'] = person.id
                                for key, value in row.items():
                                    row[key] = unicode(value).encode('utf-8')
                                writer.writerow(row)
    def handle_noargs(self, **options):

        api_key = hmac.new(settings.IEBC_API_SECRET,
                           "appid=%s" % (settings.IEBC_API_ID,),
                           hashlib.sha256).hexdigest()

        token_data = get_data(make_api_token_url(settings.IEBC_API_ID, api_key))
        token = token_data['token']

        def url(path, query_filter=None):
            """A closure to avoid repeating parameters"""
            return make_api_url(path, settings.IEBC_API_SECRET, token, query_filter)

        # To get all the candidates, we iterate over each county,
        # constituency and ward, and request the candidates for each.

        cache_directory = os.path.join(data_directory, 'api-cache-2013-02-08')

        mkdir_p(cache_directory)

        # ------------------------------------------------------------------------

        print "Getting (incorrect) party names mapping for matching..."

        parties_cache_filename = os.path.join(cache_directory, 'parties')
        party_data = get_data_with_cache(parties_cache_filename, url('/party/'))

        party_names_api = set(d['name'].strip().encode('utf-8') for d in party_data['parties'])
        party_names_db = set(o.name.strip().encode('utf-8') for o in Organisation.objects.filter(kind__slug='party'))

        with open(os.path.join(data_directory, 'party-names.csv'), 'w') as fp:
            writer = csv.writer(fp)
            for t in itertools.izip_longest(party_names_api, party_names_db):
                writer.writerow(t)

        # ------------------------------------------------------------------------

        print "Getting (incorrect) ward names mapping for matching..."

        ward_data = get_data(url('/ward/'))

        wards_from_api = sorted(ward['name'].encode('utf-8') for ward in ward_data['region']['locations'])
        wards_from_db = sorted(p.name.encode('utf-8') for p in Place.objects.filter(kind__slug='ward'))

        with open(os.path.join(data_directory, 'wards-names.csv'), 'w') as fp:
            writer = csv.writer(fp)
            for t in itertools.izip_longest(wards_from_api, wards_from_db):
                writer.writerow(t)

        # ------------------------------------------------------------------------

        headings = ['Same/Different',
                    'API Name',
                    'API Party',
                    'API Place',
                    'API Candidate Code',
                    'Mz Legal Name',
                    'Mz Other Names',
                    'Mz URL',
                    'Mz Parties Ever',
                    'Mz Aspirant Ever',
                    'Mz Politician Ever',
                    'Mz ID']

        with open(os.path.join(sys.path[0], 'names-to-check.csv'), 'w') as fp:

            writer = csv.DictWriter(fp, headings)

            writer.writerow(dict((h, h) for h in headings))

            for area_type in 'county', 'constituency', 'ward':
                cache_filename = os.path.join(cache_directory, area_type)
                area_type_data = get_data_with_cache(cache_filename, url('/%s/' % (area_type)))
                areas = area_type_data['region']['locations']
                for i, area in enumerate(areas):
                    # Get the candidates for that area:
                    code = area['code']
                    candidates_cache_filename = os.path.join(cache_directory, 'candidates-for-' + area_type + '-' + code)
                    candidate_data = get_data_with_cache(candidates_cache_filename, url('/candidate/', query_filter='%s=%s' % (area_type, code)))
                    for race in candidate_data['candidates']:
                        full_race_name = race['race']
                        race_type, place_name = parse_race_name(full_race_name)
                        place_kind, session, title = known_race_type_mapping[race_type]
                        candidates = race['candidates']
                        for candidate in candidates:
                            first_names = candidate['other_name'] or ''
                            surname = candidate['surname'] or ''
                            person = get_person_from_names(first_names, surname)
                            if person:
                                print "Got person match to:", person
                                row = {}
                                row['Same/Different'] = ''
                                row['API Name'] = first_names + ' ' + surname
                                party_data = candidate['party']
                                row['API Party'] = party_data['name'] if 'name' in party_data else ''
                                row['API Place'] = '%s (%s)' % (place_name, area_type)
                                row['API Candidate Code'] = candidate['code']
                                row['Mz Legal Name'] = person.legal_name
                                row['Mz Other Names'] = person.other_names
                                row['Mz URL'] = 'http://info.mzalendo.com' + person.get_absolute_url()
                                row['Mz Parties Ever'] = ', '.join(o.name for o in person.parties_ever())
                                for heading, positions in (('Mz Aspirant Ever', person.aspirant_positions_ever()),
                                                           ('Mz Politician Ever', person.politician_positions_ever())):
                                    row[heading] = ', '.join('%s at %s' % (p.title.name, p.place) for p in positions)
                                row['Mz ID'] = person.id
                                for key, value in row.items():
                                    row[key] = unicode(value).encode('utf-8')
                                writer.writerow(row)
Exemple #6
0
class Command(NoArgsCommand):
    help = 'Update the database with aspirants from the IEBC website'

    option_list = NoArgsCommand.option_list + (make_option(
        '--commit',
        action='store_true',
        dest='commit',
        help='Actually update the database'), )

    def handle_noargs(self, **options):

        api_key = hmac.new(settings.IEBC_API_SECRET,
                           "appid=%s" % (settings.IEBC_API_ID, ),
                           hashlib.sha256).hexdigest()

        token_data = get_data(make_api_token_url(settings.IEBC_API_ID,
                                                 api_key))
        token = token_data['token']

        def url(path, query_filter=None):
            """A closure to avoid repeating parameters"""
            return make_api_url(path, settings.IEBC_API_SECRET, token,
                                query_filter)

        # To get all the candidates, we iterate over each county,
        # constituency and ward, and request the candidates for each.

        cache_directory = os.path.join(data_directory, 'api-cache-2013-02-08')

        mkdir_p(cache_directory)

        parties_cache_filename = os.path.join(cache_directory, 'parties')
        party_data = get_data_with_cache(parties_cache_filename,
                                         url('/party/'),
                                         only_from_cache=True)
        for party in party_data['parties']:
            party_api_code = party['code']
            try:
                party_organisation = get_matching_party(party['name'],
                                                        create=False,
                                                        **options)
            except CommandError, ce:
                print >> sys.stderr, "Not setting the API code: %s" % (ce, )
            party_organisation.external_id = party_api_code
            maybe_save(party_organisation, **options)

        print "########################################################################"

        for area_type in 'county', 'constituency', 'ward':
            cache_filename = os.path.join(cache_directory, area_type)
            area_type_data = get_data_with_cache(cache_filename,
                                                 url('/%s/' % (area_type)),
                                                 only_from_cache=True)
            areas = area_type_data['region']['locations']
            for i, area in enumerate(areas):
                # Get the candidates for that area:
                code = area['code']
                candidates_cache_filename = os.path.join(
                    cache_directory,
                    'candidates-for-' + area_type + '-' + code)
                candidate_data = get_data_with_cache(candidates_cache_filename,
                                                     url('/candidate/',
                                                         query_filter='%s=%s' %
                                                         (area_type, code)),
                                                     only_from_cache=True)
                # print "got candidate_data:", candidate_data
                for race in candidate_data['candidates']:
                    full_race_name = race['race']
                    race_type, place_name = parse_race_name(full_race_name)
                    place_kind, session, title = known_race_type_mapping[
                        race_type]
                    place = get_matching_place(place_name, place_kind, session)
                    place.external_id = code
                    maybe_save(place, **options)
                    candidates = race['candidates']
                    for candidate in candidates:
                        first_names = candidate['other_name'] or ''
                        surname = candidate['surname'] or ''
                        person = get_person_from_names(first_names, surname)
                        if not person:
                            raise Exception, "Failed to find the person from '%s' '%s'" % (
                                first_names, surname)
                        aspirant_position_properties = {
                            'organisation':
                            Organisation.objects.get(name='REPUBLIC OF KENYA'),
                            'place':
                            place,
                            'person':
                            person,
                            'title':
                            title,
                            'category':
                            'political'
                        }
                        positions = Position.objects.filter(
                            **aspirant_position_properties).currently_active()
                        if positions:
                            if len(positions) > 1:
                                print >> sys.stderr, "There were multiple (%d) matches for: %s" % (
                                    len(positions),
                                    aspirant_position_properties,
                                )
                            for position in positions:
                                position.external_id = candidate['code']
                                maybe_save(position, **options)
                        else:
                            message = "There were no matches for %s" % (
                                aspirant_position_properties, )
                            # raise Exception, message
                            print >> sys.stderr, message
    def handle_noargs(self, **options):

        api_key = hmac.new(settings.IEBC_API_SECRET,
                           "appid=%s" % (settings.IEBC_API_ID,),
                           hashlib.sha256).hexdigest()

        token_data = get_data(make_api_token_url(settings.IEBC_API_ID, api_key))
        token = token_data['token']

        def url(path, query_filter=None):
            """A closure to avoid repeating parameters"""
            return make_api_url(path, settings.IEBC_API_SECRET, token, query_filter)

        aspirants_to_remove = set(Position.objects.all().aspirant_positions().exclude(title__slug__iexact='aspirant-president').currently_active())

        # To get all the candidates, we iterate over each county,
        # constituency and ward, and request the candidates for each.

        cache_directory = os.path.join(data_directory, 'api-cache-2013-03-01')

        mkdir_p(cache_directory)

        same_person_checker = SamePersonChecker(os.path.join(data_directory,
                                                             'names-manually-checked.csv'))

        failed = False

        for area_type in 'county', 'constituency', 'ward':

            cache_filename = os.path.join(cache_directory, area_type + '.json')
            area_type_data = get_data_with_cache(cache_filename, url('/%s/' % (area_type)))
            areas = area_type_data['region']['locations']
            area_name_to_codes = defaultdict(list)
            # Unfortunately areas with the same name appear multiple
            # times in these results:
            for area in areas:
                area_name_to_codes[area['name']].append(area)

            for area_name, areas in area_name_to_codes.items():
                all_candidates = defaultdict(list)
                for area in areas:
                    place_code = area['code']
                    candidates_cache_filename = os.path.join(cache_directory, 'candidates-for-' + area_type + '-' + place_code + '.json')
                    candidate_data = get_data_with_cache(candidates_cache_filename, url('/candidate/', query_filter='%s=%s' % (area_type, place_code)))
                    races = candidate_data['candidates']
                    for race in races:
                        candidates = race['candidates']
                        if not candidates:
                            continue
                        contest_type = get_contest_type(candidates)
                        all_candidates[contest_type] += race['candidates']
                for contest_type, candidates in all_candidates.items():
                    place_kind, session, title, race_type = known_race_type_mapping[contest_type]
                    succeeded = update_candidates_for_place(area_name,
                                                            place_kind,
                                                            session,
                                                            title,
                                                            race_type,
                                                            candidates,
                                                            same_person_checker,
                                                            **options)
                    if not succeeded:
                        failed = True

                    # Try to update the picture for each candidate:
                    for candidate in candidates:
                        update_picture_for_candidate(candidate, cache_directory, **options)

        if failed:
            print "Failed: you need to update", same_person_checker.csv_filename
    def handle_noargs(self, **options):

        api_key = hmac.new(settings.IEBC_API_SECRET,
                           "appid=%s" % (settings.IEBC_API_ID, ),
                           hashlib.sha256).hexdigest()

        token_data = get_data(make_api_token_url(settings.IEBC_API_ID,
                                                 api_key))
        token = token_data['token']

        def url(path, query_filter=None):
            """A closure to avoid repeating parameters"""
            return make_api_url(path, settings.IEBC_API_SECRET, token,
                                query_filter)

        aspirants_to_remove = set(
            Position.objects.all().aspirant_positions().exclude(
                title__slug__iexact='aspirant-president').currently_active())

        # To get all the candidates, we iterate over each county,
        # constituency and ward, and request the candidates for each.

        cache_directory = os.path.join(data_directory, 'api-cache-2013-03-01')

        mkdir_p(cache_directory)

        same_person_checker = SamePersonChecker(
            os.path.join(data_directory, 'names-manually-checked.csv'))

        failed = False

        for area_type in 'county', 'constituency', 'ward':

            cache_filename = os.path.join(cache_directory, area_type + '.json')
            area_type_data = get_data_with_cache(cache_filename,
                                                 url('/%s/' % (area_type)))
            areas = area_type_data['region']['locations']
            area_name_to_codes = defaultdict(list)
            # Unfortunately areas with the same name appear multiple
            # times in these results:
            for area in areas:
                area_name_to_codes[area['name']].append(area)

            for area_name, areas in area_name_to_codes.items():
                all_candidates = defaultdict(list)
                for area in areas:
                    place_code = area['code']
                    candidates_cache_filename = os.path.join(
                        cache_directory, 'candidates-for-' + area_type + '-' +
                        place_code + '.json')
                    candidate_data = get_data_with_cache(
                        candidates_cache_filename,
                        url('/candidate/',
                            query_filter='%s=%s' % (area_type, place_code)))
                    races = candidate_data['candidates']
                    for race in races:
                        candidates = race['candidates']
                        if not candidates:
                            continue
                        contest_type = get_contest_type(candidates)
                        all_candidates[contest_type] += race['candidates']
                for contest_type, candidates in all_candidates.items():
                    place_kind, session, title, race_type = known_race_type_mapping[
                        contest_type]
                    succeeded = update_candidates_for_place(
                        area_name, place_kind, session, title, race_type,
                        candidates, same_person_checker, **options)
                    if not succeeded:
                        failed = True

                    # Try to update the picture for each candidate:
                    for candidate in candidates:
                        update_picture_for_candidate(candidate,
                                                     cache_directory,
                                                     **options)

        if failed:
            print "Failed: you need to update", same_person_checker.csv_filename