def handle(self, *args, **options):
        self.verbosity = int(options.get('verbosity', 1))
        popit = create_popit_api_object()
        if len(args) != 3:
            raise CommandError("You must provide all three arguments")

        person_id, scheme, identifier = args

        try:
            person_data = popit.persons(person_id).get()['result']
        except (HttpClientError, HttpServerError):
            message = "Failed to get the person with ID {0}"
            raise CommandError(message.format(person_id))

        person_data['identifiers'].append(
            {
                'scheme': scheme,
                'identifier': identifier,
            }
        )

        try:
            popit.persons(person_id).put(person_data)
        except (HttpClientError, HttpServerError):
            raise CommandError("Failed to update that person")

        invalidate_cache_entries_from_person_data(person_data)

        # FIXME: this should create a new version in the versions
        # array too, otherwise you manually have to edit on YourNextMP
        # too to create a new version with a change message.

        print "Successfully updated {0}".format(person_id)
    def handle(self, *args, **options):
        self.verbosity = int(options.get('verbosity', 1))
        api = create_popit_api_object()
        if len(args) != 2:
            raise CommandError("You must provide all two arguments")

        person_id, other_name = args

        person = PopItPerson.create_from_popit(api, person_id)

        person.other_names.append(
            {
                'name': other_name,
                'note': options['note'],
                'start_date': options['start_date'],
                'end_date': options['end_date']
            }
        )

        person.save_to_popit(api)
        person.invalidate_cache_entries()

        # FIXME: this should create a new version in the versions
        # array too, otherwise you manually have to edit on YourNextMP
        # too to create a new version with a change message.

        print "Successfully updated {0}".format(person_id)
    def handle(self, *args, **options):
        self.verbosity = int(options.get('verbosity', 1))
        popit = create_popit_api_object()
        if len(args) != 2:
            raise CommandError("You must provide all two arguments")

        person_id, other_name = args

        try:
            person_data = popit.persons(person_id).get()['result']
        except (HttpClientError, HttpServerError):
            message = "Failed to get the person with ID {0}"
            raise CommandError(message.format(person_id))

        person_data['other_names'].append(
            {
                'name': other_name,
                'note': options['note'],
                'start_date': options['start_date'],
                'end_date': options['end_date']
            }
        )

        try:
            popit.persons(person_id).put(person_data)
        except (HttpClientError, HttpServerError):
            raise CommandError("Failed to update that person")

        # FIXME: this should create a new version in the versions
        # array too, otherwise you manually have to edit on YourNextMP
        # too to create a new version with a change message.

        print "Successfully updated {0}".format(person_id)
Example #4
0
 def handle(self, **options):
     joint_party_to_sub_parties = defaultdict(list)
     api = create_popit_api_object()
     for party in popit_unwrap_pagination(api.organizations,
                                          embed='',
                                          per_page=200):
         if party['classification'] != 'Party':
             continue
         if 'dissolution_date' in party:
             dissolution_date = party['dissolution_date']
             if dissolution_date < str(date.today()):
                 continue
         for d in party.get('descriptions', []):
             m = joint_description_re.search(d['description'])
             if not m:
                 continue
             joint_name = fix_joint_party_name(m.group('joint_name'))
             joint_party_to_sub_parties[joint_name].append(party)
     for joint_name, sub_parties in joint_party_to_sub_parties.items():
         if len(sub_parties) < 2:
             message = 'Ignoring "{joint_name}" (only made up of one party: "{sub_party}")'
             print message.format(joint_name=joint_name,
                                  sub_party=sub_parties[0]['name'])
             continue
         create_or_update_party(api, joint_name, sub_parties)
Example #5
0
def upload_photo(request, popit_person_id):
    if request.method == 'POST':
        form = UploadPersonPhotoForm(request.POST, request.FILES)
        if form.is_valid():
            # Make sure that we save the user that made the upload
            queued_image = form.save(commit=False)
            queued_image.user = request.user
            queued_image.save()
            # Record that action:
            LoggedAction.objects.create(
                user=request.user,
                action_type='photo-upload',
                ip_address=get_client_ip(request),
                popit_person_new_version='',
                popit_person_id=popit_person_id,
                source=form.cleaned_data['justification_for_use'],
            )
            return HttpResponseRedirect(
                reverse('photo-upload-success',
                        kwargs={
                            'popit_person_id':
                            form.cleaned_data['popit_person_id']
                        }))
    else:
        form = UploadPersonPhotoForm(
            initial={'popit_person_id': popit_person_id})
    api = create_popit_api_object()
    return render(
        request, 'moderation_queue/photo-upload-new.html', {
            'form': form,
            'person': PopItPerson.create_from_popit(api, popit_person_id)
        })
 def handle_label(self, output_filename, **options):
     api = create_popit_api_object()
     ntf = NamedTemporaryFile(
         delete=False,
         dir=dirname(output_filename)
     )
     all_parties = []
     for party in popit_unwrap_pagination(
             api.organizations,
             embed='',
             per_page=200
     ):
         if party['classification'] != 'Party':
             continue
         # These URLs are specific to the server this data is
         # extracted from:
         del party['url']
         del party['html_url']
         # Similarly the URLs of images:
         party.pop('image', None)
         party.pop('proxy_image', None)
         party.pop('images', None)
         # The generated id in each identifier will be different
         # between systems (or between runs on the same system) so
         # just produces noisy diffs if we include it.
         for identifier in party.get('identifiers', []):
             identifier.pop('id', None)
         all_parties.append(party)
     all_parties.sort(key=lambda p: p['id'])
     # Output to a string so that we can strip any trainling whitespace.
     output = json.dumps(all_parties, sort_keys=True, indent=4)
     output = re.sub(r'(?ms)\s*$', '', output)
     ntf.write(output)
     rename(ntf.name, output_filename)
 def handle(self, **options):
     self.mime_type_magic = magic.Magic(mime=True)
     self.api = create_popit_api_object()
     start = 0
     per_page = 50
     url = 'http://pefonline.electoralcommission.org.uk/api/search/Registrations'
     params = {
         'rows': per_page,
         'et': ["pp", "ppm"],
         'register': ["gb", "ni"],
         'regStatus': ["registered", "deregistered", "lapsed"],
     }
     total = None
     while total is None or start <= total:
         ntf = NamedTemporaryFile(delete=False)
         params['start'] = start
         try:
             resp = requests.get(url + '?' +
                                 urlencode(params, doseq=True)).json()
             if total is None:
                 total = resp['Total']
             with open(ntf.name, 'w') as f:
                 json.dump(resp['Result'], f)
             self.parse_data(ntf.name)
         finally:
             os.remove(ntf.name)
         start += per_page
    def handle(self, *args, **options):
        self.verbosity = int(options.get('verbosity', 1))
        api = create_popit_api_object()
        if len(args) != 2:
            raise CommandError("You must provide all two arguments")

        person_id, other_name = args

        person = PopItPerson.create_from_popit(api, person_id)

        person.other_names.append(
            {
                'name': other_name,
                'note': options['note'],
                'start_date': options['start_date'],
                'end_date': options['end_date']
            }
        )

        person.save_to_popit(api)
        person.invalidate_cache_entries()

        # FIXME: this should create a new version in the versions
        # array too, otherwise you manually have to edit on the
        # YourNextRepresentative site too to create a new version with
        # a change message.

        print "Successfully updated {0}".format(person_id)
    def handle(self, *args, **options):
        self.verbosity = int(options.get('verbosity', 1))
        api = create_popit_api_object()
        if len(args) != 3:
            raise CommandError("You must provide all three arguments")

        person_id, scheme, identifier = args

        person = PopItPerson.create_from_popit(api, person_id)

        person.identifiers.append(
            {
                'scheme': scheme,
                'identifier': identifier,
            }
        )

        person.save_to_popit(api)
        person.invalidate_cache_entries()

        # FIXME: this should create a new version in the versions
        # array too, otherwise you manually have to edit on YourNextMP
        # too to create a new version with a change message.

        print "Successfully updated {0}".format(person_id)
 def handle(self, **options):
     self.mime_type_magic = magic.Magic(mime=True)
     self.api = create_popit_api_object()
     start = 0
     per_page = 50
     url = 'http://pefonline.electoralcommission.org.uk/api/search/Registrations'
     params = {
         'rows': per_page,
         'et': ["pp", "ppm"],
         'register': ["gb", "ni"],
         'regStatus': ["registered", "deregistered", "lapsed"],
     }
     total = None
     while total is None or start <= total:
         ntf = NamedTemporaryFile(delete=False)
         params['start'] = start
         try:
             resp = requests.get(url + '?' + urlencode(params, doseq=True)).json()
             if total is None:
                 total = resp['Total']
             with open(ntf.name, 'w') as f:
                 json.dump(resp['Result'], f)
             self.parse_data(ntf.name)
         finally:
             os.remove(ntf.name)
         start += per_page
 def handle_label(self, output_filename, **options):
     api = create_popit_api_object()
     ntf = NamedTemporaryFile(delete=False, dir=dirname(output_filename))
     all_parties = []
     for party in popit_unwrap_pagination(api.organizations,
                                          embed='',
                                          per_page=200):
         if party['classification'] != 'Party':
             continue
         if 'dissolution_date' in party:
             dissolution_date = party['dissolution_date']
             if dissolution_date < str(date.today()):
                 continue
         # These URLs are specific to the server this data is
         # extracted from:
         del party['url']
         del party['html_url']
         # Similarly the URLs of images:
         party.pop('image', None)
         party.pop('proxy_image', None)
         party.pop('images', None)
         # The generated id in each identifier will be different
         # between systems (or between runs on the same system) so
         # just produces noisy diffs if we include it.
         for identifier in party.get('identifiers', []):
             identifier.pop('id', None)
         all_parties.append(party)
     all_parties.sort(key=lambda p: p['id'])
     # Output to a string so that we can strip any trainling whitespace.
     output = json.dumps(all_parties, sort_keys=True, indent=4)
     output = re.sub(r'(?ms)\s*$', '', output)
     ntf.write(output)
     rename(ntf.name, output_filename)
 def handle(self, **options):
     joint_party_to_sub_parties = defaultdict(list)
     api = create_popit_api_object()
     for party in popit_unwrap_pagination(
             api.organizations,
             embed='',
             per_page=200
     ):
         if party['classification'] != 'Party':
             continue
         if 'dissolution_date' in party:
             dissolution_date = party['dissolution_date']
             if dissolution_date < str(date.today()):
                 continue
         for d in party.get('descriptions', []):
             m = joint_description_re.search(d['description'])
             if not m:
                 continue
             joint_name = fix_joint_party_name(m.group('joint_name'))
             joint_party_to_sub_parties[joint_name].append(party)
     for joint_name, sub_parties in joint_party_to_sub_parties.items():
         if len(sub_parties) < 2:
             message = 'Ignoring "{joint_name}" (only made up of one party: "{sub_party}")'
             print message.format(
                 joint_name=joint_name,
                 sub_party=sub_parties[0]['name']
             )
             continue
         create_or_update_party(api, joint_name, sub_parties)
 def handle(self, **options):
     self.api = create_popit_api_object()
     ntf = NamedTemporaryFile(delete=False)
     try:
         r = requests.get('http://openelectoralcommission.org.uk/parties/index.json')
         with open(ntf.name, 'w') as f:
             json.dump(r.json(), f)
         self.parse_data(ntf.name)
     finally:
         os.remove(ntf.name)
Example #14
0
 def handle(self, *args, **kwargs):
     if len(args) < 1:
         raise CommandError("You must provide one or more PopIt person ID")
     for person_id in args:
         person = PopItPerson.create_from_popit(
             create_popit_api_object(), person_id
         )
         person.delete_memberships()
         self.create_party_memberships(person_id, person.popit_data)
         self.create_candidate_list_memberships(person_id, person.popit_data)
Example #15
0
 def handle(self, **options):
     self.api = create_popit_api_object()
     ntf = NamedTemporaryFile(delete=False)
     try:
         r = requests.get('http://openelectoralcommission.org.uk/parties/index.json')
         with open(ntf.name, 'w') as f:
             json.dump(r.json(), f)
         self.parse_data(ntf.name)
     finally:
         os.remove(ntf.name)
Example #16
0
 def handle(self, *args, **options):
     self.verbosity = int(options.get('verbosity', 1))
     self.popit = create_popit_api_object()
     with open(CONSTITUENCIES_JSON_FILE) as f:
         constituencies = json.load(f).values()
     xml = objectify.parse(ALL_MEMBERS_XML_FILE).getroot()
     self.members = [member.attrib for member in xml.findall('member')]
     people_xml = objectify.parse(PEOPLE_XML_FILE).getroot()
     people = people_xml.findall('person')
     self.id_mapping = {p.get('latestname'): p.get('id') for p in people}
     self.update_candidates(constituencies)
    def handle(self, *args, **options):
        from candidates.models import PopItPerson
        from candidates.popit import create_popit_api_object

        self.verbosity = int(options.get('verbosity', 1))
        api = create_popit_api_object()
        if len(args) != 1:
            raise CommandError("You must provide a person.js URL")
        person_js_url = args[0]
        people_data = requests.get(person_js_url).json()
        for person_data in people_data['persons']:
            twfy_person = PopItPerson.create_from_dict(person_data)
            ynmp_id = twfy_person.get_identifier('yournextmp')
            if not ynmp_id:
                continue
            parlparse_id = twfy_person.id
            ynmp_person = PopItPerson.create_from_popit(api, ynmp_id)
            existing_parlparse_id = ynmp_person.get_identifier('uk.org.publicwhip')
            if existing_parlparse_id:
                if existing_parlparse_id == parlparse_id:
                    # That's fine, there's already the right parlparse ID
                    pass
                else:
                    # Otherwise there's a mismatch, which needs investigation
                    msg = "Warning: parlparse ID mismatch between YNMP {0} "
                    msg += "and TWFY {1} for YNMP person {2}\n"
                    self.stderr.write(
                        msg.format(
                            existing_parlparse_id,
                            parlparse_id,
                            ynmp_id,
                        )
                    )
                continue
            msg = "Updating the YourNextMP person {0} with parlparse_id {1}\n"
            self.stdout.write(msg.format(ynmp_id, parlparse_id))
            ynmp_person.set_identifier(
                'uk.org.publicwhip',
                parlparse_id,
            )
            change_metadata = get_change_metadata(
                None, "Fetched a new parlparse ID"
            )
            ynmp_person.record_version(change_metadata)
            ynmp_person.save_to_popit(api)
            ynmp_person.invalidate_cache_entries()
    def handle(self, *args, **options):
        from candidates.models import PopItPerson
        from candidates.popit import create_popit_api_object

        self.verbosity = int(options.get('verbosity', 1))
        api = create_popit_api_object()
        if len(args) != 1:
            raise CommandError("You must provide a person.js URL")
        person_js_url = args[0]
        people_data = requests.get(person_js_url).json()
        for person_data in people_data['persons']:
            twfy_person = PopItPerson.create_from_dict(person_data)
            ynmp_id = twfy_person.get_identifier('yournextmp')
            if not ynmp_id:
                continue
            parlparse_id = twfy_person.id
            ynmp_person = PopItPerson.create_from_popit(api, ynmp_id)
            existing_parlparse_id = ynmp_person.get_identifier(
                'uk.org.publicwhip')
            if existing_parlparse_id:
                if existing_parlparse_id == parlparse_id:
                    # That's fine, there's already the right parlparse ID
                    pass
                else:
                    # Otherwise there's a mismatch, which needs investigation
                    msg = "Warning: parlparse ID mismatch between YNMP {0} "
                    msg += "and TWFY {1} for YNMP person {2}\n"
                    self.stderr.write(
                        msg.format(
                            existing_parlparse_id,
                            parlparse_id,
                            ynmp_id,
                        ))
                continue
            msg = "Updating the YourNextMP person {0} with parlparse_id {1}\n"
            self.stdout.write(msg.format(ynmp_id, parlparse_id))
            ynmp_person.set_identifier(
                'uk.org.publicwhip',
                parlparse_id,
            )
            change_metadata = get_change_metadata(
                None, "Fetched a new parlparse ID")
            ynmp_person.record_version(change_metadata)
            ynmp_person.save_to_popit(api)
            ynmp_person.invalidate_cache_entries()
Example #19
0
 def handle(self, *args, **kwargs):
     api = create_popit_api_object()
     if len(args) < 1:
         raise CommandError("You must provide one or more PopIt person ID")
     for person_id in args:
         invalidate_person(person_id)
         person = PopItPerson.create_from_popit(api, person_id)
         posts_to_invalidate = person.get_associated_posts()
         person.delete_memberships(api)
         # The memberships are recreated when you assign to
         # standing_in and party_memberships; this script assumes
         # these are correct and so re-setting these should
         # recreate the memberships correctly.
         person.standing_in = person.standing_in
         person.party_memberships = person.party_memberships
         person.save_to_popit(api)
         invalidate_posts(posts_to_invalidate)
         invalidate_person(person_id)
Example #20
0
    def handle(self, *args, **options):
        api = create_popit_api_object()

        for org in popit_unwrap_pagination(
            api.organizations,
            per_page=100
        ):
            org.pop('versions', None)
            org.pop('memberships', None)
            images = org.get('images', [])
            if len(images) < 2:
                continue
            print "====================================================="
            print len(images), org['id'], org['name'].encode('utf-8')
            for image in images:
                print '  --'
                print '   ' + image['notes'].encode('utf-8')
                print '   ' + image['url']
 def handle(self, *args, **kwargs):
     api = create_popit_api_object()
     if len(args) < 1:
         raise CommandError("You must provide one or more PopIt person ID")
     for person_id in args:
         invalidate_person(person_id)
         person = PopItPerson.create_from_popit(api, person_id)
         posts_to_invalidate = person.get_associated_posts()
         person.delete_memberships(api)
         # The memberships are recreated when you assign to
         # standing_in and party_memberships; this script assumes
         # these are correct and so re-setting these should
         # recreate the memberships correctly.
         person.standing_in = person.standing_in
         person.party_memberships = person.party_memberships
         person.save_to_popit(api)
         invalidate_posts(posts_to_invalidate)
         invalidate_person(person_id)
def upload_photo(request, popit_person_id):
    if request.method == 'POST':
        form = UploadPersonPhotoForm(request.POST, request.FILES)
        if form.is_valid():
            # Make sure that we save the user that made the upload
            queued_image = form.save(commit=False)
            queued_image.user = request.user
            queued_image.save()
            # Record that action:
            LoggedAction.objects.create(
                user=request.user,
                action_type='photo-upload',
                ip_address=get_client_ip(request),
                popit_person_new_version='',
                popit_person_id=popit_person_id,
                source=form.cleaned_data['justification_for_use'],
            )
            return HttpResponseRedirect(reverse(
                'photo-upload-success',
                kwargs={
                    'popit_person_id': form.cleaned_data['popit_person_id']
                }
            ))
    else:
        form = UploadPersonPhotoForm(
            initial={
                'popit_person_id': popit_person_id
            }
        )
    api = create_popit_api_object()
    return render(
        request,
        'moderation_queue/photo-upload-new.html',
        {'form': form,
         'queued_images': QueuedImage.objects.filter(
             popit_person_id=popit_person_id,
             decision='undecided',
         ).order_by('created'),
         'person': PopItPerson.create_from_popit(api, popit_person_id)}
    )
 def get_context_data(self, **kwargs):
     context = super(CreateDocumentView, self).get_context_data(**kwargs)
     api = create_popit_api_object()
     post_data = get_post_cached(api, self.kwargs['post_id'])['result']
     context['post_label'] = post_data['label']
     return context
Example #24
0
    def handle(self, username=None, **options):
        from slumber.exceptions import HttpClientError
        from candidates.popit import create_popit_api_object
        from candidates.election_specific import PARTY_DATA, shorten_post_label
        from candidates.models import PopItPerson

        if username is None:
            message = "You must supply the name of a user to be associated with the image uploads."
            raise CommandError(message)
        try:
            user = User.objects.get(username=username)
        except User.DoesNotExist:
            message = "No user with the username '{0}' could be found"
            raise CommandError(message.format(username))

        api = create_popit_api_object()

        json_filename = join(dirname(__file__), '..', '..', 'data',
                             'candidates.json')
        with open(json_filename) as f:
            all_data = json.load(f)

        # This map is needed to get getting YNR election data from
        # the election ID used in the JSON file.
        json_election_id_to_name = {
            e['pk']: e['fields']['name']
            for e in all_data if e['model'] == 'elections.election'
        }

        person_dict = {
            e['pk']: e['fields']
            for e in all_data if e['model'] == 'popolo.person'
        }

        candidate_list = [
            dict(person_id=e['pk'], election_id=e['fields']['election'])
            for e in all_data if e['model'] == 'elections.candidate'
        ]

        for candidate in candidate_list:
            vi_person_id = candidate['person_id']
            person_data = person_dict[vi_person_id]
            election_data, post_data = get_post_data(api,
                                                     candidate['election_id'],
                                                     json_election_id_to_name)
            birth_date = None
            if person_data['birth_date']:
                birth_date = str(
                    dateutil.parser.parse(person_data['birth_date'],
                                          dayfirst=True).date())
            name = person_data['name']
            gender = person_data['gender']
            image_url = person_data['image']

            person = get_existing_popit_person(vi_person_id)
            if person:
                print("Found an existing person:", person.get_absolute_url())
            else:
                print("No existing person, creating a new one:", name)
                person = PopItPerson()

            # Now update fields from the imported data:
            person.name = name
            person.gender = gender
            if birth_date:
                person.birth_date = str(birth_date)
            else:
                person.birth_date = None
            standing_in_election = {
                'post_id': post_data['id'],
                'name': shorten_post_label(post_data['label']),
            }
            if 'area' in post_data:
                standing_in_election['mapit_url'] = post_data['area'][
                    'identifier']
            person.standing_in = {election_data.slug: standing_in_election}
            person.party_memberships = {
                election_data.slug: {
                    'id': UNKNOWN_PARTY_ID,
                    'name': PARTY_DATA.party_id_to_name[UNKNOWN_PARTY_ID],
                }
            }
            person.set_identifier('import-id', vi_person_id)
            change_metadata = get_change_metadata(
                None,
                'Imported candidate from JSON',
            )

            person.record_version(change_metadata)
            try:
                person.save_to_popit(api)
                if image_url:
                    enqueue_image(person, user, image_url)
            except HttpClientError as hce:
                print("Got an HttpClientError:", hce.content)
                raise
Example #25
0
    def handle(self, **options):
        from slumber.exceptions import HttpClientError, HttpServerError
        from candidates.election_specific import PARTY_DATA, shorten_post_label
        from candidates.models import PopItPerson
        from candidates.popit import create_popit_api_object

        api = create_popit_api_object()

        csv_filename = join(dirname(__file__), '..', '..', 'data',
                            'candidates.csv')
        with open(csv_filename) as f:
            all_data = csv.DictReader(f)

            for candidate in all_data:
                vi_person_id = candidate['Distrito'] + candidate[
                    'Numero Lista'] + candidate['Posicion'] + candidate[
                        'Cargo'] + candidate['Nombre Lista']

                election_data, post_data = get_post_data(
                    api, candidate['Cargo'], candidate['Distrito'])
                if (election_data == False):
                    print("Skipping: " + candidate['Cargo'] + ", " +
                          candidate['Distrito'] + ", " + candidate['Nombre'])
                    continue

                name = candidate['Nombre']
                birth_date = None
                gender = None
                image_url = None

                person = get_existing_popit_person(vi_person_id)
                if person:
                    print("Found an existing person:",
                          person.get_absolute_url())
                else:
                    print("No existing person, creating a new one:", name)
                    person = PopItPerson()

                # Now update fields from the imported data:
                person.name = name.split(",")[1] + " " + name.split(",")[0]
                person.gender = gender
                if birth_date:
                    person.birth_date = str(birth_date)
                else:
                    person.birth_date = None
                standing_in_election = {
                    'post_id': post_data['id'],
                    'name': shorten_post_label(post_data['label']),
                    'party_list_position': candidate['Posicion'],
                }
                if 'area' in post_data:
                    standing_in_election['mapit_url'] = post_data['area'][
                        'identifier']

                person.standing_in = {election_data.slug: standing_in_election}

                party_id = get_party_id(candidate["Partido"])

                person.party_memberships = {
                    election_data.slug: {
                        'id': party_id,
                        'name': PARTY_DATA.party_id_to_name[party_id],
                    }
                }
                person.set_identifier('import-id', vi_person_id)
                change_metadata = get_change_metadata(
                    None,
                    'Imported candidate from CSV',
                )

                person.record_version(change_metadata)
                try:
                    person.save_to_popit(api)
                except HttpClientError as hce:
                    print("Got an HttpClientError:", hce.content)
                    raise
                except HttpServerError as hse:
                    print("The server error content was:", hse.content)
                    raise
    def handle(self, **options):
        from slumber.exceptions import HttpClientError, HttpServerError
        from candidates.election_specific import PARTY_DATA, shorten_post_label
        from candidates.models import PopItPerson
        from candidates.popit import create_popit_api_object

        api = create_popit_api_object()

        csv_filename = join(
            dirname(__file__), '..', '..','data', 'candidates.csv'
        )
        with open(csv_filename) as f:
            all_data = csv.DictReader(f)

            for candidate in all_data:
                vi_person_id = candidate['Distrito']+candidate['Numero Lista']+candidate['Posicion']+candidate['Cargo']+candidate['Nombre Lista']

                election_data, post_data = get_post_data(
                    api, candidate['Cargo'], candidate['Distrito']
                )
                if (election_data == False):
                    print("Skipping: "+ candidate['Cargo'] +", " + candidate['Distrito']+", " + candidate['Nombre'])
                    continue;

                name = candidate['Nombre']
                birth_date = None
                gender = None
                image_url = None

                person = get_existing_popit_person(vi_person_id)
                if person:
                    print("Found an existing person:", person.get_absolute_url())
                else:
                    print("No existing person, creating a new one:", name)
                    person = PopItPerson()

                # Now update fields from the imported data:
                person.name = name.split(",")[1] + " "  + name.split(",")[0]
                person.gender = gender
                if birth_date:
                    person.birth_date = str(birth_date)
                else:
                    person.birth_date = None
                standing_in_election = {
                    'post_id': post_data['id'],
                    'name': shorten_post_label(post_data['label']),
                    'party_list_position': candidate['Posicion'],
                }
                if 'area' in post_data:
                    standing_in_election['mapit_url'] = post_data['area']['identifier']

                person.standing_in = {
                    election_data.slug: standing_in_election
                }

                party_id = get_party_id(candidate["Partido"]);

                person.party_memberships = {
                    election_data.slug: {
                        'id': party_id,
                        'name': PARTY_DATA.party_id_to_name[party_id],
                    }
                }
                person.set_identifier('import-id', vi_person_id)
                change_metadata = get_change_metadata(
                    None,
                    'Imported candidate from CSV',
                )

                person.record_version(change_metadata)
                try:
                    person.save_to_popit(api)
                except HttpClientError as hce:
                    print("Got an HttpClientError:", hce.content)
                    raise
                except HttpServerError as hse:
                    print("The server error content was:", hse.content)
                    raise
 def handle(self, **options):
     self.api = create_popit_api_object()
     self.scraper_directory = join(
         settings.BASE_DIR, 'data', 'UK-Political-Parties', 'scraper'
     )
     self.parse_data(join(self.scraper_directory, 'all_data.json'))
    def handle(self, username=None, **options):
        from slumber.exceptions import HttpClientError
        from candidates.popit import create_popit_api_object
        from candidates.election_specific import PARTY_DATA, shorten_post_label
        from candidates.models import PopItPerson

        if username is None:
            message = "You must supply the name of a user to be associated with the image uploads."
            raise CommandError(message)
        try:
            user = User.objects.get(username=username)
        except User.DoesNotExist:
            message = "No user with the username '{0}' could be found"
            raise CommandError(message.format(username))

        api = create_popit_api_object()

        json_filename = join(
            dirname(__file__), '..', '..','data', 'candidates.json'
        )
        with open(json_filename) as f:
            all_data = json.load(f)

        # This map is needed to get getting YNR election data from
        # the election ID used in the JSON file.
        json_election_id_to_name = {
            e['pk']: e['fields']['name']
            for e in all_data if e['model'] == 'elections.election'
        }

        person_dict = {
            e['pk']: e['fields']
            for e in all_data if e['model'] == 'popolo.person'
        }

        candidate_list = [
            dict(person_id=e['pk'], election_id=e['fields']['election'])
            for e in all_data if e['model'] == 'elections.candidate'
        ]

        for candidate in candidate_list:
            vi_person_id = candidate['person_id']
            person_data = person_dict[vi_person_id]
            election_data, post_data = get_post_data(
                api, candidate['election_id'], json_election_id_to_name
            )
            birth_date = None
            if person_data['birth_date']:
                birth_date = str(dateutil.parser.parse(
                    person_data['birth_date'], dayfirst=True
                ).date())
            name = person_data['name']
            gender = person_data['gender']
            image_url = person_data['image']

            person = get_existing_popit_person(vi_person_id)
            if person:
                print("Found an existing person:", person.get_absolute_url())
            else:
                print("No existing person, creating a new one:", name)
                person = PopItPerson()

            # Now update fields from the imported data:
            person.name = name
            person.gender = gender
            if birth_date:
                person.birth_date = str(birth_date)
            else:
                person.birth_date = None
            standing_in_election = {
                'post_id': post_data['id'],
                'name': shorten_post_label(post_data['label']),
            }
            if 'area' in post_data:
                standing_in_election['mapit_url'] = post_data['area']['identifier']
            person.standing_in = {
                election_data.slug: standing_in_election
            }
            person.party_memberships = {
                election_data.slug: {
                    'id': UNKNOWN_PARTY_ID,
                    'name': PARTY_DATA.party_id_to_name[UNKNOWN_PARTY_ID],
                }
            }
            person.set_identifier('import-id', vi_person_id)
            change_metadata = get_change_metadata(
                None,
                'Imported candidate from JSON',
            )

            person.record_version(change_metadata)
            try:
                person.save_to_popit(api)
                if image_url:
                    enqueue_image(person, user, image_url)
            except HttpClientError as hce:
                print("Got an HttpClientError:", hce.content)
                raise
    def handle(self, username=None, **options):

        election_data = {
            'prv-2015': 'listedescandidatsauxelectionslegislativeslisteprovincialeanptic.csv',
            'nat-2015': 'listedescandidatsauxelectionslegislativesanptic.csv'
            }

        field_map = {
            'prv-2015': {
                'region': 1,
                'party': 4,
                'list_order': 5,
                'first_name': 7,
                'last_name': 6,
                'gender': 8,
                'birth_date': 9,
                'party_short': 3
            },
            'nat-2015': {
                'region': 0,
                'party': 2,
                'list_order': 3,
                'first_name': 5,
                'last_name': 4,
                'gender': 6,
                'birth_date': 7,
                'party_short': 2
            }
        }

        api = create_popit_api_object()

        party_id_missing = {}
        party_name_to_id = {}
        for party_id, party_name in PARTY_DATA.party_id_to_name.items():
            party_name_to_id[party_name] = party_id

        for election_id, filename in election_data.items():
            csv_filename = join(
                dirname(__file__), '..', '..', 'data', filename
            )

            fields = field_map[election_id]

            with codecs.open(csv_filename, 'r', encoding='windows-1252') as f:

                initial = True
                for candidate in unicode_csv_reader(f):
                    # skip header line
                    if initial:
                        initial = False
                        continue

                    region = candidate[fields['region']]
                    party = candidate[fields['party']]
                    party_list_order = candidate[fields['list_order']]
                    first_name = string.capwords(candidate[fields['first_name']])
                    last_name = string.capwords(candidate[fields['last_name']])
                    gender = candidate[fields['gender']]
                    birth_date = None

                    if candidate[fields['birth_date']] is not None:
                        birth_date = str(dateutil.parser.parse(
                            candidate[fields['birth_date']], dayfirst=True
                        ).date())

                    name = first_name + ' ' + last_name

                    id = '-'.join([
                        re.sub('[^\w]*', '', re.sub(r' ', '-', strip_accents(name.lower()))),
                        re.sub('[^\w]*', '', candidate[fields['party_short']].lower()),
                        birth_date
                    ])

                    # national candidate
                    if region == 'PAYS':
                        region = 'Burkina Faso'
                    election_data, post_data = get_post_data(
                        api, election_id, region
                    )

                    # debug
                    # tmp = '%s %s %s (%s) - %s (%s)' % ( id, first_name, last_name, party, region, post_data['label'] )
                    # print tmp

                    person = get_existing_popit_person(id)
                    if person:
                        # print "Found an existing person:", person.get_absolute_url()
                        pass
                    else:
                        print "No existing person, creating a new one:", name
                        person = PopItPerson()

                    person.set_identifier('import-id', id)
                    person.family_name = last_name
                    person.given_name = first_name
                    person.name = name
                    person.gender = gender
                    if birth_date:
                        person.birth_date = str(birth_date)
                    else:
                        person.birth_date = None

                    standing_in_election = {
                        'post_id': post_data['id'],
                        'name': AREA_POST_DATA.shorten_post_label(
                            election_data.slug,
                            post_data['label'],
                        ),
                        'party_list_position': party_list_order,
                    }

                    if 'area' in post_data:
                        standing_in_election['mapit_url'] = post_data['area']['identifier']

                    person.standing_in = {
                        election_data.slug: standing_in_election
                    }

                    change_metadata = get_change_metadata(
                        None,
                        'Imported candidate from CSV',
                    )

                    party_comp = re.sub(' +', ' ', party)
                    party_id = UNKNOWN_PARTY_ID
                    if party_comp in party_name_to_id.keys():
                        party_id = party_name_to_id[party_comp]
                        party = party_comp
                    else:
                        party_id = party_name_to_id['Unknown Party']
                        party = 'Unknown Party'

                    if party_id == UNKNOWN_PARTY_ID and party_comp not in party_id_missing.keys():
                        party_id_missing[party_comp] = 1

                    person.party_memberships = {
                        election_data.slug: {
                            'id': party_id,
                            'name': party,
                            'imported_name': party_comp
                        }
                    }

                    person.record_version(change_metadata)
                    try:
                        person.save_to_popit(api)
                    except HttpClientError as hce:
                        print "Got an HttpClientError:", hce.content
                        raise

        if len(party_id_missing) > 0:
            print "Unmatched party names:"
            for name in party_id_missing.keys():
                print name
Example #30
0
    def handle(self, username=None, **options):
        from slumber.exceptions import HttpClientError
        from candidates.election_specific import PARTY_DATA, shorten_post_label
        from candidates.models import PopItPerson
        from candidates.popit import create_popit_api_object

        election_data = {
            'prv-2015':
            'listedescandidatsauxelectionslegislativeslisteprovincialeanptic.csv',
            'nat-2015': 'listedescandidatsauxelectionslegislativesanptic.csv'
        }

        field_map = {
            'prv-2015': {
                'region': 1,
                'party': 4,
                'list_order': 5,
                'first_name': 7,
                'last_name': 6,
                'gender': 8,
                'birth_date': 9,
                'party_short': 3
            },
            'nat-2015': {
                'region': 0,
                'party': 2,
                'list_order': 3,
                'first_name': 5,
                'last_name': 4,
                'gender': 6,
                'birth_date': 7,
                'party_short': 2
            }
        }

        api = create_popit_api_object()

        party_id_missing = {}
        party_name_to_id = {}
        for party_id, party_name in PARTY_DATA.party_id_to_name.items():
            party_name_to_id[party_name] = party_id

        for election_id, filename in election_data.items():
            csv_filename = join(dirname(__file__), '..', '..', 'data',
                                filename)

            fields = field_map[election_id]

            with codecs.open(csv_filename, 'r', encoding='windows-1252') as f:

                initial = True
                for candidate in unicode_csv_reader(f):
                    # skip header line
                    if initial:
                        initial = False
                        continue

                    region = candidate[fields['region']]
                    party = candidate[fields['party']]
                    party_list_order = candidate[fields['list_order']]
                    first_name = string.capwords(
                        candidate[fields['first_name']])
                    last_name = string.capwords(candidate[fields['last_name']])
                    gender = candidate[fields['gender']]
                    birth_date = None

                    if candidate[fields['birth_date']] is not None:
                        birth_date = str(
                            dateutil.parser.parse(
                                candidate[fields['birth_date']],
                                dayfirst=True).date())

                    name = first_name + ' ' + last_name

                    id = '-'.join([
                        re.sub('[^\w]*', '',
                               re.sub(r' ', '-', strip_accents(name.lower()))),
                        re.sub('[^\w]*', '',
                               candidate[fields['party_short']].lower()),
                        birth_date
                    ])

                    # national candidate
                    if region == 'PAYS':
                        region = 'Burkina Faso'
                    election_data, post_data = get_post_data(
                        api, election_id, region)

                    # debug
                    # tmp = '%s %s %s (%s) - %s (%s)' % ( id, first_name, last_name, party, region, post_data['label'] )
                    # print(tmp)

                    person = get_existing_popit_person(id)
                    if person:
                        # print("Found an existing person:", person.get_absolute_url())
                        pass
                    else:
                        print("No existing person, creating a new one:", name)
                        person = PopItPerson()

                    person.set_identifier('import-id', id)
                    person.family_name = last_name
                    person.given_name = first_name
                    person.name = name
                    person.gender = gender
                    if birth_date:
                        person.birth_date = str(birth_date)
                    else:
                        person.birth_date = None

                    standing_in_election = {
                        'post_id': post_data['id'],
                        'name': shorten_post_label(post_data['label']),
                        'party_list_position': party_list_order,
                    }

                    if 'area' in post_data:
                        standing_in_election['mapit_url'] = post_data['area'][
                            'identifier']

                    person.standing_in = {
                        election_data.slug: standing_in_election
                    }

                    change_metadata = get_change_metadata(
                        None,
                        'Imported candidate from CSV',
                    )

                    party_comp = re.sub(' +', ' ', party)
                    party_id = UNKNOWN_PARTY_ID
                    if party_comp in party_name_to_id.keys():
                        party_id = party_name_to_id[party_comp]
                        party = party_comp
                    else:
                        party_id = party_name_to_id['Unknown Party']
                        party = 'Unknown Party'

                    if party_id == UNKNOWN_PARTY_ID and party_comp not in party_id_missing.keys(
                    ):
                        party_id_missing[party_comp] = 1

                    person.party_memberships = {
                        election_data.slug: {
                            'id': party_id,
                            'name': party,
                            'imported_name': party_comp
                        }
                    }

                    person.record_version(change_metadata)
                    try:
                        person.save_to_popit(api)
                    except HttpClientError as hce:
                        print("Got an HttpClientError:", hce.content)
                        raise

        if len(party_id_missing) > 0:
            print("Unmatched party names:")
            for name in party_id_missing.keys():
                print(name)
    def handle(self, **options):
        from slumber.exceptions import HttpClientError
        from candidates.cache import get_post_cached, UnknownPostException
        from candidates.election_specific import PARTY_DATA, shorten_post_label
        from candidates.models import PopItPerson
        from candidates.popit import create_popit_api_object

        spreadsheet_url = 'https://docs.google.com/spreadsheets/d/{0}/pub?output=csv'\
                              .format(GOOGLE_DOC_ID)

        candidate_list = requests.get(spreadsheet_url)

        content = StringIO(unicode(candidate_list.content))
        reader = csv.DictReader(content)

        api = create_popit_api_object()

        for row in reader:

            try:
                election_data = Election.objects.get_by_slug('council-member-2015')
                ocd_division = election_data.post_id_format.format(area_id=row['Ward'])
                post_data = get_post_cached(api, ocd_division)['result']
            except (UnknownPostException, memcache.Client.MemcachedKeyCharacterError):
                election_data = Election.objects.get_by_slug('school-board-2015')
                post_data = get_post_cached(api, election_data.post_id_format)['result']

            person_id = slugify(row['Name'])

            person = get_existing_popit_person(person_id)

            if person:
                print("Found an existing person:", row['Name'])
            else:
                print("No existing person, creating a new one:", row['Name'])
                person = PopItPerson()

            person.name = row['Name']

            # TODO: Get these attributes in the spreadsheet
            # person.gender = gender
            # if birth_date:
            #     person.birth_date = str(birth_date)
            # else:
            #     person.birth_date = None

            person.email = row['Campaign Email']
            person.facebook_personal_url = row["Candidate's Personal Facebook Profile"]
            person.facebook_page_url = row['Campaign Facebook Page']


            person.twitter_username = row['Campaign Twitter']\
                                          .replace('N', '')\
                                          .replace('N/A', '')\
                                          .replace('http://twitter.com/', '')\
                                          .replace('https://twitter.com/', '')

            person.linkedin_url = row['LinkedIn']
            person.homepage_url = row['Campaign Website\n']

            standing_in_election = {
                'post_id': post_data['id'],
                'name': shorten_post_label(post_data['label']),
            }
            if 'area' in post_data:
                standing_in_election['mapit_url'] = post_data['area']['identifier']
            person.standing_in = {
                election_data.slug: standing_in_election
            }

            if 'dfl' in row['Party'].lower():
                party_id = 'party:101'
            elif 'green' in row['Party'].lower():
                party_id = 'party:201'
            elif 'independence' in row['Party'].lower():
                party_id = 'party:301'
            else:
                party_id = 'party:401'


            party_name = PARTY_DATA.party_id_to_name[party_id]

            person.party_memberships = {
                election_data.slug: {
                    'id': party_id,
                    'name': party_name,
                }
            }

            person.set_identifier('import-id', person_id)
            change_metadata = get_change_metadata(
                None,
                'Imported candidate from Google Spreadsheet',
            )

            person.record_version(change_metadata)
            try:
                person.save_to_popit(api)

                # TODO: Get candidate Images
                # if image_url:
                #     enqueue_image(person, user, image_url)
            except HttpClientError as hce:
                print "Got an HttpClientError:", hce.content
                raise
 def handle(self, **options):
     self.api = create_popit_api_object()
     self.scraper_directory = join(settings.BASE_DIR, 'data',
                                   'UK-Political-Parties', 'scraper')
     self.parse_data(join(self.scraper_directory, 'all_data.json'))
Example #33
0
 def get_context_data(self, **kwargs):
     context = super(CreateDocumentView, self).get_context_data(**kwargs)
     api = create_popit_api_object()
     post_data = get_post_cached(api, self.kwargs['post_id'])['result']
     context['post_label'] = post_data['label']
     return context