def handle(self, *args, **options):
self.verbosity = int(options.get('verbosity', 1))
popit = create_popit_api_object()
if len(args) != 3:
raise CommandError("You must provide all three arguments")
person_id, scheme, identifier = args
try:
person_data = popit.persons(person_id).get()['result']
except (HttpClientError, HttpServerError):
message = "Failed to get the person with ID {0}"
raise CommandError(message.format(person_id))
person_data['identifiers'].append(
{
'scheme': scheme,
'identifier': identifier,
}
)
try:
popit.persons(person_id).put(person_data)
except (HttpClientError, HttpServerError):
raise CommandError("Failed to update that person")
invalidate_cache_entries_from_person_data(person_data)
# FIXME: this should create a new version in the versions
# array too, otherwise you manually have to edit on YourNextMP
# too to create a new version with a change message.
print "Successfully updated {0}".format(person_id)
def handle(self, *args, **options):
self.verbosity = int(options.get('verbosity', 1))
api = create_popit_api_object()
if len(args) != 2:
raise CommandError("You must provide all two arguments")
person_id, other_name = args
person = PopItPerson.create_from_popit(api, person_id)
person.other_names.append(
{
'name': other_name,
'note': options['note'],
'start_date': options['start_date'],
'end_date': options['end_date']
}
)
person.save_to_popit(api)
person.invalidate_cache_entries()
# FIXME: this should create a new version in the versions
# array too, otherwise you manually have to edit on YourNextMP
# too to create a new version with a change message.
print "Successfully updated {0}".format(person_id)
def handle(self, *args, **options):
self.verbosity = int(options.get('verbosity', 1))
popit = create_popit_api_object()
if len(args) != 2:
raise CommandError("You must provide all two arguments")
person_id, other_name = args
try:
person_data = popit.persons(person_id).get()['result']
except (HttpClientError, HttpServerError):
message = "Failed to get the person with ID {0}"
raise CommandError(message.format(person_id))
person_data['other_names'].append(
{
'name': other_name,
'note': options['note'],
'start_date': options['start_date'],
'end_date': options['end_date']
}
)
try:
popit.persons(person_id).put(person_data)
except (HttpClientError, HttpServerError):
raise CommandError("Failed to update that person")
# FIXME: this should create a new version in the versions
# array too, otherwise you manually have to edit on YourNextMP
# too to create a new version with a change message.
print "Successfully updated {0}".format(person_id)
def handle(self, **options):
joint_party_to_sub_parties = defaultdict(list)
api = create_popit_api_object()
for party in popit_unwrap_pagination(api.organizations,
embed='',
per_page=200):
if party['classification'] != 'Party':
continue
if 'dissolution_date' in party:
dissolution_date = party['dissolution_date']
if dissolution_date < str(date.today()):
continue
for d in party.get('descriptions', []):
m = joint_description_re.search(d['description'])
if not m:
continue
joint_name = fix_joint_party_name(m.group('joint_name'))
joint_party_to_sub_parties[joint_name].append(party)
for joint_name, sub_parties in joint_party_to_sub_parties.items():
if len(sub_parties) < 2:
message = 'Ignoring "{joint_name}" (only made up of one party: "{sub_party}")'
print message.format(joint_name=joint_name,
sub_party=sub_parties[0]['name'])
continue
create_or_update_party(api, joint_name, sub_parties)
def upload_photo(request, popit_person_id):
if request.method == 'POST':
form = UploadPersonPhotoForm(request.POST, request.FILES)
if form.is_valid():
# Make sure that we save the user that made the upload
queued_image = form.save(commit=False)
queued_image.user = request.user
queued_image.save()
# Record that action:
LoggedAction.objects.create(
user=request.user,
action_type='photo-upload',
ip_address=get_client_ip(request),
popit_person_new_version='',
popit_person_id=popit_person_id,
source=form.cleaned_data['justification_for_use'],
)
return HttpResponseRedirect(
reverse('photo-upload-success',
kwargs={
'popit_person_id':
form.cleaned_data['popit_person_id']
}))
else:
form = UploadPersonPhotoForm(
initial={'popit_person_id': popit_person_id})
api = create_popit_api_object()
return render(
request, 'moderation_queue/photo-upload-new.html', {
'form': form,
'person': PopItPerson.create_from_popit(api, popit_person_id)
})
def handle_label(self, output_filename, **options):
api = create_popit_api_object()
ntf = NamedTemporaryFile(
delete=False,
dir=dirname(output_filename)
)
all_parties = []
for party in popit_unwrap_pagination(
api.organizations,
embed='',
per_page=200
):
if party['classification'] != 'Party':
continue
# These URLs are specific to the server this data is
# extracted from:
del party['url']
del party['html_url']
# Similarly the URLs of images:
party.pop('image', None)
party.pop('proxy_image', None)
party.pop('images', None)
# The generated id in each identifier will be different
# between systems (or between runs on the same system) so
# just produces noisy diffs if we include it.
for identifier in party.get('identifiers', []):
identifier.pop('id', None)
all_parties.append(party)
all_parties.sort(key=lambda p: p['id'])
# Output to a string so that we can strip any trainling whitespace.
output = json.dumps(all_parties, sort_keys=True, indent=4)
output = re.sub(r'(?ms)\s*$', '', output)
ntf.write(output)
rename(ntf.name, output_filename)
def handle(self, **options):
self.mime_type_magic = magic.Magic(mime=True)
self.api = create_popit_api_object()
start = 0
per_page = 50
url = 'http://pefonline.electoralcommission.org.uk/api/search/Registrations'
params = {
'rows': per_page,
'et': ["pp", "ppm"],
'register': ["gb", "ni"],
'regStatus': ["registered", "deregistered", "lapsed"],
}
total = None
while total is None or start <= total:
ntf = NamedTemporaryFile(delete=False)
params['start'] = start
try:
resp = requests.get(url + '?' +
urlencode(params, doseq=True)).json()
if total is None:
total = resp['Total']
with open(ntf.name, 'w') as f:
json.dump(resp['Result'], f)
self.parse_data(ntf.name)
finally:
os.remove(ntf.name)
start += per_page
def handle(self, *args, **options):
self.verbosity = int(options.get('verbosity', 1))
api = create_popit_api_object()
if len(args) != 2:
raise CommandError("You must provide all two arguments")
person_id, other_name = args
person = PopItPerson.create_from_popit(api, person_id)
person.other_names.append(
{
'name': other_name,
'note': options['note'],
'start_date': options['start_date'],
'end_date': options['end_date']
}
)
person.save_to_popit(api)
person.invalidate_cache_entries()
# FIXME: this should create a new version in the versions
# array too, otherwise you manually have to edit on the
# YourNextRepresentative site too to create a new version with
# a change message.
print "Successfully updated {0}".format(person_id)
def handle(self, *args, **options):
self.verbosity = int(options.get('verbosity', 1))
api = create_popit_api_object()
if len(args) != 3:
raise CommandError("You must provide all three arguments")
person_id, scheme, identifier = args
person = PopItPerson.create_from_popit(api, person_id)
person.identifiers.append(
{
'scheme': scheme,
'identifier': identifier,
}
)
person.save_to_popit(api)
person.invalidate_cache_entries()
# FIXME: this should create a new version in the versions
# array too, otherwise you manually have to edit on YourNextMP
# too to create a new version with a change message.
print "Successfully updated {0}".format(person_id)
def handle(self, **options):
self.mime_type_magic = magic.Magic(mime=True)
self.api = create_popit_api_object()
start = 0
per_page = 50
url = 'http://pefonline.electoralcommission.org.uk/api/search/Registrations'
params = {
'rows': per_page,
'et': ["pp", "ppm"],
'register': ["gb", "ni"],
'regStatus': ["registered", "deregistered", "lapsed"],
}
total = None
while total is None or start <= total:
ntf = NamedTemporaryFile(delete=False)
params['start'] = start
try:
resp = requests.get(url + '?' + urlencode(params, doseq=True)).json()
if total is None:
total = resp['Total']
with open(ntf.name, 'w') as f:
json.dump(resp['Result'], f)
self.parse_data(ntf.name)
finally:
os.remove(ntf.name)
start += per_page
def handle_label(self, output_filename, **options):
api = create_popit_api_object()
ntf = NamedTemporaryFile(delete=False, dir=dirname(output_filename))
all_parties = []
for party in popit_unwrap_pagination(api.organizations,
embed='',
per_page=200):
if party['classification'] != 'Party':
continue
if 'dissolution_date' in party:
dissolution_date = party['dissolution_date']
if dissolution_date < str(date.today()):
continue
# These URLs are specific to the server this data is
# extracted from:
del party['url']
del party['html_url']
# Similarly the URLs of images:
party.pop('image', None)
party.pop('proxy_image', None)
party.pop('images', None)
# The generated id in each identifier will be different
# between systems (or between runs on the same system) so
# just produces noisy diffs if we include it.
for identifier in party.get('identifiers', []):
identifier.pop('id', None)
all_parties.append(party)
all_parties.sort(key=lambda p: p['id'])
# Output to a string so that we can strip any trainling whitespace.
output = json.dumps(all_parties, sort_keys=True, indent=4)
output = re.sub(r'(?ms)\s*$', '', output)
ntf.write(output)
rename(ntf.name, output_filename)
def handle(self, **options):
joint_party_to_sub_parties = defaultdict(list)
api = create_popit_api_object()
for party in popit_unwrap_pagination(
api.organizations,
embed='',
per_page=200
):
if party['classification'] != 'Party':
continue
if 'dissolution_date' in party:
dissolution_date = party['dissolution_date']
if dissolution_date < str(date.today()):
continue
for d in party.get('descriptions', []):
m = joint_description_re.search(d['description'])
if not m:
continue
joint_name = fix_joint_party_name(m.group('joint_name'))
joint_party_to_sub_parties[joint_name].append(party)
for joint_name, sub_parties in joint_party_to_sub_parties.items():
if len(sub_parties) < 2:
message = 'Ignoring "{joint_name}" (only made up of one party: "{sub_party}")'
print message.format(
joint_name=joint_name,
sub_party=sub_parties[0]['name']
)
continue
create_or_update_party(api, joint_name, sub_parties)
def handle(self, **options):
self.api = create_popit_api_object()
ntf = NamedTemporaryFile(delete=False)
try:
r = requests.get('http://openelectoralcommission.org.uk/parties/index.json')
with open(ntf.name, 'w') as f:
json.dump(r.json(), f)
self.parse_data(ntf.name)
finally:
os.remove(ntf.name)
def handle(self, *args, **kwargs):
if len(args) < 1:
raise CommandError("You must provide one or more PopIt person ID")
for person_id in args:
person = PopItPerson.create_from_popit(
create_popit_api_object(), person_id
)
person.delete_memberships()
self.create_party_memberships(person_id, person.popit_data)
self.create_candidate_list_memberships(person_id, person.popit_data)
def handle(self, **options):
self.api = create_popit_api_object()
ntf = NamedTemporaryFile(delete=False)
try:
r = requests.get('http://openelectoralcommission.org.uk/parties/index.json')
with open(ntf.name, 'w') as f:
json.dump(r.json(), f)
self.parse_data(ntf.name)
finally:
os.remove(ntf.name)
def handle(self, *args, **options):
self.verbosity = int(options.get('verbosity', 1))
self.popit = create_popit_api_object()
with open(CONSTITUENCIES_JSON_FILE) as f:
constituencies = json.load(f).values()
xml = objectify.parse(ALL_MEMBERS_XML_FILE).getroot()
self.members = [member.attrib for member in xml.findall('member')]
people_xml = objectify.parse(PEOPLE_XML_FILE).getroot()
people = people_xml.findall('person')
self.id_mapping = {p.get('latestname'): p.get('id') for p in people}
self.update_candidates(constituencies)
def handle(self, *args, **options):
from candidates.models import PopItPerson
from candidates.popit import create_popit_api_object
self.verbosity = int(options.get('verbosity', 1))
api = create_popit_api_object()
if len(args) != 1:
raise CommandError("You must provide a person.js URL")
person_js_url = args[0]
people_data = requests.get(person_js_url).json()
for person_data in people_data['persons']:
twfy_person = PopItPerson.create_from_dict(person_data)
ynmp_id = twfy_person.get_identifier('yournextmp')
if not ynmp_id:
continue
parlparse_id = twfy_person.id
ynmp_person = PopItPerson.create_from_popit(api, ynmp_id)
existing_parlparse_id = ynmp_person.get_identifier('uk.org.publicwhip')
if existing_parlparse_id:
if existing_parlparse_id == parlparse_id:
# That's fine, there's already the right parlparse ID
pass
else:
# Otherwise there's a mismatch, which needs investigation
msg = "Warning: parlparse ID mismatch between YNMP {0} "
msg += "and TWFY {1} for YNMP person {2}\n"
self.stderr.write(
msg.format(
existing_parlparse_id,
parlparse_id,
ynmp_id,
)
)
continue
msg = "Updating the YourNextMP person {0} with parlparse_id {1}\n"
self.stdout.write(msg.format(ynmp_id, parlparse_id))
ynmp_person.set_identifier(
'uk.org.publicwhip',
parlparse_id,
)
change_metadata = get_change_metadata(
None, "Fetched a new parlparse ID"
)
ynmp_person.record_version(change_metadata)
ynmp_person.save_to_popit(api)
ynmp_person.invalidate_cache_entries()
def handle(self, *args, **options):
from candidates.models import PopItPerson
from candidates.popit import create_popit_api_object
self.verbosity = int(options.get('verbosity', 1))
api = create_popit_api_object()
if len(args) != 1:
raise CommandError("You must provide a person.js URL")
person_js_url = args[0]
people_data = requests.get(person_js_url).json()
for person_data in people_data['persons']:
twfy_person = PopItPerson.create_from_dict(person_data)
ynmp_id = twfy_person.get_identifier('yournextmp')
if not ynmp_id:
continue
parlparse_id = twfy_person.id
ynmp_person = PopItPerson.create_from_popit(api, ynmp_id)
existing_parlparse_id = ynmp_person.get_identifier(
'uk.org.publicwhip')
if existing_parlparse_id:
if existing_parlparse_id == parlparse_id:
# That's fine, there's already the right parlparse ID
pass
else:
# Otherwise there's a mismatch, which needs investigation
msg = "Warning: parlparse ID mismatch between YNMP {0} "
msg += "and TWFY {1} for YNMP person {2}\n"
self.stderr.write(
msg.format(
existing_parlparse_id,
parlparse_id,
ynmp_id,
))
continue
msg = "Updating the YourNextMP person {0} with parlparse_id {1}\n"
self.stdout.write(msg.format(ynmp_id, parlparse_id))
ynmp_person.set_identifier(
'uk.org.publicwhip',
parlparse_id,
)
change_metadata = get_change_metadata(
None, "Fetched a new parlparse ID")
ynmp_person.record_version(change_metadata)
ynmp_person.save_to_popit(api)
ynmp_person.invalidate_cache_entries()
def handle(self, *args, **kwargs):
api = create_popit_api_object()
if len(args) < 1:
raise CommandError("You must provide one or more PopIt person ID")
for person_id in args:
invalidate_person(person_id)
person = PopItPerson.create_from_popit(api, person_id)
posts_to_invalidate = person.get_associated_posts()
person.delete_memberships(api)
# The memberships are recreated when you assign to
# standing_in and party_memberships; this script assumes
# these are correct and so re-setting these should
# recreate the memberships correctly.
person.standing_in = person.standing_in
person.party_memberships = person.party_memberships
person.save_to_popit(api)
invalidate_posts(posts_to_invalidate)
invalidate_person(person_id)
def handle(self, *args, **options):
api = create_popit_api_object()
for org in popit_unwrap_pagination(
api.organizations,
per_page=100
):
org.pop('versions', None)
org.pop('memberships', None)
images = org.get('images', [])
if len(images) < 2:
continue
print "====================================================="
print len(images), org['id'], org['name'].encode('utf-8')
for image in images:
print ' --'
print ' ' + image['notes'].encode('utf-8')
print ' ' + image['url']
def handle(self, *args, **kwargs):
api = create_popit_api_object()
if len(args) < 1:
raise CommandError("You must provide one or more PopIt person ID")
for person_id in args:
invalidate_person(person_id)
person = PopItPerson.create_from_popit(api, person_id)
posts_to_invalidate = person.get_associated_posts()
person.delete_memberships(api)
# The memberships are recreated when you assign to
# standing_in and party_memberships; this script assumes
# these are correct and so re-setting these should
# recreate the memberships correctly.
person.standing_in = person.standing_in
person.party_memberships = person.party_memberships
person.save_to_popit(api)
invalidate_posts(posts_to_invalidate)
invalidate_person(person_id)
def upload_photo(request, popit_person_id):
if request.method == 'POST':
form = UploadPersonPhotoForm(request.POST, request.FILES)
if form.is_valid():
# Make sure that we save the user that made the upload
queued_image = form.save(commit=False)
queued_image.user = request.user
queued_image.save()
# Record that action:
LoggedAction.objects.create(
user=request.user,
action_type='photo-upload',
ip_address=get_client_ip(request),
popit_person_new_version='',
popit_person_id=popit_person_id,
source=form.cleaned_data['justification_for_use'],
)
return HttpResponseRedirect(reverse(
'photo-upload-success',
kwargs={
'popit_person_id': form.cleaned_data['popit_person_id']
}
))
else:
form = UploadPersonPhotoForm(
initial={
'popit_person_id': popit_person_id
}
)
api = create_popit_api_object()
return render(
request,
'moderation_queue/photo-upload-new.html',
{'form': form,
'queued_images': QueuedImage.objects.filter(
popit_person_id=popit_person_id,
decision='undecided',
).order_by('created'),
'person': PopItPerson.create_from_popit(api, popit_person_id)}
)
def get_context_data(self, **kwargs):
context = super(CreateDocumentView, self).get_context_data(**kwargs)
api = create_popit_api_object()
post_data = get_post_cached(api, self.kwargs['post_id'])['result']
context['post_label'] = post_data['label']
return context
def handle(self, username=None, **options):
from slumber.exceptions import HttpClientError
from candidates.popit import create_popit_api_object
from candidates.election_specific import PARTY_DATA, shorten_post_label
from candidates.models import PopItPerson
if username is None:
message = "You must supply the name of a user to be associated with the image uploads."
raise CommandError(message)
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
message = "No user with the username '{0}' could be found"
raise CommandError(message.format(username))
api = create_popit_api_object()
json_filename = join(dirname(__file__), '..', '..', 'data',
'candidates.json')
with open(json_filename) as f:
all_data = json.load(f)
# This map is needed to get getting YNR election data from
# the election ID used in the JSON file.
json_election_id_to_name = {
e['pk']: e['fields']['name']
for e in all_data if e['model'] == 'elections.election'
}
person_dict = {
e['pk']: e['fields']
for e in all_data if e['model'] == 'popolo.person'
}
candidate_list = [
dict(person_id=e['pk'], election_id=e['fields']['election'])
for e in all_data if e['model'] == 'elections.candidate'
]
for candidate in candidate_list:
vi_person_id = candidate['person_id']
person_data = person_dict[vi_person_id]
election_data, post_data = get_post_data(api,
candidate['election_id'],
json_election_id_to_name)
birth_date = None
if person_data['birth_date']:
birth_date = str(
dateutil.parser.parse(person_data['birth_date'],
dayfirst=True).date())
name = person_data['name']
gender = person_data['gender']
image_url = person_data['image']
person = get_existing_popit_person(vi_person_id)
if person:
print("Found an existing person:", person.get_absolute_url())
else:
print("No existing person, creating a new one:", name)
person = PopItPerson()
# Now update fields from the imported data:
person.name = name
person.gender = gender
if birth_date:
person.birth_date = str(birth_date)
else:
person.birth_date = None
standing_in_election = {
'post_id': post_data['id'],
'name': shorten_post_label(post_data['label']),
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area'][
'identifier']
person.standing_in = {election_data.slug: standing_in_election}
person.party_memberships = {
election_data.slug: {
'id': UNKNOWN_PARTY_ID,
'name': PARTY_DATA.party_id_to_name[UNKNOWN_PARTY_ID],
}
}
person.set_identifier('import-id', vi_person_id)
change_metadata = get_change_metadata(
None,
'Imported candidate from JSON',
)
person.record_version(change_metadata)
try:
person.save_to_popit(api)
if image_url:
enqueue_image(person, user, image_url)
except HttpClientError as hce:
print("Got an HttpClientError:", hce.content)
raise
def handle(self, **options):
from slumber.exceptions import HttpClientError, HttpServerError
from candidates.election_specific import PARTY_DATA, shorten_post_label
from candidates.models import PopItPerson
from candidates.popit import create_popit_api_object
api = create_popit_api_object()
csv_filename = join(dirname(__file__), '..', '..', 'data',
'candidates.csv')
with open(csv_filename) as f:
all_data = csv.DictReader(f)
for candidate in all_data:
vi_person_id = candidate['Distrito'] + candidate[
'Numero Lista'] + candidate['Posicion'] + candidate[
'Cargo'] + candidate['Nombre Lista']
election_data, post_data = get_post_data(
api, candidate['Cargo'], candidate['Distrito'])
if (election_data == False):
print("Skipping: " + candidate['Cargo'] + ", " +
candidate['Distrito'] + ", " + candidate['Nombre'])
continue
name = candidate['Nombre']
birth_date = None
gender = None
image_url = None
person = get_existing_popit_person(vi_person_id)
if person:
print("Found an existing person:",
person.get_absolute_url())
else:
print("No existing person, creating a new one:", name)
person = PopItPerson()
# Now update fields from the imported data:
person.name = name.split(",")[1] + " " + name.split(",")[0]
person.gender = gender
if birth_date:
person.birth_date = str(birth_date)
else:
person.birth_date = None
standing_in_election = {
'post_id': post_data['id'],
'name': shorten_post_label(post_data['label']),
'party_list_position': candidate['Posicion'],
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area'][
'identifier']
person.standing_in = {election_data.slug: standing_in_election}
party_id = get_party_id(candidate["Partido"])
person.party_memberships = {
election_data.slug: {
'id': party_id,
'name': PARTY_DATA.party_id_to_name[party_id],
}
}
person.set_identifier('import-id', vi_person_id)
change_metadata = get_change_metadata(
None,
'Imported candidate from CSV',
)
person.record_version(change_metadata)
try:
person.save_to_popit(api)
except HttpClientError as hce:
print("Got an HttpClientError:", hce.content)
raise
except HttpServerError as hse:
print("The server error content was:", hse.content)
raise
def handle(self, **options):
from slumber.exceptions import HttpClientError, HttpServerError
from candidates.election_specific import PARTY_DATA, shorten_post_label
from candidates.models import PopItPerson
from candidates.popit import create_popit_api_object
api = create_popit_api_object()
csv_filename = join(
dirname(__file__), '..', '..','data', 'candidates.csv'
)
with open(csv_filename) as f:
all_data = csv.DictReader(f)
for candidate in all_data:
vi_person_id = candidate['Distrito']+candidate['Numero Lista']+candidate['Posicion']+candidate['Cargo']+candidate['Nombre Lista']
election_data, post_data = get_post_data(
api, candidate['Cargo'], candidate['Distrito']
)
if (election_data == False):
print("Skipping: "+ candidate['Cargo'] +", " + candidate['Distrito']+", " + candidate['Nombre'])
continue;
name = candidate['Nombre']
birth_date = None
gender = None
image_url = None
person = get_existing_popit_person(vi_person_id)
if person:
print("Found an existing person:", person.get_absolute_url())
else:
print("No existing person, creating a new one:", name)
person = PopItPerson()
# Now update fields from the imported data:
person.name = name.split(",")[1] + " " + name.split(",")[0]
person.gender = gender
if birth_date:
person.birth_date = str(birth_date)
else:
person.birth_date = None
standing_in_election = {
'post_id': post_data['id'],
'name': shorten_post_label(post_data['label']),
'party_list_position': candidate['Posicion'],
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area']['identifier']
person.standing_in = {
election_data.slug: standing_in_election
}
party_id = get_party_id(candidate["Partido"]);
person.party_memberships = {
election_data.slug: {
'id': party_id,
'name': PARTY_DATA.party_id_to_name[party_id],
}
}
person.set_identifier('import-id', vi_person_id)
change_metadata = get_change_metadata(
None,
'Imported candidate from CSV',
)
person.record_version(change_metadata)
try:
person.save_to_popit(api)
except HttpClientError as hce:
print("Got an HttpClientError:", hce.content)
raise
except HttpServerError as hse:
print("The server error content was:", hse.content)
raise
def handle(self, **options):
self.api = create_popit_api_object()
self.scraper_directory = join(
settings.BASE_DIR, 'data', 'UK-Political-Parties', 'scraper'
)
self.parse_data(join(self.scraper_directory, 'all_data.json'))
def handle(self, username=None, **options):
from slumber.exceptions import HttpClientError
from candidates.popit import create_popit_api_object
from candidates.election_specific import PARTY_DATA, shorten_post_label
from candidates.models import PopItPerson
if username is None:
message = "You must supply the name of a user to be associated with the image uploads."
raise CommandError(message)
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
message = "No user with the username '{0}' could be found"
raise CommandError(message.format(username))
api = create_popit_api_object()
json_filename = join(
dirname(__file__), '..', '..','data', 'candidates.json'
)
with open(json_filename) as f:
all_data = json.load(f)
# This map is needed to get getting YNR election data from
# the election ID used in the JSON file.
json_election_id_to_name = {
e['pk']: e['fields']['name']
for e in all_data if e['model'] == 'elections.election'
}
person_dict = {
e['pk']: e['fields']
for e in all_data if e['model'] == 'popolo.person'
}
candidate_list = [
dict(person_id=e['pk'], election_id=e['fields']['election'])
for e in all_data if e['model'] == 'elections.candidate'
]
for candidate in candidate_list:
vi_person_id = candidate['person_id']
person_data = person_dict[vi_person_id]
election_data, post_data = get_post_data(
api, candidate['election_id'], json_election_id_to_name
)
birth_date = None
if person_data['birth_date']:
birth_date = str(dateutil.parser.parse(
person_data['birth_date'], dayfirst=True
).date())
name = person_data['name']
gender = person_data['gender']
image_url = person_data['image']
person = get_existing_popit_person(vi_person_id)
if person:
print("Found an existing person:", person.get_absolute_url())
else:
print("No existing person, creating a new one:", name)
person = PopItPerson()
# Now update fields from the imported data:
person.name = name
person.gender = gender
if birth_date:
person.birth_date = str(birth_date)
else:
person.birth_date = None
standing_in_election = {
'post_id': post_data['id'],
'name': shorten_post_label(post_data['label']),
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area']['identifier']
person.standing_in = {
election_data.slug: standing_in_election
}
person.party_memberships = {
election_data.slug: {
'id': UNKNOWN_PARTY_ID,
'name': PARTY_DATA.party_id_to_name[UNKNOWN_PARTY_ID],
}
}
person.set_identifier('import-id', vi_person_id)
change_metadata = get_change_metadata(
None,
'Imported candidate from JSON',
)
person.record_version(change_metadata)
try:
person.save_to_popit(api)
if image_url:
enqueue_image(person, user, image_url)
except HttpClientError as hce:
print("Got an HttpClientError:", hce.content)
raise
def handle(self, username=None, **options):
election_data = {
'prv-2015': 'listedescandidatsauxelectionslegislativeslisteprovincialeanptic.csv',
'nat-2015': 'listedescandidatsauxelectionslegislativesanptic.csv'
}
field_map = {
'prv-2015': {
'region': 1,
'party': 4,
'list_order': 5,
'first_name': 7,
'last_name': 6,
'gender': 8,
'birth_date': 9,
'party_short': 3
},
'nat-2015': {
'region': 0,
'party': 2,
'list_order': 3,
'first_name': 5,
'last_name': 4,
'gender': 6,
'birth_date': 7,
'party_short': 2
}
}
api = create_popit_api_object()
party_id_missing = {}
party_name_to_id = {}
for party_id, party_name in PARTY_DATA.party_id_to_name.items():
party_name_to_id[party_name] = party_id
for election_id, filename in election_data.items():
csv_filename = join(
dirname(__file__), '..', '..', 'data', filename
)
fields = field_map[election_id]
with codecs.open(csv_filename, 'r', encoding='windows-1252') as f:
initial = True
for candidate in unicode_csv_reader(f):
# skip header line
if initial:
initial = False
continue
region = candidate[fields['region']]
party = candidate[fields['party']]
party_list_order = candidate[fields['list_order']]
first_name = string.capwords(candidate[fields['first_name']])
last_name = string.capwords(candidate[fields['last_name']])
gender = candidate[fields['gender']]
birth_date = None
if candidate[fields['birth_date']] is not None:
birth_date = str(dateutil.parser.parse(
candidate[fields['birth_date']], dayfirst=True
).date())
name = first_name + ' ' + last_name
id = '-'.join([
re.sub('[^\w]*', '', re.sub(r' ', '-', strip_accents(name.lower()))),
re.sub('[^\w]*', '', candidate[fields['party_short']].lower()),
birth_date
])
# national candidate
if region == 'PAYS':
region = 'Burkina Faso'
election_data, post_data = get_post_data(
api, election_id, region
)
# debug
# tmp = '%s %s %s (%s) - %s (%s)' % ( id, first_name, last_name, party, region, post_data['label'] )
# print tmp
person = get_existing_popit_person(id)
if person:
# print "Found an existing person:", person.get_absolute_url()
pass
else:
print "No existing person, creating a new one:", name
person = PopItPerson()
person.set_identifier('import-id', id)
person.family_name = last_name
person.given_name = first_name
person.name = name
person.gender = gender
if birth_date:
person.birth_date = str(birth_date)
else:
person.birth_date = None
standing_in_election = {
'post_id': post_data['id'],
'name': AREA_POST_DATA.shorten_post_label(
election_data.slug,
post_data['label'],
),
'party_list_position': party_list_order,
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area']['identifier']
person.standing_in = {
election_data.slug: standing_in_election
}
change_metadata = get_change_metadata(
None,
'Imported candidate from CSV',
)
party_comp = re.sub(' +', ' ', party)
party_id = UNKNOWN_PARTY_ID
if party_comp in party_name_to_id.keys():
party_id = party_name_to_id[party_comp]
party = party_comp
else:
party_id = party_name_to_id['Unknown Party']
party = 'Unknown Party'
if party_id == UNKNOWN_PARTY_ID and party_comp not in party_id_missing.keys():
party_id_missing[party_comp] = 1
person.party_memberships = {
election_data.slug: {
'id': party_id,
'name': party,
'imported_name': party_comp
}
}
person.record_version(change_metadata)
try:
person.save_to_popit(api)
except HttpClientError as hce:
print "Got an HttpClientError:", hce.content
raise
if len(party_id_missing) > 0:
print "Unmatched party names:"
for name in party_id_missing.keys():
print name
def handle(self, username=None, **options):
from slumber.exceptions import HttpClientError
from candidates.election_specific import PARTY_DATA, shorten_post_label
from candidates.models import PopItPerson
from candidates.popit import create_popit_api_object
election_data = {
'prv-2015':
'listedescandidatsauxelectionslegislativeslisteprovincialeanptic.csv',
'nat-2015': 'listedescandidatsauxelectionslegislativesanptic.csv'
}
field_map = {
'prv-2015': {
'region': 1,
'party': 4,
'list_order': 5,
'first_name': 7,
'last_name': 6,
'gender': 8,
'birth_date': 9,
'party_short': 3
},
'nat-2015': {
'region': 0,
'party': 2,
'list_order': 3,
'first_name': 5,
'last_name': 4,
'gender': 6,
'birth_date': 7,
'party_short': 2
}
}
api = create_popit_api_object()
party_id_missing = {}
party_name_to_id = {}
for party_id, party_name in PARTY_DATA.party_id_to_name.items():
party_name_to_id[party_name] = party_id
for election_id, filename in election_data.items():
csv_filename = join(dirname(__file__), '..', '..', 'data',
filename)
fields = field_map[election_id]
with codecs.open(csv_filename, 'r', encoding='windows-1252') as f:
initial = True
for candidate in unicode_csv_reader(f):
# skip header line
if initial:
initial = False
continue
region = candidate[fields['region']]
party = candidate[fields['party']]
party_list_order = candidate[fields['list_order']]
first_name = string.capwords(
candidate[fields['first_name']])
last_name = string.capwords(candidate[fields['last_name']])
gender = candidate[fields['gender']]
birth_date = None
if candidate[fields['birth_date']] is not None:
birth_date = str(
dateutil.parser.parse(
candidate[fields['birth_date']],
dayfirst=True).date())
name = first_name + ' ' + last_name
id = '-'.join([
re.sub('[^\w]*', '',
re.sub(r' ', '-', strip_accents(name.lower()))),
re.sub('[^\w]*', '',
candidate[fields['party_short']].lower()),
birth_date
])
# national candidate
if region == 'PAYS':
region = 'Burkina Faso'
election_data, post_data = get_post_data(
api, election_id, region)
# debug
# tmp = '%s %s %s (%s) - %s (%s)' % ( id, first_name, last_name, party, region, post_data['label'] )
# print(tmp)
person = get_existing_popit_person(id)
if person:
# print("Found an existing person:", person.get_absolute_url())
pass
else:
print("No existing person, creating a new one:", name)
person = PopItPerson()
person.set_identifier('import-id', id)
person.family_name = last_name
person.given_name = first_name
person.name = name
person.gender = gender
if birth_date:
person.birth_date = str(birth_date)
else:
person.birth_date = None
standing_in_election = {
'post_id': post_data['id'],
'name': shorten_post_label(post_data['label']),
'party_list_position': party_list_order,
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area'][
'identifier']
person.standing_in = {
election_data.slug: standing_in_election
}
change_metadata = get_change_metadata(
None,
'Imported candidate from CSV',
)
party_comp = re.sub(' +', ' ', party)
party_id = UNKNOWN_PARTY_ID
if party_comp in party_name_to_id.keys():
party_id = party_name_to_id[party_comp]
party = party_comp
else:
party_id = party_name_to_id['Unknown Party']
party = 'Unknown Party'
if party_id == UNKNOWN_PARTY_ID and party_comp not in party_id_missing.keys(
):
party_id_missing[party_comp] = 1
person.party_memberships = {
election_data.slug: {
'id': party_id,
'name': party,
'imported_name': party_comp
}
}
person.record_version(change_metadata)
try:
person.save_to_popit(api)
except HttpClientError as hce:
print("Got an HttpClientError:", hce.content)
raise
if len(party_id_missing) > 0:
print("Unmatched party names:")
for name in party_id_missing.keys():
print(name)
def handle(self, **options):
from slumber.exceptions import HttpClientError
from candidates.cache import get_post_cached, UnknownPostException
from candidates.election_specific import PARTY_DATA, shorten_post_label
from candidates.models import PopItPerson
from candidates.popit import create_popit_api_object
spreadsheet_url = 'https://docs.google.com/spreadsheets/d/{0}/pub?output=csv'\
.format(GOOGLE_DOC_ID)
candidate_list = requests.get(spreadsheet_url)
content = StringIO(unicode(candidate_list.content))
reader = csv.DictReader(content)
api = create_popit_api_object()
for row in reader:
try:
election_data = Election.objects.get_by_slug('council-member-2015')
ocd_division = election_data.post_id_format.format(area_id=row['Ward'])
post_data = get_post_cached(api, ocd_division)['result']
except (UnknownPostException, memcache.Client.MemcachedKeyCharacterError):
election_data = Election.objects.get_by_slug('school-board-2015')
post_data = get_post_cached(api, election_data.post_id_format)['result']
person_id = slugify(row['Name'])
person = get_existing_popit_person(person_id)
if person:
print("Found an existing person:", row['Name'])
else:
print("No existing person, creating a new one:", row['Name'])
person = PopItPerson()
person.name = row['Name']
# TODO: Get these attributes in the spreadsheet
# person.gender = gender
# if birth_date:
# person.birth_date = str(birth_date)
# else:
# person.birth_date = None
person.email = row['Campaign Email']
person.facebook_personal_url = row["Candidate's Personal Facebook Profile"]
person.facebook_page_url = row['Campaign Facebook Page']
person.twitter_username = row['Campaign Twitter']\
.replace('N', '')\
.replace('N/A', '')\
.replace('http://twitter.com/', '')\
.replace('https://twitter.com/', '')
person.linkedin_url = row['LinkedIn']
person.homepage_url = row['Campaign Website\n']
standing_in_election = {
'post_id': post_data['id'],
'name': shorten_post_label(post_data['label']),
}
if 'area' in post_data:
standing_in_election['mapit_url'] = post_data['area']['identifier']
person.standing_in = {
election_data.slug: standing_in_election
}
if 'dfl' in row['Party'].lower():
party_id = 'party:101'
elif 'green' in row['Party'].lower():
party_id = 'party:201'
elif 'independence' in row['Party'].lower():
party_id = 'party:301'
else:
party_id = 'party:401'
party_name = PARTY_DATA.party_id_to_name[party_id]
person.party_memberships = {
election_data.slug: {
'id': party_id,
'name': party_name,
}
}
person.set_identifier('import-id', person_id)
change_metadata = get_change_metadata(
None,
'Imported candidate from Google Spreadsheet',
)
person.record_version(change_metadata)
try:
person.save_to_popit(api)
# TODO: Get candidate Images
# if image_url:
# enqueue_image(person, user, image_url)
except HttpClientError as hce:
print "Got an HttpClientError:", hce.content
raise
def handle(self, **options):
self.api = create_popit_api_object()
self.scraper_directory = join(settings.BASE_DIR, 'data',
'UK-Political-Parties', 'scraper')
self.parse_data(join(self.scraper_directory, 'all_data.json'))
def get_context_data(self, **kwargs):
context = super(CreateDocumentView, self).get_context_data(**kwargs)
api = create_popit_api_object()
post_data = get_post_cached(api, self.kwargs['post_id'])['result']
context['post_label'] = post_data['label']
return context