def test_find_no_results(self):
person = Person(forename='Willem Caspar',
surname_prefix='de',
surname='Jonge',
initials='J.')
tkperson = openkamer.parliament.find_tkapi_person(person)
self.assertIsNone(tkperson)
def test_find_person_no_results_jean(self):
person = Person(forename='Jean Baptiste Anne',
surname_prefix='de',
surname='Gerlache de Biourge',
initials='J.B.A.')
tkperson = openkamer.parliament.find_tkapi_person(person)
self.assertIsNone(tkperson)
def signup(request):
data = request.POST
if 'username' not in data or 'password' not in data or 'email' not in data or data[
'username'] == '' or data['password'] == '' or data['email'] == '':
return utils.createError('Invalid input form.')
_username = data.get('username', '')
_email = data.get('email', '')
_password = data.get('password', '')
#--validation--
if Person.objects.filter(username=_username):
return utils.createError('Username has been taken.')
if Person.objects.filter(email=_email):
return utils.createError('Email has been taken.')
if len(_password) < 8:
return utils.createError('Length of password must be at least 8.')
#--create person--
_salt = utils.getRandomString()
_hashed_password = utils.hashPassword(_password, _salt)
person = Person(username=_username,
salt=_salt,
password=_hashed_password,
email=_email)
person.save()
return utils.createResponse({
'action': 'Sign up successfully',
'username': person.username,
'email': person.email
})
def test_person_str():
p = Person(
first_name='a',
last_name='b',
birth_date='1969-01-01',
)
assert str(p) == 'a b 1969-01-01'
def load_authors():
users_mapping = load_users_mapping()
for user_api in users_data_stream('users'):
mapped_user = users_mapping[str(user_api['id'])]
if not mapped_user['duplicate'] and not mapped_user['delete']:
mapped_user_title = '{0} {1}'.format(mapped_user['first_name'],
mapped_user['last_name'])
mapped_user_slug = slugify(mapped_user_title)
print("id is: ")
print(user_api['id'])
db_user = Person.objects.filter(slug=mapped_user_slug).first()
role_info = get_role_info(user_api['roles'])
if not db_user and mapped_user_slug:
db_user = Person(
search_description='',
seo_title='',
show_in_menus=False,
slug=mapped_user_slug,
title=mapped_user_title,
first_name=mapped_user['first_name'],
last_name=mapped_user['last_name'],
position_at_new_america=user_api['title'],
role=role_info['role_title'],
email=user_api['email'],
expert=role_info['expert'],
depth=4,
profile_image=download_image(
user_api['image'],
mapped_user_slug + "_person_image.jpeg"),
short_bio=user_api['short_bio'],
long_bio=user_api['long_bio'],
)
our_people_page.add_child(instance=db_user)
db_user.save()
print("New user")
print(db_user)
elif db_user and mapped_user_slug:
db_user.search_description = ''
db_user.seo_title = ''
db_user.show_in_menus = False
db_user.slug = mapped_user_slug
db_user.title = mapped_user_title
db_user.first_name = mapped_user['first_name']
db_user.last_name = mapped_user['last_name']
db_user.position_at_new_america = user_api['title']
db_user.role = role_info['role_title']
db_user.email = user_api['email']
db_user.expert = role_info['expert']
db_user.depth = 4
db_user.short_bio = user_api['short_bio']
db_user.long_bio = user_api['long_bio']
db_user.profile_image = download_image(
user_api['image'], mapped_user_slug + "_person_image.jpeg")
print("Existing user")
print(db_user)
db_user.save()
def main():
from person.models import Person
with open("1.json") as jsonfile:
json_data = json.load(jsonfile)
PersonList = []
for dicobj in json_data:
person = Person(volume=dicobj["volume"], postdate=dicobj["postdate"], gender=dicobj["gender"],idnum=dicobj["idnum"],city=dicobj["city"],dec=dicobj["dec"],image_urls=dicobj["image_urls"][0] )
PersonList.append(person)
Person.objects.bulk_create(PersonList)
def create_person(person_data={}):
people_page = OurPeoplePage.objects.first()
if people_page == None:
home_page = HomePage.objects.first()
people_page = home_page.add_child(instance=OurPeoplePage(
title='Our People'))
person = people_page.add_child(instance=Person(
**PostFactory.person_data(**person_data)))
return person
def create_person(self, data):
fields_data, m2m_fields_data = self.split_data(data)
person = Person(**fields_data)
# person.full_clean() # full_clean called in save method
person.save()
for field_name, id_list in m2m_fields_data.items():
m2m_data = data.get(field_name)
if m2m_data:
getattr(person, field_name).set(id_list)
return person
def setUp(self):
self.login()
site = Site.objects.get()
page = Page.get_first_root_node()
home = HomePage(title='New America')
home_page = page.add_child(instance=home)
site.root_page = home
site.save()
# People objects to test template tags
# that create byline and determine author order
program_page_1 = home_page.add_child(
instance=Program(title='OTI',
name='OTI',
slug='oti',
description='OTI',
location=False,
depth=3))
program_page_1.save()
our_people_page = home_page.add_child(instance=OurPeoplePage(
title='Our People',
depth=3,
))
our_people_page.save()
self.first_person = Person(
title='First Person',
slug='first-person',
first_name='first',
last_name='person',
role='Central Staff',
depth=4,
)
our_people_page.add_child(instance=self.first_person)
# Using policy papers to test the other post types
all_policy_papers_home_page = home_page.add_child(
instance=AllPolicyPapersHomePage(title="Policy Papers"))
program_policy_papers_page = program_page_1.add_child(
instance=ProgramPolicyPapersPage(title='OTI Policy Papers',
slug='oti-policy-papers'))
self.policy_paper = PolicyPaper(title='Policy Paper 1',
slug='policy-paper-1',
date='2016-06-15',
depth=5)
program_policy_papers_page.add_child(instance=self.policy_paper)
self.policy_paper.save()
PostAuthorRelationship(author=self.first_person,
post=self.policy_paper).save()
all_policy_papers_home_page.save()
def handle(self, *args, **kwargs):
filename = kwargs['filename']
with open(filename, encoding='utf-8') as f:
models = json.load(f)
# Verify all data before Insertion, Fail if any one data is corrupted
record_count = 0
for m in models:
record_count = record_count + 1
valid_status = person_validation.validate_add(m)
if not valid_status[0]:
raise ValidationError(valid_status[1])
for person in models:
person_data = person.get('data')[0]
enabled = person_data.get('enabled')
guid = person_data.get('guid')
first_name = person.get('first_name')
last_name = person.get('last_name')
city = person.get('city')
p_id = person.get('id')
person_data_model_obj = PersonData(enabled=enabled, guid=guid)
save_flag = True
try:
person_data_model_obj.save()
except ValueError as e:
save_flag = False
person_data_model_obj.delete()
self.stderr.write(
'PersonData Creation in DB Failed {id = ' + str(p_id) +
'}', e)
try:
if save_flag:
person_model = Person(id=p_id,
first_name=first_name,
last_name=last_name,
city=city,
data=person_data_model_obj)
person_model.save(force_insert=True)
except IntegrityError as e:
person_data_model_obj.delete()
self.stderr.write(
'Record Creation in DB Failed {id = ' + str(p_id) + '}', e)
except ValueError as e:
person_data_model_obj.delete()
self.stderr.write(
'Record Creation in DB Failed {id = ' + str(p_id) + '}', e)
def create_person(self):
person = Person()
person.sex = self.__generate_sex()
person.birth_date = self.__generate_date()
person.first_name = self.__generate_name(person.sex)
person.last_name = self.__generate_surname(person.sex)
person.pesel = self.__generate_pesel(person.birth_date, person.sex)
person.email = self.__generate_email(person.first_name,
person.last_name)
person.phone = self.__generate_phone()
person.password = self.__generate_password()
logging.basicConfig(level=logging.DEBUG)
logging.info(
f"Created person {person.first_name} {person.last_name} with pesel {person.pesel}"
)
return person
def test_person_validate_nirs(nir, expected):
person = Person(
first_name='a',
last_name='b',
nir=nir,
birth_date='1969-01-01',
email="*****@*****.**",
)
if not expected:
person.full_clean()
assert person.nir == str(nir)
else:
with pytest.raises(ValidationError) as raised:
person.full_clean()
assert len(raised.value.error_dict['nir']) == expected
def make_person(name, building, relation, address=None, city=None):
#now associate applicant with building:
#first find/make person
people = Person.objects.filter(city=city).filter(name=name)
person = None
#check if a previous building object in the db exists
if people.exists():
person = people[0]
print "Already had Person: %s" % person.name
else:
#if not,
#CREATE A NEW PERSON OBJECT HERE
person = Person()
person.name = name
if city:
person.city = city
if address:
person.address = address
person.save()
#then find/make association:
bpeople = BuildingPerson.objects.filter(building=building).filter(person=person)
bperson = None
#check if a previous building_person object in the db exists
if bpeople.exists():
bperson = bpeople[0]
print "Already had BuildingPerson: %s with: %s" % (bperson.person.name, bperson.building.address)
else:
#if not,
#CREATE A NEW BUILDING PERSON OBJECT HERE
bperson = BuildingPerson()
bperson.person = person
bperson.building = building
bperson.relation = relation
bperson.save()
return (person, bperson)
def personPage(request):
form = PersonCreateForm
context = {}
context["form"] = form
if request.method == "POST":
form = PersonCreateForm(request.POST)
if form.is_valid():
personname = request.POST.get("personname")
age = request.POST.get("age")
salary = request.POST.get("salary")
desig = request.POST.get("desig")
print(personname, ",", age, ",", salary, ",", desig)
person = Person(personname=personname,
age=age,
salary=salary,
desig=desig)
person.save()
return redirect("getdetails")
return render(request, "person/person.html", context)
def __call__(self, request, *args, **kwargs):
"""
Due to security concerns, do not overload this function or
any of its subclass overrides.
"""
self.request = request
self.args = args
self.kwargs = kwargs
if self.request.user.is_authenticated():
self.user = self.request.user
try:
self.person = Person.objects.get(user=self.user)
except Person.DoesNotExist:
self.person = Person(user=self.user)
self.person.save()
return self.allow(*args, **kwargs)
else:
return self.deny()
def create_person(request):
# redirect to sign in not logged in
if 'id' not in request.session.keys():
return redirect('/sign_in/')
# if this is a POST request we need to process the form dat
if request.method == 'POST':
# create a form instance and populate it with data from the request:
form = PersonForm(request.POST)
# check whether it's valid:
if form.is_valid():
form_data = form.cleaned_data
current_user = User.objects.get(pk=request.user.id)
new_row = Person(
user=current_user,
name=form_data['name'],
message=form_data['message'],
created=datetime.datetime.now(),
start_date=form_data['start_date'],
last_send=None,
next_send=form_data['start_date'],
interval=form_data['interval'],
interval_type=form_data['interval_type'],
snooze=0,
snooze_interval=None,
snooze_interval_type=None,
snooze_last_send=None,
snooze_next_send=None,
in_deleted=0
)
new_row.save()
return redirect('/view_person_list/')
else:
form = PersonForm()
return render(request, 'person/create_person.html', {'form': form})
def test_find_person_haegen(self):
person = Person(forename='Melanie',
surname='Schultz van Haegen',
initials='M.H.')
tkperson = openkamer.parliament.find_tkapi_person(person)
self.assertIsNotNone(tkperson)
def test_find_person_no_results_dekker(self):
"""Should not be confused with Sander Dekker."""
person = Person(forename='Suzanne', surname='Dekker', initials='S.')
tkperson = openkamer.parliament.find_tkapi_person(person)
self.assertIsNone(tkperson)
def test_find_person_wrong_initials(self):
person = Person(surname='Wiersma', initials='H.')
tkperson = openkamer.parliament.find_tkapi_person(person)
self.assertIsNone(tkperson)
def test_find_person_arissen(self):
person = Person(surname='Merel Arissen',
forename='Femke',
initials='F.M.')
tkperson = openkamer.parliament.find_tkapi_person(person)
self.assertEqual(tkperson.achternaam, 'Kooten-Arissen')
def test_find_person_pater_postma(self):
person = Person(surname='Pater-Postma',
forename='Wytske',
initials='W.L.')
tkperson = openkamer.parliament.find_tkapi_person(person)
self.assertEqual(tkperson.achternaam, 'Postma')
def _create_person(self, person_data):
"""Returns a newly created Person object from `person_data`."""
person = Person(**person_data)
person.save()
return person
def PersonRegistration(request, slug):
if slug == 'doer':
is_doer = True
is_poster = False
elif slug == 'poster':
is_poster = True
is_doer = False
if request.user.is_authenticated():
return HttpResponseRedirect('/profile/')
if request.method == 'POST':
form = RegistrationForm(request.POST)
if form.is_valid():
user = User.objects.create_user(
username=form.cleaned_data['email'],
email=form.cleaned_data['email'],
password=form.cleaned_data['password'],
)
firstname = form.cleaned_data['firstname']
lastname = form.cleaned_data['lastname']
user.first_name = firstname
user.last_name = lastname
name = '%s %s' % (user.first_name, user.last_name)
p = PersonClassification.objects.get(slug=slug)
a = slug
slug = str(user.id) + str(datetime.now().day) + str(
datetime.now().year)
skills = form.cleaned_data['skills']
person = Person(
person=user,
person_class=p,
is_verified=False,
is_active=False,
address=form.cleaned_data['address'],
birthday=form.cleaned_data['birthday'],
firstname=form.cleaned_data['firstname'],
lastname=form.cleaned_data['lastname'],
created_at=datetime.now(),
contactno=form.cleaned_data['contactno'],
#classification = a,
)
user.save()
person.save()
person.skills = skills
if person.person_class.__str__() == 'poster':
template = 'register_poster.html'
else:
template = 'register_doer.html'
'''
send verification instructions to user
'''
subject = 'Welcome to Tulong.ph'
message = 'Hi %s we from tulong.ph, gladly welcomes you. Please reply to this email if you have any questions.' % person.firstname
to = person.person.email
#send_mail(subject,message,'*****@*****.**',[to],fail_silently=False)
name = "%s %s" % (person.firstname, person.lastname)
context = {'full_name': name, 'person': person}
text_content = render_to_string('email/register.html', context)
html_content = render_to_string('email/register_html.html',
context)
msg = EmailMultiAlternatives(subject, text_content,
settings.EMAIL_HOST_USER, [to])
msg.attach_alternative(html_content, "text/html")
msg.send()
#send_mail(subject, email_content,
# settings.EMAIL_HOST_USER, [to],fail_silently=False)
'''
send the new user to the admin
'''
person_classification = person.person_class.__str__()
subject = 'New User as %s' % (person_classification)
message = 'User %s Registered as %s ' % (person.person.email,
person_classification)
to = '*****@*****.**'
send_mail(subject,
message,
'*****@*****.**', [to],
fail_silently=False)
return render_to_response(template, {
'status': person,
'is_doer': is_doer,
'is_poster': is_poster,
},
context_instance=RequestContext(request))
#return HttpResponseRedirect('/login/')
else:
return render_to_response('register.html', {
'form': form,
'is_doer': is_doer,
'is_poster': is_poster,
},
context_instance=RequestContext(request))
else:
''' user is not submitting the form, show them a blank registration form '''
form = RegistrationForm()
context = {
'form': form,
'is_doer': is_doer,
'is_poster': is_poster,
}
return render_to_response('register.html',
context,
context_instance=RequestContext(request))
def add_person_view(request):
"""
Function to add new Person
"""
# Predefined structure for response object.
response_data = {'status': '', 'code': None, 'message': None, 'data': []}
logger.info('Person Creation Endpoint Accessed : /api/new')
if request.method == 'PUT':
try:
models = json.loads(request.body)
# Verify all data before Insertion, Fail if any one data is corrupted
record_count = 0
for m in models:
record_count = record_count + 1
valid_status = person_validation.validate_add(m)
if not valid_status[0]:
raise ValidationError(valid_status[1])
for person in models:
person_data = person.get('data')[0]
enabled = person_data.get('enabled')
guid = person_data.get('guid')
first_name = person.get('first_name')
last_name = person.get('last_name')
city = person.get('city')
p_id = person.get('id')
person_data_model_obj = PersonData(enabled=enabled, guid=guid)
save_flag = True
try:
person_data_model_obj.save()
except ValueError as e:
save_flag = False
person_data_model_obj.delete()
response_data['status'] = False
response_data['code'] = 500
response_data[
'message'] = 'PersonData Creation in DB Failed {id = ' + str(
p_id) + '}'
logger.error('PersonData Creation in DB Failed {id = ' +
str(p_id) + '}')
try:
if save_flag:
person_model = Person(id=p_id,
first_name=first_name,
last_name=last_name,
city=city,
data=person_data_model_obj)
person_model.save(force_insert=True)
response_data['status'] = True
response_data['code'] = 200
response_data[
'message'] = 'Data Creation Success, id=' + str(
p_id)
response_data['data'] = []
logger.info(
'Person Record Created Successfully : id=' +
str(p_id))
except IntegrityError as e:
person_data_model_obj.delete()
response_data['status'] = False
response_data['code'] = 500
response_data[
'message'] = 'Record Creation in DB Failed {id = ' + str(
p_id) + '}'
logger.error(
'PersonData Creation in DB Failed {id = ' + str(p_id) +
'}', e)
except ValueError as e:
person_data_model_obj.delete()
response_data['status'] = False
response_data['code'] = 500
response_data[
'message'] = 'Record Creation in DB Failed {id = ' + str(
p_id) + '}'
logger.error(
'PersonData Creation in DB Failed {id = ' + str(p_id) +
'}', e)
except ValidationError as e:
response_data['status'] = False
response_data['code'] = 400
response_data['message'] = e.args[0]
logger.error(
'PersonData Validation Failed in /api/new ' + str(e.args[0]),
e)
except JSONDecodeError as e:
response_data['status'] = False
response_data['code'] = 400
response_data['message'] = "JSON Error = " + str(e.args[0])
logger.error('JSON Decode Error in /api/new', e)
else:
response_data['status'] = False
response_data['code'] = 400 # Bad Request
response_data['message'] = "Method not supported"
response_data['data'] = []
logger.error('Method not Supported in /api/new')
return JsonResponse(response_data, json_dumps_params=JSON_PARAMS)
def registration(request):
if request.user.is_authenticated():
return HttpResponseRedirect('/profile/')
if request.method == 'POST':
form = RegistrationForm(request.POST)
if form.is_valid():
user = User.objects.create_user(
username=form.cleaned_data['email'],
email=form.cleaned_data['email'],
password=form.cleaned_data['password'],
)
firstname = form.cleaned_data['firstname']
lastname = form.cleaned_data['lastname']
user.first_name = firstname
user.last_name = lastname
name = '%s %s' % (user.first_name, user.last_name)
a = request.session['classification']
p = PersonClassification.objects.get(slug=a)
slug = str(user.id) + str(datetime.now().day) + str(
datetime.now().year)
skills = form.cleaned_data['skills'],
person = Person(
person=user,
person_class=p,
is_verified=False,
is_active=False,
address=form.cleaned_data['address'],
birthday=form.cleaned_data['birthday'],
firstname=form.cleaned_data['firstname'],
lastname=form.cleaned_data['lastname'],
created_at=datetime.now(),
contactno=form.cleaned_data['contactno'],
classification=a,
)
user.save()
person.save()
person.skills = skills
if a == 'poster':
template = 'register_poster.html'
else:
template = 'register_doer.html'
'''
send verification instructions to user
'''
subject = 'Welcome to Tulong.ph'
message = 'Hi %s we from tulong.ph, gladly welcomes you. Please reply to this email if you have any questions.' % person.firstname
to = person.person.email
email_from = '*****@*****.**'
send_mail(subject, message, email_from, [to], fail_silently=False)
'''
send the new user to the admin
'''
subject = 'new user'
message = 'user %s registered' % person.firstname
to = email_from
send_mail(subject, message, email_from, [to], fail_silently=False)
return render_to_response(template, {'status': person},
context_instance=RequestContext(request))
else:
return render_to_response(
'register.html', {
'form': form,
'test': request.session['classification'],
},
context_instance=RequestContext(request))
else:
''' user is not submitting the form, show them a blank registration form '''
form = RegistrationForm()
context = {
'form': form,
'test': request.session['classification'],
}
return render_to_response('register.html',
context,
context_instance=RequestContext(request))
def test_find_person_common_surname_initials_without_dots(self):
person = Person(forename='Jan', surname='Vries', initials='JM')
tkperson = openkamer.parliament.find_tkapi_person(person)
self.assertEqual(tkperson.achternaam, person.surname)
def main(options):
"""
Update Person and PersonRole models.
Do safe update: touch only those records
which have been changed.
"""
BASE_PATH = CONGRESS_LEGISLATORS_PATH
SRC_FILES = ['legislators-current', 'legislators-historical', 'legislators-social-media', 'executive'] # order matters
for p in SRC_FILES:
f = BASE_PATH + p + ".yaml"
if not File.objects.is_changed(f) and not options.force:
log.info('File %s was not changed' % f)
else:
# file modified...
break
else:
# no 'break' ==> no files modified
return
# Start parsing.
had_error = False
# Get combined data.
legislator_data = { }
leg_id_map = { }
for p in SRC_FILES:
log.info('Opening %s...' % p)
f = BASE_PATH + p + ".yaml"
y = yaml_load(f)
for m in y:
if p != 'legislators-social-media':
govtrack_id = m["id"].get("govtrack")
# For the benefit of the social media file, make a mapping of IDs.
for k, v in m["id"].items():
if type(v) != list:
leg_id_map[(k,v)] = govtrack_id
else:
# GovTrack IDs are not always listed in this file.
govtrack_id = None
for k, v in m["id"].items():
if type(v) != list and (k, v) in leg_id_map:
govtrack_id = leg_id_map[(k,v)]
break
if not govtrack_id:
print "No GovTrack ID:"
pprint.pprint(m)
had_error = True
continue
if govtrack_id not in legislator_data:
legislator_data[govtrack_id] = m
elif p == "legislators-social-media":
legislator_data[govtrack_id]["social"] = m["social"]
elif p == "executive":
legislator_data[govtrack_id]["terms"].extend( m["terms"] )
else:
raise ValueError("Duplication in an unexpected way (%d, %s)." % (govtrack_id, p))
person_processor = PersonProcessor()
role_processor = PersonRoleProcessor()
existing_persons = set(Person.objects.values_list('pk', flat=True))
processed_persons = set()
created_persons = set()
progress = Progress(total=len(legislator_data))
log.info('Processing persons')
for node in legislator_data.values():
# Wrap each iteration in try/except
# so that if some node breaks the parsing process
# then other nodes could be parsed
try:
person = person_processor.process(Person(), node)
# Create cached name strings. This is done again later
# after the roles are updated.
person.set_names()
# Now try to load the person with such ID from
# database. If found it then just update it
# else create new Person object
try:
ex_person = Person.objects.get(pk=person.pk)
if person_processor.changed(ex_person, person) or options.force:
# If the person has PK of existing record,
# coming in via the YAML-specified GovTrack ID,
# then Django ORM will update existing record
if not options.force:
log.warn("Updated %s" % person)
person.save()
except Person.DoesNotExist:
created_persons.add(person.pk)
person.save()
log.warn("Created %s" % person)
processed_persons.add(person.pk)
# Process roles of the person
roles = list(PersonRole.objects.filter(person=person))
existing_roles = set(PersonRole.objects.filter(person=person).values_list('pk', flat=True))
processed_roles = set()
role_list = []
for role in node['terms']:
role = role_processor.process(PersonRole(), role)
role.person = person
role.current = role.startdate <= datetime.now().date() and role.enddate >= datetime.now().date() # \
#and CURRENT_CONGRESS in role.congress_numbers()
# Scan for most recent leadership role within the time period of this term,
# which isn't great for Senators because it's likely it changed a few times
# within a term, especially if there was a party switch.
role.leadership_title = None
for leadership_node in node.get("leadership_roles", []):
# must match on date and chamber
if leadership_node["start"] >= role.enddate.isoformat(): continue # might start on the same day but is for the next Congress
if "end" in leadership_node and leadership_node["end"] <= role.startdate.isoformat(): continue # might start on the same day but is for the previous Congress
if leadership_node["chamber"] != RoleType.by_value(role.role_type).congress_chamber.lower(): continue
role.leadership_title = leadership_node["title"]
# Try to match this role with one already in the database.
# First search for an exact match on type/start/end.
ex_role = None
for r in roles:
if role.role_type == r.role_type and r.startdate == role.startdate and r.enddate == role.enddate:
ex_role = r
break
# Otherwise match on type/start only.
if not ex_role:
for r in roles:
if role.role_type == r.role_type and r.startdate == role.startdate:
ex_role = r
break
if ex_role:
# These roles correspond.
processed_roles.add(ex_role.id)
role.id = ex_role.id
if role_processor.changed(ex_role, role) or options.force:
role.save()
role_list.append(role)
if not options.force:
log.warn("Updated %s" % role)
roles.remove(ex_role) # don't need to try matching this to any other node
else:
# Didn't find a matching role.
if len([r for r in roles if r.role_type == role.role_type]) > 0:
print role, "is one of these?"
for ex_role in roles:
print "\t", ex_role
raise Exception("There is an unmatched role.")
log.warn("Created %s" % role)
role.save()
role_list.append(role)
# create the events for the roles after all have been loaded
# because we don't create events for ends of terms and
# starts of terms that are adjacent.
if not options.disable_events:
for i in xrange(len(role_list)):
role_list[i].create_events(
role_list[i-1] if i > 0 else None,
role_list[i+1] if i < len(role_list)-1 else None
)
removed_roles = existing_roles - processed_roles
for pk in removed_roles:
pr = PersonRole.objects.get(pk=pk)
print pr.person.id, pr
raise ValueError("Deleted role??")
log.warn("Deleted %s" % pr)
pr.delete()
# The name can't be determined until all of the roles are set. If
# it changes, re-save. Unfortunately roles are cached so this actually
# doesn't work yet. Re-run the parser to fix names.
nn = (person.name, person.sortname)
if hasattr(person, "role"): delattr(person, "role") # clear the cached info
person.set_names()
if nn != (person.name, person.sortname):
log.warn("%s is now %s." % (nn[0], person.name))
person.save()
except Exception, ex:
# Catch unexpected exceptions and log them
pprint.pprint(node)
log.error('', exc_info=ex)
had_error = True
progress.tick()
def test_find_person_bijsterveldt(self):
person = Person(forename='Marja',
surname='Bijsterveldt',
initials='J.M.')
tkperson = openkamer.parliament.find_tkapi_person(person)
self.assertEqual(tkperson.achternaam, 'Bijsterveldt-Vliegenthart')
def main(options):
"""
Update Person and PersonRole models.
Do safe update: touch only those records
which have been changed.
"""
BASE_PATH = CONGRESS_LEGISLATORS_PATH
SRC_FILES = [
'legislators-current', 'legislators-historical',
'legislators-social-media', 'executive'
] # order matters
for p in SRC_FILES:
f = BASE_PATH + p + ".yaml"
if not File.objects.is_changed(f) and not options.force:
log.info('File %s was not changed' % f)
else:
# file modified...
break
else:
# no 'break' ==> no files modified
return
# Start parsing.
had_error = False
# Get combined data.
legislator_data = {}
leg_id_map = {}
for p in SRC_FILES:
log.info('Opening %s...' % p)
f = BASE_PATH + p + ".yaml"
y = yaml_load(f)
for m in y:
if p == "legislators-current":
# We know all terms but the last are non-current and the last is.
for r in m["terms"]:
r["current"] = False
m["terms"][-1]["current"] = True
elif p == "legislators-historical":
# We know all terms are non-current.
for r in m["terms"]:
r["current"] = False
if p != 'legislators-social-media':
govtrack_id = m["id"].get("govtrack")
# For the benefit of the social media file, make a mapping of IDs.
for k, v in m["id"].items():
if type(v) != list:
leg_id_map[(k, v)] = govtrack_id
else:
# GovTrack IDs are not always listed in this file.
govtrack_id = None
for k, v in m["id"].items():
if type(v) != list and (k, v) in leg_id_map:
govtrack_id = leg_id_map[(k, v)]
break
if not govtrack_id:
print("No GovTrack ID:")
pprint.pprint(m)
had_error = True
continue
if govtrack_id not in legislator_data:
legislator_data[govtrack_id] = m
elif p == "legislators-social-media":
legislator_data[govtrack_id]["social"] = m["social"]
elif p == "executive":
legislator_data[govtrack_id]["terms"].extend(m["terms"])
else:
raise ValueError("Duplication in an unexpected way (%d, %s)." %
(govtrack_id, p))
person_processor = PersonProcessor()
role_processor = PersonRoleProcessor()
existing_persons = set(Person.objects.values_list('pk', flat=True))
processed_persons = set()
created_persons = set()
progress = Progress(total=len(legislator_data))
log.info('Processing persons')
for node in legislator_data.values():
# Wrap each iteration in try/except
# so that if some node breaks the parsing process
# then other nodes could be parsed
try:
person = person_processor.process(Person(), node)
# Create cached name strings. This is done again later
# after the roles are updated.
person.set_names()
# Now try to load the person with such ID from
# database. If found it then just update it
# else create new Person object
try:
ex_person = Person.objects.get(pk=person.pk)
if person_processor.changed(ex_person,
person) or options.force:
# If the person has PK of existing record,
# coming in via the YAML-specified GovTrack ID,
# then Django ORM will update existing record
if not options.force:
log.warn("Updated %s" % person)
person.save()
except Person.DoesNotExist:
created_persons.add(person.pk)
person.save()
log.warn("Created %s" % person)
processed_persons.add(person.pk)
# Parse all of the roles.
new_roles = []
for termnode in node['terms']:
role = role_processor.process(PersonRole(), termnode)
role.person = person
role.extra = filter_yaml_term_structure(
termnode) # copy in the whole YAML structure
# Is this role current? For legislators, same as whether it came from legislators-current, which eases Jan 3 transitions when we can't distinguish by date.
if "current" in termnode:
role.current = termnode["current"]
# But executives...
else:
now = datetime.now().date()
role.current = role.startdate <= now and role.enddate >= now
# Because of date overlaps at noon transition dates, ensure that only the last term that covers
# today is current --- reset past roles to not current. Doesn't handle turning off retirning people tho.
for r in new_roles:
r.current = False
# Scan for most recent leadership role within the time period of this term,
# which isn't great for Senators because it's likely it changed a few times
# within a term, especially if there was a party switch.
role.leadership_title = None
for leadership_node in node.get("leadership_roles", []):
# must match on date and chamber
if leadership_node["start"] >= role.enddate.isoformat():
continue # might start on the same day but is for the next Congress
if "end" in leadership_node and leadership_node[
"end"] <= role.startdate.isoformat():
continue # might start on the same day but is for the previous Congress
if leadership_node["chamber"] != RoleType.by_value(
role.role_type).congress_chamber.lower():
continue
role.leadership_title = leadership_node["title"]
new_roles.append(role)
# Try matching the new roles to existing db records. Since we don't have a primry key
# in the source data, we have to match on the record values. But because of errors in data,
# term start/end dates can change, so matching has to be a little fuzzy.
existing_roles = list(PersonRole.objects.filter(person=person))
matches = []
def run_match_rule(rule):
import itertools
for new_role, existing_role in itertools.product(
new_roles, existing_roles):
if new_role not in new_roles or existing_role not in existing_roles:
continue # already matched on a previous iteration
if new_role.role_type != existing_role.role_type: continue
if new_role.state != existing_role.state: continue
if rule(new_role, existing_role):
matches.append((new_role, existing_role))
new_roles.remove(new_role)
existing_roles.remove(existing_role)
# First match exactly, then exact on just one date, then on contractions and expansions.
run_match_rule(lambda new_role, existing_role: new_role.startdate
== existing_role.startdate and new_role.enddate ==
existing_role.enddate)
run_match_rule(lambda new_role, existing_role: new_role.startdate
== existing_role.startdate or new_role.enddate ==
existing_role.enddate)
run_match_rule(lambda new_role, existing_role: new_role.startdate
>= existing_role.startdate and new_role.enddate <=
existing_role.enddate)
run_match_rule(lambda new_role, existing_role: new_role.startdate
<= existing_role.startdate and new_role.enddate >=
existing_role.enddate)
# Update the database entries that correspond with records in the data file.
did_update_any = False
for new_role, existing_role in matches:
new_role.id = existing_role.id
if role_processor.changed(existing_role,
new_role) or options.force:
new_role.save()
did_update_any = True
if not options.force:
log.warn("Updated %s" % new_role)
# If we have mutliple records on disk that didn't match and multiple records in the database
# that didn't match, then we don't know how to align them.
if len(new_roles) > 0 and len(existing_roles) > 0:
print(new_roles)
print(existing_roles)
raise Exception("There is an unmatched role.")
# Otherwise if there are any unmatched new roles, we can just add them.
for role in new_roles:
log.warn("Created %s" % role)
role.save()
did_update_any = True
# And likewise for any existing roles that are left over.
for pr in existing_roles:
print(pr.person.id, pr)
raise ValueError("Deleted role??")
log.warn("Deleted %s" % pr)
pr.delete()
if did_update_any and not options.disable_events:
# Create the events for the roles after all have been loaded
# because we don't create events for ends of terms and
# starts of terms that are adjacent. Refresh the list to get
# the roles in order.
role_list = list(
PersonRole.objects.filter(
person=person).order_by('startdate'))
for i in range(len(role_list)):
role_list[i].create_events(
role_list[i - 1] if i > 0 else None,
role_list[i + 1] if i < len(role_list) - 1 else None)
# The name can't be determined until all of the roles are set. If
# it changes, re-save. Unfortunately roles are cached so this actually
# doesn't work yet. Re-run the parser to fix names.
nn = (person.name, person.sortname)
if hasattr(person, "role"):
delattr(person, "role") # clear the cached info
person._most_recent_role = None # clear cache here too
person.set_names()
if nn != (person.name, person.sortname):
log.warn("%s is now %s." % (nn[0], person.name))
person.save()
except Exception as ex:
# Catch unexpected exceptions and log them
pprint.pprint(node)
log.error('', exc_info=ex)
had_error = True
progress.tick()
log.info('Processed persons: %d' % len(processed_persons))
log.info('Created persons: %d' % len(created_persons))
if not had_error:
# Remove person which were not found in XML file
removed_persons = existing_persons - processed_persons
for pk in removed_persons:
p = Person.objects.get(pk=pk)
if p.roles.all().count() > 0:
log.warn("Missing? Deleted? %d: %s" % (p.id, p))
else:
log.warn("Deleting... %d: %s (remember to prune_index!)" %
(p.id, p))
raise Exception("Won't delete!")
p.delete()
log.info('Missing/deleted persons: %d' % len(removed_persons))
# Mark the files as processed.
for p in SRC_FILES:
f = BASE_PATH + p + ".yaml"
File.objects.save_file(f)
update_twitter_list()
def test_find_person_ozturk(self):
person = Person(forename='Selçuk', surname='Öztürk', initials='S')
tkperson = openkamer.parliament.find_tkapi_person(person)
self.assertEqual(tkperson.achternaam, 'Öztürk')
