def test_find_no_results(self): person = Person(forename='Willem Caspar', surname_prefix='de', surname='Jonge', initials='J.') tkperson = openkamer.parliament.find_tkapi_person(person) self.assertIsNone(tkperson)
def test_find_person_no_results_jean(self): person = Person(forename='Jean Baptiste Anne', surname_prefix='de', surname='Gerlache de Biourge', initials='J.B.A.') tkperson = openkamer.parliament.find_tkapi_person(person) self.assertIsNone(tkperson)
def signup(request): data = request.POST if 'username' not in data or 'password' not in data or 'email' not in data or data[ 'username'] == '' or data['password'] == '' or data['email'] == '': return utils.createError('Invalid input form.') _username = data.get('username', '') _email = data.get('email', '') _password = data.get('password', '') #--validation-- if Person.objects.filter(username=_username): return utils.createError('Username has been taken.') if Person.objects.filter(email=_email): return utils.createError('Email has been taken.') if len(_password) < 8: return utils.createError('Length of password must be at least 8.') #--create person-- _salt = utils.getRandomString() _hashed_password = utils.hashPassword(_password, _salt) person = Person(username=_username, salt=_salt, password=_hashed_password, email=_email) person.save() return utils.createResponse({ 'action': 'Sign up successfully', 'username': person.username, 'email': person.email })
def test_person_str(): p = Person( first_name='a', last_name='b', birth_date='1969-01-01', ) assert str(p) == 'a b 1969-01-01'
def load_authors(): users_mapping = load_users_mapping() for user_api in users_data_stream('users'): mapped_user = users_mapping[str(user_api['id'])] if not mapped_user['duplicate'] and not mapped_user['delete']: mapped_user_title = '{0} {1}'.format(mapped_user['first_name'], mapped_user['last_name']) mapped_user_slug = slugify(mapped_user_title) print("id is: ") print(user_api['id']) db_user = Person.objects.filter(slug=mapped_user_slug).first() role_info = get_role_info(user_api['roles']) if not db_user and mapped_user_slug: db_user = Person( search_description='', seo_title='', show_in_menus=False, slug=mapped_user_slug, title=mapped_user_title, first_name=mapped_user['first_name'], last_name=mapped_user['last_name'], position_at_new_america=user_api['title'], role=role_info['role_title'], email=user_api['email'], expert=role_info['expert'], depth=4, profile_image=download_image( user_api['image'], mapped_user_slug + "_person_image.jpeg"), short_bio=user_api['short_bio'], long_bio=user_api['long_bio'], ) our_people_page.add_child(instance=db_user) db_user.save() print("New user") print(db_user) elif db_user and mapped_user_slug: db_user.search_description = '' db_user.seo_title = '' db_user.show_in_menus = False db_user.slug = mapped_user_slug db_user.title = mapped_user_title db_user.first_name = mapped_user['first_name'] db_user.last_name = mapped_user['last_name'] db_user.position_at_new_america = user_api['title'] db_user.role = role_info['role_title'] db_user.email = user_api['email'] db_user.expert = role_info['expert'] db_user.depth = 4 db_user.short_bio = user_api['short_bio'] db_user.long_bio = user_api['long_bio'] db_user.profile_image = download_image( user_api['image'], mapped_user_slug + "_person_image.jpeg") print("Existing user") print(db_user) db_user.save()
def main(): from person.models import Person with open("1.json") as jsonfile: json_data = json.load(jsonfile) PersonList = [] for dicobj in json_data: person = Person(volume=dicobj["volume"], postdate=dicobj["postdate"], gender=dicobj["gender"],idnum=dicobj["idnum"],city=dicobj["city"],dec=dicobj["dec"],image_urls=dicobj["image_urls"][0] ) PersonList.append(person) Person.objects.bulk_create(PersonList)
def create_person(person_data={}): people_page = OurPeoplePage.objects.first() if people_page == None: home_page = HomePage.objects.first() people_page = home_page.add_child(instance=OurPeoplePage( title='Our People')) person = people_page.add_child(instance=Person( **PostFactory.person_data(**person_data))) return person
def create_person(self, data): fields_data, m2m_fields_data = self.split_data(data) person = Person(**fields_data) # person.full_clean() # full_clean called in save method person.save() for field_name, id_list in m2m_fields_data.items(): m2m_data = data.get(field_name) if m2m_data: getattr(person, field_name).set(id_list) return person
def setUp(self): self.login() site = Site.objects.get() page = Page.get_first_root_node() home = HomePage(title='New America') home_page = page.add_child(instance=home) site.root_page = home site.save() # People objects to test template tags # that create byline and determine author order program_page_1 = home_page.add_child( instance=Program(title='OTI', name='OTI', slug='oti', description='OTI', location=False, depth=3)) program_page_1.save() our_people_page = home_page.add_child(instance=OurPeoplePage( title='Our People', depth=3, )) our_people_page.save() self.first_person = Person( title='First Person', slug='first-person', first_name='first', last_name='person', role='Central Staff', depth=4, ) our_people_page.add_child(instance=self.first_person) # Using policy papers to test the other post types all_policy_papers_home_page = home_page.add_child( instance=AllPolicyPapersHomePage(title="Policy Papers")) program_policy_papers_page = program_page_1.add_child( instance=ProgramPolicyPapersPage(title='OTI Policy Papers', slug='oti-policy-papers')) self.policy_paper = PolicyPaper(title='Policy Paper 1', slug='policy-paper-1', date='2016-06-15', depth=5) program_policy_papers_page.add_child(instance=self.policy_paper) self.policy_paper.save() PostAuthorRelationship(author=self.first_person, post=self.policy_paper).save() all_policy_papers_home_page.save()
def handle(self, *args, **kwargs): filename = kwargs['filename'] with open(filename, encoding='utf-8') as f: models = json.load(f) # Verify all data before Insertion, Fail if any one data is corrupted record_count = 0 for m in models: record_count = record_count + 1 valid_status = person_validation.validate_add(m) if not valid_status[0]: raise ValidationError(valid_status[1]) for person in models: person_data = person.get('data')[0] enabled = person_data.get('enabled') guid = person_data.get('guid') first_name = person.get('first_name') last_name = person.get('last_name') city = person.get('city') p_id = person.get('id') person_data_model_obj = PersonData(enabled=enabled, guid=guid) save_flag = True try: person_data_model_obj.save() except ValueError as e: save_flag = False person_data_model_obj.delete() self.stderr.write( 'PersonData Creation in DB Failed {id = ' + str(p_id) + '}', e) try: if save_flag: person_model = Person(id=p_id, first_name=first_name, last_name=last_name, city=city, data=person_data_model_obj) person_model.save(force_insert=True) except IntegrityError as e: person_data_model_obj.delete() self.stderr.write( 'Record Creation in DB Failed {id = ' + str(p_id) + '}', e) except ValueError as e: person_data_model_obj.delete() self.stderr.write( 'Record Creation in DB Failed {id = ' + str(p_id) + '}', e)
def create_person(self): person = Person() person.sex = self.__generate_sex() person.birth_date = self.__generate_date() person.first_name = self.__generate_name(person.sex) person.last_name = self.__generate_surname(person.sex) person.pesel = self.__generate_pesel(person.birth_date, person.sex) person.email = self.__generate_email(person.first_name, person.last_name) person.phone = self.__generate_phone() person.password = self.__generate_password() logging.basicConfig(level=logging.DEBUG) logging.info( f"Created person {person.first_name} {person.last_name} with pesel {person.pesel}" ) return person
def test_person_validate_nirs(nir, expected): person = Person( first_name='a', last_name='b', nir=nir, birth_date='1969-01-01', email="*****@*****.**", ) if not expected: person.full_clean() assert person.nir == str(nir) else: with pytest.raises(ValidationError) as raised: person.full_clean() assert len(raised.value.error_dict['nir']) == expected
def make_person(name, building, relation, address=None, city=None): #now associate applicant with building: #first find/make person people = Person.objects.filter(city=city).filter(name=name) person = None #check if a previous building object in the db exists if people.exists(): person = people[0] print "Already had Person: %s" % person.name else: #if not, #CREATE A NEW PERSON OBJECT HERE person = Person() person.name = name if city: person.city = city if address: person.address = address person.save() #then find/make association: bpeople = BuildingPerson.objects.filter(building=building).filter(person=person) bperson = None #check if a previous building_person object in the db exists if bpeople.exists(): bperson = bpeople[0] print "Already had BuildingPerson: %s with: %s" % (bperson.person.name, bperson.building.address) else: #if not, #CREATE A NEW BUILDING PERSON OBJECT HERE bperson = BuildingPerson() bperson.person = person bperson.building = building bperson.relation = relation bperson.save() return (person, bperson)
def personPage(request): form = PersonCreateForm context = {} context["form"] = form if request.method == "POST": form = PersonCreateForm(request.POST) if form.is_valid(): personname = request.POST.get("personname") age = request.POST.get("age") salary = request.POST.get("salary") desig = request.POST.get("desig") print(personname, ",", age, ",", salary, ",", desig) person = Person(personname=personname, age=age, salary=salary, desig=desig) person.save() return redirect("getdetails") return render(request, "person/person.html", context)
def __call__(self, request, *args, **kwargs): """ Due to security concerns, do not overload this function or any of its subclass overrides. """ self.request = request self.args = args self.kwargs = kwargs if self.request.user.is_authenticated(): self.user = self.request.user try: self.person = Person.objects.get(user=self.user) except Person.DoesNotExist: self.person = Person(user=self.user) self.person.save() return self.allow(*args, **kwargs) else: return self.deny()
def create_person(request): # redirect to sign in not logged in if 'id' not in request.session.keys(): return redirect('/sign_in/') # if this is a POST request we need to process the form dat if request.method == 'POST': # create a form instance and populate it with data from the request: form = PersonForm(request.POST) # check whether it's valid: if form.is_valid(): form_data = form.cleaned_data current_user = User.objects.get(pk=request.user.id) new_row = Person( user=current_user, name=form_data['name'], message=form_data['message'], created=datetime.datetime.now(), start_date=form_data['start_date'], last_send=None, next_send=form_data['start_date'], interval=form_data['interval'], interval_type=form_data['interval_type'], snooze=0, snooze_interval=None, snooze_interval_type=None, snooze_last_send=None, snooze_next_send=None, in_deleted=0 ) new_row.save() return redirect('/view_person_list/') else: form = PersonForm() return render(request, 'person/create_person.html', {'form': form})
def test_find_person_haegen(self): person = Person(forename='Melanie', surname='Schultz van Haegen', initials='M.H.') tkperson = openkamer.parliament.find_tkapi_person(person) self.assertIsNotNone(tkperson)
def test_find_person_no_results_dekker(self): """Should not be confused with Sander Dekker.""" person = Person(forename='Suzanne', surname='Dekker', initials='S.') tkperson = openkamer.parliament.find_tkapi_person(person) self.assertIsNone(tkperson)
def test_find_person_wrong_initials(self): person = Person(surname='Wiersma', initials='H.') tkperson = openkamer.parliament.find_tkapi_person(person) self.assertIsNone(tkperson)
def test_find_person_arissen(self): person = Person(surname='Merel Arissen', forename='Femke', initials='F.M.') tkperson = openkamer.parliament.find_tkapi_person(person) self.assertEqual(tkperson.achternaam, 'Kooten-Arissen')
def test_find_person_pater_postma(self): person = Person(surname='Pater-Postma', forename='Wytske', initials='W.L.') tkperson = openkamer.parliament.find_tkapi_person(person) self.assertEqual(tkperson.achternaam, 'Postma')
def _create_person(self, person_data): """Returns a newly created Person object from `person_data`.""" person = Person(**person_data) person.save() return person
def PersonRegistration(request, slug): if slug == 'doer': is_doer = True is_poster = False elif slug == 'poster': is_poster = True is_doer = False if request.user.is_authenticated(): return HttpResponseRedirect('/profile/') if request.method == 'POST': form = RegistrationForm(request.POST) if form.is_valid(): user = User.objects.create_user( username=form.cleaned_data['email'], email=form.cleaned_data['email'], password=form.cleaned_data['password'], ) firstname = form.cleaned_data['firstname'] lastname = form.cleaned_data['lastname'] user.first_name = firstname user.last_name = lastname name = '%s %s' % (user.first_name, user.last_name) p = PersonClassification.objects.get(slug=slug) a = slug slug = str(user.id) + str(datetime.now().day) + str( datetime.now().year) skills = form.cleaned_data['skills'] person = Person( person=user, person_class=p, is_verified=False, is_active=False, address=form.cleaned_data['address'], birthday=form.cleaned_data['birthday'], firstname=form.cleaned_data['firstname'], lastname=form.cleaned_data['lastname'], created_at=datetime.now(), contactno=form.cleaned_data['contactno'], #classification = a, ) user.save() person.save() person.skills = skills if person.person_class.__str__() == 'poster': template = 'register_poster.html' else: template = 'register_doer.html' ''' send verification instructions to user ''' subject = 'Welcome to Tulong.ph' message = 'Hi %s we from tulong.ph, gladly welcomes you. Please reply to this email if you have any questions.' % person.firstname to = person.person.email #send_mail(subject,message,'*****@*****.**',[to],fail_silently=False) name = "%s %s" % (person.firstname, person.lastname) context = {'full_name': name, 'person': person} text_content = render_to_string('email/register.html', context) html_content = render_to_string('email/register_html.html', context) msg = EmailMultiAlternatives(subject, text_content, settings.EMAIL_HOST_USER, [to]) msg.attach_alternative(html_content, "text/html") msg.send() #send_mail(subject, email_content, # settings.EMAIL_HOST_USER, [to],fail_silently=False) ''' send the new user to the admin ''' person_classification = person.person_class.__str__() subject = 'New User as %s' % (person_classification) message = 'User %s Registered as %s ' % (person.person.email, person_classification) to = '*****@*****.**' send_mail(subject, message, '*****@*****.**', [to], fail_silently=False) return render_to_response(template, { 'status': person, 'is_doer': is_doer, 'is_poster': is_poster, }, context_instance=RequestContext(request)) #return HttpResponseRedirect('/login/') else: return render_to_response('register.html', { 'form': form, 'is_doer': is_doer, 'is_poster': is_poster, }, context_instance=RequestContext(request)) else: ''' user is not submitting the form, show them a blank registration form ''' form = RegistrationForm() context = { 'form': form, 'is_doer': is_doer, 'is_poster': is_poster, } return render_to_response('register.html', context, context_instance=RequestContext(request))
def add_person_view(request): """ Function to add new Person """ # Predefined structure for response object. response_data = {'status': '', 'code': None, 'message': None, 'data': []} logger.info('Person Creation Endpoint Accessed : /api/new') if request.method == 'PUT': try: models = json.loads(request.body) # Verify all data before Insertion, Fail if any one data is corrupted record_count = 0 for m in models: record_count = record_count + 1 valid_status = person_validation.validate_add(m) if not valid_status[0]: raise ValidationError(valid_status[1]) for person in models: person_data = person.get('data')[0] enabled = person_data.get('enabled') guid = person_data.get('guid') first_name = person.get('first_name') last_name = person.get('last_name') city = person.get('city') p_id = person.get('id') person_data_model_obj = PersonData(enabled=enabled, guid=guid) save_flag = True try: person_data_model_obj.save() except ValueError as e: save_flag = False person_data_model_obj.delete() response_data['status'] = False response_data['code'] = 500 response_data[ 'message'] = 'PersonData Creation in DB Failed {id = ' + str( p_id) + '}' logger.error('PersonData Creation in DB Failed {id = ' + str(p_id) + '}') try: if save_flag: person_model = Person(id=p_id, first_name=first_name, last_name=last_name, city=city, data=person_data_model_obj) person_model.save(force_insert=True) response_data['status'] = True response_data['code'] = 200 response_data[ 'message'] = 'Data Creation Success, id=' + str( p_id) response_data['data'] = [] logger.info( 'Person Record Created Successfully : id=' + str(p_id)) except IntegrityError as e: person_data_model_obj.delete() response_data['status'] = False response_data['code'] = 500 response_data[ 'message'] = 'Record Creation in DB Failed {id = ' + str( p_id) + '}' logger.error( 'PersonData Creation in DB Failed {id = ' + str(p_id) + '}', e) except ValueError as e: person_data_model_obj.delete() response_data['status'] = False response_data['code'] = 500 response_data[ 'message'] = 'Record Creation in DB Failed {id = ' + str( p_id) + '}' logger.error( 'PersonData Creation in DB Failed {id = ' + str(p_id) + '}', e) except ValidationError as e: response_data['status'] = False response_data['code'] = 400 response_data['message'] = e.args[0] logger.error( 'PersonData Validation Failed in /api/new ' + str(e.args[0]), e) except JSONDecodeError as e: response_data['status'] = False response_data['code'] = 400 response_data['message'] = "JSON Error = " + str(e.args[0]) logger.error('JSON Decode Error in /api/new', e) else: response_data['status'] = False response_data['code'] = 400 # Bad Request response_data['message'] = "Method not supported" response_data['data'] = [] logger.error('Method not Supported in /api/new') return JsonResponse(response_data, json_dumps_params=JSON_PARAMS)
def registration(request): if request.user.is_authenticated(): return HttpResponseRedirect('/profile/') if request.method == 'POST': form = RegistrationForm(request.POST) if form.is_valid(): user = User.objects.create_user( username=form.cleaned_data['email'], email=form.cleaned_data['email'], password=form.cleaned_data['password'], ) firstname = form.cleaned_data['firstname'] lastname = form.cleaned_data['lastname'] user.first_name = firstname user.last_name = lastname name = '%s %s' % (user.first_name, user.last_name) a = request.session['classification'] p = PersonClassification.objects.get(slug=a) slug = str(user.id) + str(datetime.now().day) + str( datetime.now().year) skills = form.cleaned_data['skills'], person = Person( person=user, person_class=p, is_verified=False, is_active=False, address=form.cleaned_data['address'], birthday=form.cleaned_data['birthday'], firstname=form.cleaned_data['firstname'], lastname=form.cleaned_data['lastname'], created_at=datetime.now(), contactno=form.cleaned_data['contactno'], classification=a, ) user.save() person.save() person.skills = skills if a == 'poster': template = 'register_poster.html' else: template = 'register_doer.html' ''' send verification instructions to user ''' subject = 'Welcome to Tulong.ph' message = 'Hi %s we from tulong.ph, gladly welcomes you. Please reply to this email if you have any questions.' % person.firstname to = person.person.email email_from = '*****@*****.**' send_mail(subject, message, email_from, [to], fail_silently=False) ''' send the new user to the admin ''' subject = 'new user' message = 'user %s registered' % person.firstname to = email_from send_mail(subject, message, email_from, [to], fail_silently=False) return render_to_response(template, {'status': person}, context_instance=RequestContext(request)) else: return render_to_response( 'register.html', { 'form': form, 'test': request.session['classification'], }, context_instance=RequestContext(request)) else: ''' user is not submitting the form, show them a blank registration form ''' form = RegistrationForm() context = { 'form': form, 'test': request.session['classification'], } return render_to_response('register.html', context, context_instance=RequestContext(request))
def test_find_person_common_surname_initials_without_dots(self): person = Person(forename='Jan', surname='Vries', initials='JM') tkperson = openkamer.parliament.find_tkapi_person(person) self.assertEqual(tkperson.achternaam, person.surname)
def main(options): """ Update Person and PersonRole models. Do safe update: touch only those records which have been changed. """ BASE_PATH = CONGRESS_LEGISLATORS_PATH SRC_FILES = ['legislators-current', 'legislators-historical', 'legislators-social-media', 'executive'] # order matters for p in SRC_FILES: f = BASE_PATH + p + ".yaml" if not File.objects.is_changed(f) and not options.force: log.info('File %s was not changed' % f) else: # file modified... break else: # no 'break' ==> no files modified return # Start parsing. had_error = False # Get combined data. legislator_data = { } leg_id_map = { } for p in SRC_FILES: log.info('Opening %s...' % p) f = BASE_PATH + p + ".yaml" y = yaml_load(f) for m in y: if p != 'legislators-social-media': govtrack_id = m["id"].get("govtrack") # For the benefit of the social media file, make a mapping of IDs. for k, v in m["id"].items(): if type(v) != list: leg_id_map[(k,v)] = govtrack_id else: # GovTrack IDs are not always listed in this file. govtrack_id = None for k, v in m["id"].items(): if type(v) != list and (k, v) in leg_id_map: govtrack_id = leg_id_map[(k,v)] break if not govtrack_id: print "No GovTrack ID:" pprint.pprint(m) had_error = True continue if govtrack_id not in legislator_data: legislator_data[govtrack_id] = m elif p == "legislators-social-media": legislator_data[govtrack_id]["social"] = m["social"] elif p == "executive": legislator_data[govtrack_id]["terms"].extend( m["terms"] ) else: raise ValueError("Duplication in an unexpected way (%d, %s)." % (govtrack_id, p)) person_processor = PersonProcessor() role_processor = PersonRoleProcessor() existing_persons = set(Person.objects.values_list('pk', flat=True)) processed_persons = set() created_persons = set() progress = Progress(total=len(legislator_data)) log.info('Processing persons') for node in legislator_data.values(): # Wrap each iteration in try/except # so that if some node breaks the parsing process # then other nodes could be parsed try: person = person_processor.process(Person(), node) # Create cached name strings. This is done again later # after the roles are updated. person.set_names() # Now try to load the person with such ID from # database. If found it then just update it # else create new Person object try: ex_person = Person.objects.get(pk=person.pk) if person_processor.changed(ex_person, person) or options.force: # If the person has PK of existing record, # coming in via the YAML-specified GovTrack ID, # then Django ORM will update existing record if not options.force: log.warn("Updated %s" % person) person.save() except Person.DoesNotExist: created_persons.add(person.pk) person.save() log.warn("Created %s" % person) processed_persons.add(person.pk) # Process roles of the person roles = list(PersonRole.objects.filter(person=person)) existing_roles = set(PersonRole.objects.filter(person=person).values_list('pk', flat=True)) processed_roles = set() role_list = [] for role in node['terms']: role = role_processor.process(PersonRole(), role) role.person = person role.current = role.startdate <= datetime.now().date() and role.enddate >= datetime.now().date() # \ #and CURRENT_CONGRESS in role.congress_numbers() # Scan for most recent leadership role within the time period of this term, # which isn't great for Senators because it's likely it changed a few times # within a term, especially if there was a party switch. role.leadership_title = None for leadership_node in node.get("leadership_roles", []): # must match on date and chamber if leadership_node["start"] >= role.enddate.isoformat(): continue # might start on the same day but is for the next Congress if "end" in leadership_node and leadership_node["end"] <= role.startdate.isoformat(): continue # might start on the same day but is for the previous Congress if leadership_node["chamber"] != RoleType.by_value(role.role_type).congress_chamber.lower(): continue role.leadership_title = leadership_node["title"] # Try to match this role with one already in the database. # First search for an exact match on type/start/end. ex_role = None for r in roles: if role.role_type == r.role_type and r.startdate == role.startdate and r.enddate == role.enddate: ex_role = r break # Otherwise match on type/start only. if not ex_role: for r in roles: if role.role_type == r.role_type and r.startdate == role.startdate: ex_role = r break if ex_role: # These roles correspond. processed_roles.add(ex_role.id) role.id = ex_role.id if role_processor.changed(ex_role, role) or options.force: role.save() role_list.append(role) if not options.force: log.warn("Updated %s" % role) roles.remove(ex_role) # don't need to try matching this to any other node else: # Didn't find a matching role. if len([r for r in roles if r.role_type == role.role_type]) > 0: print role, "is one of these?" for ex_role in roles: print "\t", ex_role raise Exception("There is an unmatched role.") log.warn("Created %s" % role) role.save() role_list.append(role) # create the events for the roles after all have been loaded # because we don't create events for ends of terms and # starts of terms that are adjacent. if not options.disable_events: for i in xrange(len(role_list)): role_list[i].create_events( role_list[i-1] if i > 0 else None, role_list[i+1] if i < len(role_list)-1 else None ) removed_roles = existing_roles - processed_roles for pk in removed_roles: pr = PersonRole.objects.get(pk=pk) print pr.person.id, pr raise ValueError("Deleted role??") log.warn("Deleted %s" % pr) pr.delete() # The name can't be determined until all of the roles are set. If # it changes, re-save. Unfortunately roles are cached so this actually # doesn't work yet. Re-run the parser to fix names. nn = (person.name, person.sortname) if hasattr(person, "role"): delattr(person, "role") # clear the cached info person.set_names() if nn != (person.name, person.sortname): log.warn("%s is now %s." % (nn[0], person.name)) person.save() except Exception, ex: # Catch unexpected exceptions and log them pprint.pprint(node) log.error('', exc_info=ex) had_error = True progress.tick()
def test_find_person_bijsterveldt(self): person = Person(forename='Marja', surname='Bijsterveldt', initials='J.M.') tkperson = openkamer.parliament.find_tkapi_person(person) self.assertEqual(tkperson.achternaam, 'Bijsterveldt-Vliegenthart')
def main(options): """ Update Person and PersonRole models. Do safe update: touch only those records which have been changed. """ BASE_PATH = CONGRESS_LEGISLATORS_PATH SRC_FILES = [ 'legislators-current', 'legislators-historical', 'legislators-social-media', 'executive' ] # order matters for p in SRC_FILES: f = BASE_PATH + p + ".yaml" if not File.objects.is_changed(f) and not options.force: log.info('File %s was not changed' % f) else: # file modified... break else: # no 'break' ==> no files modified return # Start parsing. had_error = False # Get combined data. legislator_data = {} leg_id_map = {} for p in SRC_FILES: log.info('Opening %s...' % p) f = BASE_PATH + p + ".yaml" y = yaml_load(f) for m in y: if p == "legislators-current": # We know all terms but the last are non-current and the last is. for r in m["terms"]: r["current"] = False m["terms"][-1]["current"] = True elif p == "legislators-historical": # We know all terms are non-current. for r in m["terms"]: r["current"] = False if p != 'legislators-social-media': govtrack_id = m["id"].get("govtrack") # For the benefit of the social media file, make a mapping of IDs. for k, v in m["id"].items(): if type(v) != list: leg_id_map[(k, v)] = govtrack_id else: # GovTrack IDs are not always listed in this file. govtrack_id = None for k, v in m["id"].items(): if type(v) != list and (k, v) in leg_id_map: govtrack_id = leg_id_map[(k, v)] break if not govtrack_id: print("No GovTrack ID:") pprint.pprint(m) had_error = True continue if govtrack_id not in legislator_data: legislator_data[govtrack_id] = m elif p == "legislators-social-media": legislator_data[govtrack_id]["social"] = m["social"] elif p == "executive": legislator_data[govtrack_id]["terms"].extend(m["terms"]) else: raise ValueError("Duplication in an unexpected way (%d, %s)." % (govtrack_id, p)) person_processor = PersonProcessor() role_processor = PersonRoleProcessor() existing_persons = set(Person.objects.values_list('pk', flat=True)) processed_persons = set() created_persons = set() progress = Progress(total=len(legislator_data)) log.info('Processing persons') for node in legislator_data.values(): # Wrap each iteration in try/except # so that if some node breaks the parsing process # then other nodes could be parsed try: person = person_processor.process(Person(), node) # Create cached name strings. This is done again later # after the roles are updated. person.set_names() # Now try to load the person with such ID from # database. If found it then just update it # else create new Person object try: ex_person = Person.objects.get(pk=person.pk) if person_processor.changed(ex_person, person) or options.force: # If the person has PK of existing record, # coming in via the YAML-specified GovTrack ID, # then Django ORM will update existing record if not options.force: log.warn("Updated %s" % person) person.save() except Person.DoesNotExist: created_persons.add(person.pk) person.save() log.warn("Created %s" % person) processed_persons.add(person.pk) # Parse all of the roles. new_roles = [] for termnode in node['terms']: role = role_processor.process(PersonRole(), termnode) role.person = person role.extra = filter_yaml_term_structure( termnode) # copy in the whole YAML structure # Is this role current? For legislators, same as whether it came from legislators-current, which eases Jan 3 transitions when we can't distinguish by date. if "current" in termnode: role.current = termnode["current"] # But executives... else: now = datetime.now().date() role.current = role.startdate <= now and role.enddate >= now # Because of date overlaps at noon transition dates, ensure that only the last term that covers # today is current --- reset past roles to not current. Doesn't handle turning off retirning people tho. for r in new_roles: r.current = False # Scan for most recent leadership role within the time period of this term, # which isn't great for Senators because it's likely it changed a few times # within a term, especially if there was a party switch. role.leadership_title = None for leadership_node in node.get("leadership_roles", []): # must match on date and chamber if leadership_node["start"] >= role.enddate.isoformat(): continue # might start on the same day but is for the next Congress if "end" in leadership_node and leadership_node[ "end"] <= role.startdate.isoformat(): continue # might start on the same day but is for the previous Congress if leadership_node["chamber"] != RoleType.by_value( role.role_type).congress_chamber.lower(): continue role.leadership_title = leadership_node["title"] new_roles.append(role) # Try matching the new roles to existing db records. Since we don't have a primry key # in the source data, we have to match on the record values. But because of errors in data, # term start/end dates can change, so matching has to be a little fuzzy. existing_roles = list(PersonRole.objects.filter(person=person)) matches = [] def run_match_rule(rule): import itertools for new_role, existing_role in itertools.product( new_roles, existing_roles): if new_role not in new_roles or existing_role not in existing_roles: continue # already matched on a previous iteration if new_role.role_type != existing_role.role_type: continue if new_role.state != existing_role.state: continue if rule(new_role, existing_role): matches.append((new_role, existing_role)) new_roles.remove(new_role) existing_roles.remove(existing_role) # First match exactly, then exact on just one date, then on contractions and expansions. run_match_rule(lambda new_role, existing_role: new_role.startdate == existing_role.startdate and new_role.enddate == existing_role.enddate) run_match_rule(lambda new_role, existing_role: new_role.startdate == existing_role.startdate or new_role.enddate == existing_role.enddate) run_match_rule(lambda new_role, existing_role: new_role.startdate >= existing_role.startdate and new_role.enddate <= existing_role.enddate) run_match_rule(lambda new_role, existing_role: new_role.startdate <= existing_role.startdate and new_role.enddate >= existing_role.enddate) # Update the database entries that correspond with records in the data file. did_update_any = False for new_role, existing_role in matches: new_role.id = existing_role.id if role_processor.changed(existing_role, new_role) or options.force: new_role.save() did_update_any = True if not options.force: log.warn("Updated %s" % new_role) # If we have mutliple records on disk that didn't match and multiple records in the database # that didn't match, then we don't know how to align them. if len(new_roles) > 0 and len(existing_roles) > 0: print(new_roles) print(existing_roles) raise Exception("There is an unmatched role.") # Otherwise if there are any unmatched new roles, we can just add them. for role in new_roles: log.warn("Created %s" % role) role.save() did_update_any = True # And likewise for any existing roles that are left over. for pr in existing_roles: print(pr.person.id, pr) raise ValueError("Deleted role??") log.warn("Deleted %s" % pr) pr.delete() if did_update_any and not options.disable_events: # Create the events for the roles after all have been loaded # because we don't create events for ends of terms and # starts of terms that are adjacent. Refresh the list to get # the roles in order. role_list = list( PersonRole.objects.filter( person=person).order_by('startdate')) for i in range(len(role_list)): role_list[i].create_events( role_list[i - 1] if i > 0 else None, role_list[i + 1] if i < len(role_list) - 1 else None) # The name can't be determined until all of the roles are set. If # it changes, re-save. Unfortunately roles are cached so this actually # doesn't work yet. Re-run the parser to fix names. nn = (person.name, person.sortname) if hasattr(person, "role"): delattr(person, "role") # clear the cached info person._most_recent_role = None # clear cache here too person.set_names() if nn != (person.name, person.sortname): log.warn("%s is now %s." % (nn[0], person.name)) person.save() except Exception as ex: # Catch unexpected exceptions and log them pprint.pprint(node) log.error('', exc_info=ex) had_error = True progress.tick() log.info('Processed persons: %d' % len(processed_persons)) log.info('Created persons: %d' % len(created_persons)) if not had_error: # Remove person which were not found in XML file removed_persons = existing_persons - processed_persons for pk in removed_persons: p = Person.objects.get(pk=pk) if p.roles.all().count() > 0: log.warn("Missing? Deleted? %d: %s" % (p.id, p)) else: log.warn("Deleting... %d: %s (remember to prune_index!)" % (p.id, p)) raise Exception("Won't delete!") p.delete() log.info('Missing/deleted persons: %d' % len(removed_persons)) # Mark the files as processed. for p in SRC_FILES: f = BASE_PATH + p + ".yaml" File.objects.save_file(f) update_twitter_list()
def test_find_person_ozturk(self): person = Person(forename='Selçuk', surname='Öztürk', initials='S') tkperson = openkamer.parliament.find_tkapi_person(person) self.assertEqual(tkperson.achternaam, 'Öztürk')