def test_extended_people_filter(session): people = ExtendedPersonCollection(session) people.add(first_name="Hans", last_name="Maulwurf") people.add(first_name="Waylon", last_name="Śmithers") people.add(first_name="Lenny", last_name="leonard") people.add(first_name="Carl", last_name="Çarlson") ned = people.add(first_name="Ned", last_name="Flanders") agencies = ExtendedAgencyCollection(session) agencies.add_root(title="Police").add_person(ned.id, "Snitch") agencies.add_root(title="Ħospital").add_person(ned.id, "Volunteer") agencies.add_root(title="Moe's Tavern") assert [p.last_name for p in people.query() ] == ["Çarlson", "Flanders", "leonard", "Maulwurf", "Śmithers"] people = people.for_filter(letter="C") assert [p.last_name for p in people.query()] == ['Çarlson'] people = people.for_filter(letter="c") assert [p.last_name for p in people.query()] == ['Çarlson'] people = people.for_filter(letter="L") assert [p.last_name for p in people.query()] == ['leonard'] people = people.for_filter(agency="Police") assert [p.last_name for p in people.query()] == [] people = people.for_filter(letter=None) assert [p.last_name for p in people.query()] == ['Flanders']
def test_membership_form(session): people = ExtendedPersonCollection(session) request = DummyRequest(session) doctor = people.add(first_name="Nick", last_name="Rivera") form = MembershipForm() form.request = request form.on_request() assert form.person_id.choices == [(str(doctor.id), 'Rivera Nick')] form = MembershipForm(DummyPostData({ 'person_id': str(doctor.id), 'title': "Doctor", 'since': "2000", 'addition': "Surgery", 'note': "Retired", 'prefix': "*", })) assert form.get_useful_data() == { 'person_id': str(doctor.id), 'title': "Doctor", 'addition': "Surgery", 'note': "Retired", 'since': "2000", 'prefix': "*", }
def test_extended_people_pagination(session): people = ExtendedPersonCollection(session) assert people.pages_count == 0 assert people.batch == [] for letter in ascii_uppercase: people.add(first_name=letter.lower(), last_name=letter) assert people.query().count() == 26 people = ExtendedPersonCollection(session) assert people.subset_count == 26 assert people.pages_count == 2 assert len(people.batch) == 20 assert len(people.next.batch) == 6 assert len(people.page_by_index(1).batch) == 6
def test_extended_people_used_letters(session): assert ExtendedPersonCollection(session).used_letters == [] people = ExtendedPersonCollection(session) people.add(first_name="Hans", last_name="Maulwurf") people.add(first_name="Waylon", last_name="Śmithers") people.add(first_name="Lenny", last_name="leonard") people.add(first_name="Carl", last_name="Çarlson") people.add(first_name="Ned", last_name="Flanders") assert people.used_letters == ['C', 'F', 'L', 'M', 'S']
def test_extended_people_exclude_hidden(session): people = ExtendedPersonCollection(session) assert people.exclude_hidden is False person = people.add(first_name="Hans", last_name="Maulwurf") assert people.query().count() == 1 people.exclude_hidden = True assert people.query().count() == 1 person.is_hidden_from_public = True assert people.query().count() == 0 person.is_hidden_from_public = False assert people.query().count() == 1
def test_extended_people_used_agencies(session): assert ExtendedPersonCollection(session).used_agencies == [] agencies = ExtendedAgencyCollection(session) police = agencies.add_root(title="Police") hospital = agencies.add_root(title="Ħospital") agencies.add_root(title="Moe's Tavern") assert ExtendedPersonCollection(session).used_agencies == [] people = ExtendedPersonCollection(session) ned = people.add(first_name="Ned", last_name="Flanders") police.add_person(ned.id, "Snitch") hospital.add_person(ned.id, "Volunteer") assert people.used_agencies == ['Ħospital', 'Police']
def test_membership_form_choices(session): people = ExtendedPersonCollection(session) people.add(first_name="Nick", last_name="Rivera") people.add(first_name="Nick", last_name="Rivera", phone="1234") people.add(first_name="Nick", last_name="Rivera", phone="5555", email="x") people.add(first_name="Nick", last_name="Rivera", phone_direct="4") people.add(first_name="Nick", last_name="Rivera", address="Street") people.add(first_name="Nick", last_name="Rivera", email="*****@*****.**") agencies = ExtendedAgencyCollection(session) agency = agencies.add_root(title="Hospital") doc = people.add(first_name="Nick", last_name="Rivera") agency.add_person(doc.id, "Doc") request = DummyRequest(session) form = MembershipForm() form.request = request form.on_request() assert sorted([x[1] for x in form.person_id.choices]) == [ 'Rivera Nick', 'Rivera Nick (1234)', 'Rivera Nick (4)', 'Rivera Nick (5555)', 'Rivera Nick (Hospital)', 'Rivera Nick (Street)', 'Rivera Nick ([email protected])' ]
def _import(request, app): EXPORT_FIELDS = { 'academic_title': 'person.academic_title', 'address': 'person.address', 'direct_number': 'person.phone_direct', 'firstname': 'person.first_name', 'lastname': 'person.last_name', 'occupation': 'person.profession', 'phone': 'person.phone', 'political_party': 'person.political_party', 'postfix': 'membership.addition', 'role': 'membership.title', 'start': 'membership.since', 'title': 'person.title', 'year': 'person.born', } class LinkFilter(Filter): """ Uses the href rather than the content of an a-tag. """ def __iter__(self): in_link = False for token in Filter.__iter__(self): if token.get('name') == 'a': if token['type'] == 'StartTag': in_link = True data = token['data'][(None, 'href')] data = data.replace('mailto:', '') yield {'type': 'Characters', 'data': data} elif token['type'] == 'EndTag': in_link = False elif token['type'] == 'Characters': if not in_link: yield token else: yield token cleaner = Cleaner(tags=['a', 'p', 'br'], attributes={'a': 'href'}, strip=True, filters=[LinkFilter, whitespace_filter]) session = app.session() if clear: click.secho("Deleting all agencies", fg='yellow') for root in AgencyCollection(session).roots: session.delete(root) click.secho("Deleting all people", fg='yellow') for person in PersonCollection(session).query(): session.delete(person) workbook = open_workbook(file) click.secho("Importing agencies", fg='green') agencies = ExtendedAgencyCollection(session) people = ExtendedPersonCollection(session) sheet = workbook.sheet_by_name('Organisationen') ids = {} parents = {} alphabetical = [] for row in range(1, sheet.nrows): if skip_root and row == 1: continue if row and (row % 50 == 0): app.es_indexer.process() # We use our own, internal IDs which are auto-incremented external_id = int(sheet.cell_value(row, 0)) # Remove the HTML code from the portrait, prepend the description portrait = '\n'.join( (sheet.cell_value(row, 3).strip(), html_to_text(cleaner.clean(sheet.cell_value(row, 4))))) portrait = portrait.replace('\n\n', '\n').strip() # Re-map the export fields export_fields = sheet.cell_value(row, 7) or 'role,title' export_fields = export_fields.split(',') export_fields = [EXPORT_FIELDS[field] for field in export_fields] agency = agencies.add( parent=parents.get(external_id), title=sheet.cell_value(row, 2).strip(), portrait=portrait, export_fields=export_fields, is_hidden_from_public=sheet.cell_value(row, 8) == 'private', order=external_id, ) ids[external_id] = agency.id # Download and add the organigram if not skip_download: organigram_url = sheet.cell_value(row, 6) if organigram_url: response = get(organigram_url) response.raise_for_status() agency.organigram_file = BytesIO(response.content) if sheet.cell_value(row, 5): alphabetical.append(agency.id) for child in sheet.cell_value(row, 1).split(','): if child: child = int(child) parents[child] = agency # Let's make sure, the order have nice, cohere values def defrag_ordering(agency): for order, child in enumerate(agency.children): child.order = order defrag_ordering(child) for order, root in enumerate(agencies.roots): root.order = order defrag_ordering(root) click.secho("Importing people and memberships", fg='green') sheet = workbook.sheet_by_name('Personen') for row in range(1, sheet.nrows): if row and (row % 50 == 0): app.es_indexer.process() notes = '\n'.join((sheet.cell_value(row, 13).strip(), sheet.cell_value(row, 14).strip())).strip() person = people.add( academic_title=sheet.cell_value(row, 0).strip(), profession=sheet.cell_value(row, 1).strip(), function=(sheet.cell_value(row, 17).strip() if sheet.ncols > 17 else ''), first_name=sheet.cell_value(row, 2).strip(), last_name=sheet.cell_value(row, 3).strip(), political_party=sheet.cell_value(row, 4).strip(), born=sheet.cell_value(row, 5).strip(), email=sheet.cell_value(row, 6).strip(), address=sheet.cell_value(row, 7).strip(), phone=sheet.cell_value(row, 8).strip(), phone_direct=sheet.cell_value(row, 9).strip(), salutation=sheet.cell_value(row, 10).strip(), website=sheet.cell_value(row, 12).strip(), is_hidden_from_public=sheet.cell_value(row, 15) == 'private', notes=notes, ) memberships = sheet.cell_value(row, 16).split('//') for membership in memberships: if membership: matched = re.match( r'^\((\d*)\)\((.*)\)\((.*)\)\((.*)\)' r'\((.*)\)\((.*)\)\((\d*)\)\((\d*)\)$', membership) if matched: values = matched.groups() else: # old version before order_within_person existed matched = re.match( r'^\((\d*)\)\((.*)\)\((.*)\)\((.*)\)' r'\((.*)\)\((.*)\)\((\d*)\)$', membership) values = list(matched.groups()) values.append('0') person.memberships.append( ExtendedAgencyMembership( agency_id=ids[int(values[0])], title=values[1] or "", since=values[2] or None, prefix=values[3], addition=values[4], note=values[5], order_within_agency=int(values[6]), order_within_person=int(values[7]), )) # Order the memberships alphabetically, if desired for id_ in alphabetical: agencies.by_id(id_).sort_relationships() # Show a tree view of what we imported if visualize: click.secho("Imported data:", fg='green') def show(agency, level): text = f'{agency.title}\n' for membership in agency.memberships: person = membership.person text += f'* {membership.title}: {person.title}\n' click.echo(indent(text.strip(), level * ' ')) for child in agency.children: show(child, level + 1) for root in agencies.roots: show(root, 1) # Abort the transaction if requested if dry_run: transaction.abort() click.secho("Aborting transaction", fg='yellow')
def test_extended_people(session): people = ExtendedPersonCollection(session) person = people.add(first_name="Hans", last_name="Maulwurf") assert isinstance(person, ExtendedPerson) assert isinstance(people.query().one(), ExtendedPerson)