Example #1
0
def test_multiple_posts_class():
    create_jurisdiction()

    org = Organization.objects.create(id="fnd",
                                      name="Foundation",
                                      classification="foundation",
                                      jurisdiction_id="fnd-jid")
    hari = Person.objects.create(id="hs", name="Hari Seldon")
    founder = Post.objects.create(id='f',
                                  label="founder",
                                  role="Founder",
                                  organization=org)
    chair = Post.objects.create(id='c',
                                label="chair",
                                role="Chair",
                                organization=org)

    m1 = ScrapeMembership(person_id=hari.id,
                          organization_id=org.id,
                          post_id=founder.id)
    m2 = ScrapeMembership(person_id=hari.id,
                          organization_id=org.id,
                          post_id=chair.id)

    dumb_imp = DumbMockImporter()
    memimp = MembershipImporter('fnd-jid', dumb_imp, dumb_imp, dumb_imp)
    memimp.import_data([m1.as_dict(), m2.as_dict()])

    # ensure that the memberships attached in the right places
    assert org.memberships.count() == 2
    assert hari.memberships.count() == 2
    assert founder.memberships.count() == 1
    assert chair.memberships.count() == 1
def test_full_membership():
    org = Organization.objects.create(id="fnd", name="Foundation", classification="foundation",
                                      jurisdiction_id="fnd-jid")
    hari = Person.objects.create(id="hs", name="Hari Seldon")
    robot = Person.objects.create(id="robot", name="R. Daneel Olivaw")
    post = Post.objects.create(id='f', label="founder", role="Founder", organization=org)

    # add a membership through a post
    m1 = ScrapeMembership(person_id=hari.id, organization_id=org.id, post_id=post.id)
    m1.add_contact_detail(type='phone', value='555-555-1234', note='this is fake')
    m1.add_link('http://example.com/link')

    # add a membership direct to an organization
    m2 = ScrapeMembership(person_id=robot.id, organization_id=org.id, label='member',
                          role='member')

    dumb_imp = DumbMockImporter()
    memimp = MembershipImporter('fnd-jid', dumb_imp, dumb_imp, dumb_imp)
    memimp.import_data([m1.as_dict(), m2.as_dict()])

    # ensure that the memberships attached in the right places
    assert org.memberships.count() == 2
    assert hari.memberships.count() == 1
    assert robot.memberships.count() == 1
    assert post.memberships.count() == 1

    # ensure that the first membership has contact details and links
    m = hari.memberships.get()
    cd = m.contact_details.get()
    assert cd.type == 'phone'
    assert cd.value == '555-555-1234'
    assert cd.note == 'this is fake'
    assert m.links.all()[0].url == 'http://example.com/link'
Example #3
0
def test_full_membership():
    create_jurisdiction()
    org = Organization.objects.create(id="fnd", name="Foundation", classification="foundation",
                                      jurisdiction_id="fnd-jid")
    hari = Person.objects.create(id="hs", name="Hari Seldon")
    robot = Person.objects.create(id="robot", name="R. Daneel Olivaw")
    post = Post.objects.create(id='f', label="founder", role="Founder", organization=org)

    # add a membership through a post
    m1 = ScrapeMembership(person_id=hari.id, organization_id=org.id, post_id=post.id)
    m1.add_contact_detail(type='phone', value='555-555-1234', note='this is fake')
    m1.add_link('http://example.com/link')

    # add a membership direct to an organization
    m2 = ScrapeMembership(person_id=robot.id, organization_id=org.id, label='member',
                          role='member')

    dumb_imp = DumbMockImporter()
    memimp = MembershipImporter('fnd-jid', dumb_imp, dumb_imp, dumb_imp)
    memimp.import_data([m1.as_dict(), m2.as_dict()])

    # ensure that the memberships attached in the right places
    assert org.memberships.count() == 2
    assert hari.memberships.count() == 1
    assert robot.memberships.count() == 1
    assert post.memberships.count() == 1

    # ensure that the first membership has contact details and links
    m = hari.memberships.get()
    cd = m.contact_details.get()
    assert cd.type == 'phone'
    assert cd.value == '555-555-1234'
    assert cd.note == 'this is fake'
    assert m.links.all()[0].url == 'http://example.com/link'
Example #4
0
    def scrape(self):
        method = 'metadata/{}?'.format(self.state)
        self.metadata = self.api(method)

        method = 'legislators/?state={}&fields=id&active=False'.format(
            self.state)
        for result in self.api(method):
            if result['id'] not in ('NJL000013', ):
                yield self.scrape_legislator(result['id'])

        method = 'committees/?state={}&fields=id,parent_id'.format(self.state)
        results = sorted(self.api(method),
                         key=lambda x: x.get('parent_id', '') or '')
        for result in results:
            yield self.scrape_committee(result['id'])

        for leg_id, com_id, role, start, end in self._roles:

            # resolve OpenStates committee id to JSON ids
            if com_id in self._committees:
                com_id = self._committees[com_id]._id
            else:
                continue

            if leg_id == 'NJL000013':
                leg_id = 'NJL000015'
            yield Membership(person_id=self._people[leg_id]._id,
                             organization_id=com_id,
                             role=role,
                             label=role,
                             start_date=str(start),
                             end_date=str(end))
def test_unmatched_person():
    create_jurisdiction()

    org = Organization.objects.create(id="fnd", name="Foundation", classification="foundation",
                                      jurisdiction_id="fnd-jid")
    # not a real person, won't have a person_id after import
    m1 = ScrapeMembership(person_name='Harry Seldom', organization_id=org.id,
                          person_id=None
                          )

    dumb_imp = DumbMockImporter()
    memimp = MembershipImporter('fnd-jid', dumb_imp, dumb_imp, dumb_imp)
    memimp.import_data([m1.as_dict()])

    # ensure that the memberships attached in the right places
    assert org.memberships.count() == 1

    membership = org.memberships.get()
    assert membership.person_id is None
    assert membership.person_name == 'Harry Seldom'
def test_multiple_posts_class():
    create_jurisdiction()

    org = Organization.objects.create(id="fnd", name="Foundation", classification="foundation",
                                      jurisdiction_id="fnd-jid")
    hari = Person.objects.create(id="hs", name="Hari Seldon")
    founder = Post.objects.create(id='f', label="founder", role="Founder", organization=org)
    chair = Post.objects.create(id='c', label="chair", role="Chair", organization=org)

    m1 = ScrapeMembership(person_id=hari.id, organization_id=org.id, post_id=founder.id)
    m2 = ScrapeMembership(person_id=hari.id, organization_id=org.id, post_id=chair.id)

    dumb_imp = DumbMockImporter()
    memimp = MembershipImporter('fnd-jid', dumb_imp, dumb_imp, dumb_imp)
    memimp.import_data([m1.as_dict(), m2.as_dict()])

    # ensure that the memberships attached in the right places
    assert org.memberships.count() == 2
    assert hari.memberships.count() == 2
    assert founder.memberships.count() == 1
    assert chair.memberships.count() == 1
Example #7
0
def test_unmatched_person():
    create_jurisdiction()

    org = Organization.objects.create(id="fnd",
                                      name="Foundation",
                                      classification="foundation",
                                      jurisdiction_id="fnd-jid")
    # not a real person, won't have a person_id after import
    m1 = ScrapeMembership(person_name='Harry Seldom',
                          organization_id=org.id,
                          person_id=None)

    dumb_imp = DumbMockImporter()
    memimp = MembershipImporter('fnd-jid', dumb_imp, dumb_imp, dumb_imp)
    memimp.import_data([m1.as_dict()])

    # ensure that the memberships attached in the right places
    assert org.memberships.count() == 1

    membership = org.memberships.get()
    assert membership.person_id is None
    assert membership.person_name == 'Harry Seldom'
def test_basic_invalid_membership():
    membership = Membership(person_id=33, organization_id="orga_id")
    with pytest.raises(ValueError):
        membership.validate()
Example #9
0
    def scrape_legislator(self, legislator_id):
        old = self.api('legislators/' + legislator_id + '?')
        # just not needed
        id = old.pop('id')

        old.pop('created_at')
        old.pop('updated_at')
        old.pop('country', None)
        old.pop('level', None)
        old.pop('state')
        old.pop('leg_id')
        old.pop('active')
        # junk keys
        old.pop('suffix', None)
        old.pop('notice', None)
        old.pop('csrfmiddlewaretoken', None)
        old.pop('office_address', None)
        old.pop('office_phone', None)

        # translated
        district = old.pop('district', None)
        chamber = old.pop('chamber', None)
        image = old.pop('photo_url', '')
        name = old.pop('full_name')
        party = old.pop('party', None)

        if party in ('Nonpartisan', 'unknown', 'Unknown', 'Unaffiliated',
                     "Non Affiliated", " "):
            party = None
        elif party == 'Democrat':
            party = 'Democratic'

        if self.state in ('ne', 'dc'):
            chamber = 'legislature'

        if chamber == 'upper' and self.state == 'pr':
            pr_district = {
                '1': 'I',
                '2': 'II',
                '3': 'III',
                '4': 'IV',
                '5': 'V',
                '6': 'VI',
                '7': 'VII',
                '8': 'VIII',
            }
            if district in pr_district:
                district = pr_district[district]

        if '2008-2011' in old:
            old['old_roles']['2008-2011'] = old.pop('2008-2011')

        old_roles = old.pop('old_roles', {})

        if old['roles'] and 'Lt. Governor' in [
                x['type'] for x in old['roles']
        ]:
            new = Person(name=name,
                         district=district,
                         party=party,
                         image=image)
            self.jurisdiction._executive.add_post('Lt. Governor', 'lt-gov')
            membership = Membership(
                person_id=new._id,
                role="Lt. Governor",
                organization_id=self.jurisdiction._executive._id)
            new._related.append(membership)
        else:
            new = Person(name=name, party=party, image=image)

        if id in birthdays:
            new.birth_date = birthdays[id]

        # various ids
        id_types = {
            'votesmart_id': 'votesmart',
            'transparencydata_id': 'influence-explorer',
            'nimsp_id': 'nimsp',
            'nimsp_candidate_id': 'nimsp-candidate',
        }
        for idname, scheme in id_types.items():
            val = old.pop(idname, None)
            if val:
                new.add_identifier(val, scheme=scheme)
        for id in old.pop('all_ids'):
            new.add_identifier(id, scheme='openstates')
            self._people[id] = new

        # contact details
        email = old.pop('email', None)
        if email:
            new.add_contact_detail(type='email', value=email, note='')
        office_keys = {
            'fax': 'fax',
            'phone': 'voice',
            'email': 'email',
            'address': 'address'
        }
        for office in old.pop('offices'):
            for key, type in office_keys.items():
                if office.get(key):
                    if 'Office Hours' in office[key] and self.state == 'pa':
                        for x in office[key].split('Office Hours: '):
                            if x:
                                new.add_contact_detail(type=type,
                                                       value=x,
                                                       note=office['name'])
                    else:
                        new.add_contact_detail(type=type,
                                               value=office[key],
                                               note=office['name'])

        # links
        link = old.pop('url', None)
        if link:
            new.add_link(link)

        #for utah, conflict of interest is in links
        if self.state == 'ut':
            links = old.pop('+links', [])
            for l in links:
                new.add_link(note="conflict of interest form", url=l)

        # sources
        for source in old.pop('sources'):
            source.pop('retrieved', None)
            source.pop('+page', None)
            new.add_source(**source)

        # roles
        for role in old.pop('roles'):
            self.process_role(new, role, leg_id=id)

        for role_list in old_roles.values():
            for role in role_list:
                self.process_role(new, role, leg_id=id)

        # ignore most of the names for now
        old.pop('first_name')
        old.pop('middle_name')
        old.pop('suffixes')
        old.pop('nickname', None)
        new.sort_name = old.pop('last_name')

        #some places have legacy names without underscores
        old.pop('+firstname', None)
        old.pop('+lastname', None)

        gender = old.pop('+gender', None)
        if gender:
            new.gender = gender
        biography = old.pop('+biography', None)
        if biography:
            new.biography = biography
        birth_date = old.pop('+birth_date', None)
        if birth_date:
            new.birth_date = birth_date

        # keys to keep
        to_extras = [
            '+occupation', '+twitter', '+facebook_url', '+sworn_in_date',
            '+profession', '+secretary', '+office_hours', '+resident_county',
            '+district_name', '+leg_status', '+legal_position', '+title',
            '+start_year', '+end_date', 'occupation', '+oregon_member_id',
            '+facebook', '+youtube', '+instagram'
        ]
        for k in to_extras:
            v = old.pop(k, None)
            if v:
                new.extras[k.replace('+', '')] = v

        # keys not to keep
        to_pop = [
            '+office_fax',
            '+phone',
            '+room',
            '+fax',
            '+email',
            '+url',
            '+photo',
            '+notice',
            '+page',
            '+suffix',
            '+city',
            '+address',
            '+additional_info_url',
            '+contact_form',
            '+fax_number',
            '+phone_number',
            '+business_phone',
            '+email_address',
            '+img_url',
            '+office_phone',
            '+disctict_name',
            '+office_loc',
            '+leg_url',
            '+office',
            '+district_address',
            '+capital_address',
            '+bis_phone',
            '+capital_phone',
            '+org_info',
            '+role',
            '+other_phone',
            '+home_phone',
            '+zip',
            '+zipcode',
            '+county',
            '+capitol_phone',
            '+image_url',
            '+header',
            '+town_represented',
            '+full_address',
            '+capitol_address',
            '+website',
            '+district_phone',
            '+district_offices',
            '+party',
            '+district',
            '+capitol_office',
            '+office_address',
        ]
        for k in to_pop:
            old.pop(k, None)

        # ensure we got it all
        assert not old, old.keys()

        return new
Example #10
0
def test_full_membership():
    create_jurisdiction()
    org = Organization.objects.create(id="fnd",
                                      name="Foundation",
                                      classification="foundation",
                                      jurisdiction_id="fnd-jid")
    hari = Person.objects.create(id="hs", name="Hari Seldon")
    robot = Person.objects.create(id="robot", name="R. Daneel Olivaw")
    post = Post.objects.create(id='f',
                               label="founder",
                               role="Founder",
                               organization=org)

    # add a membership through a post, with a start date
    m1 = ScrapeMembership(person_id=hari.id,
                          organization_id=org.id,
                          post_id=post.id,
                          start_date='2020-03-10',
                          end_date='2021-06-30')
    m1.add_contact_detail(type='phone',
                          value='555-555-1234',
                          note='this is fake')
    m1.add_link('http://example.com/link')

    # add a membership direct to an organization, with an end date
    m2 = ScrapeMembership(person_id=robot.id,
                          organization_id=org.id,
                          label='member',
                          role='member',
                          end_date='2019-11-09')

    dumb_imp = DumbMockImporter()
    memimp = MembershipImporter('fnd-jid', dumb_imp, dumb_imp, dumb_imp)
    memimp.import_data([m1.as_dict(), m2.as_dict()])

    # ensure that the memberships attached in the right places
    assert org.memberships.count() == 2
    assert hari.memberships.count() == 1
    assert robot.memberships.count() == 1
    assert post.memberships.count() == 1

    # ensure that the first membership has contact details and links
    m = hari.memberships.get()
    cd = m.contact_details.get()
    assert cd.type == 'phone'
    assert cd.value == '555-555-1234'
    assert cd.note == 'this is fake'
    assert m.links.all()[0].url == 'http://example.com/link'

    # update the imported memberships (i.e., change attributes that are not
    # in the spec) and confirm they resolve correctly
    memimp2 = MembershipImporter('fnd-jid', dumb_imp, dumb_imp, dumb_imp)

    m1.end_date = '2022-03-10'
    m2.extras = {'note': 'bleep blorp'}

    import_log = memimp2.import_data([m1.as_dict(), m2.as_dict()])

    assert import_log['membership']['insert'] == 0
    assert import_log['membership']['update'] == 2

    # confirm the membership resolved based on start date and its end date was updated
    assert hari.memberships.count() == 1
    assert hari.memberships.get().end_date == '2022-03-10'

    # confirm the membership resolved based on end date and its extras were updated
    assert robot.memberships.count() == 1
    assert robot.memberships.get().extras == {'note': 'bleep blorp'}
Example #11
0
def test_basic_invalid_membership():
    membership = Membership(person_id=33, organization_id="orga_id")
    with pytest.raises(ValueError):
        membership.validate()
Example #12
0
def test_basic_membership():
    m = Membership(person_id='person', organization_id='org')
    assert 'person' in str(m) and 'org' in str(m)
    def scrape_current_legislators(self, repos):
        for repo in repos:
            CURRENT_LEGISLATORS = self.get_url(repo)

            people = self.yamlize(CURRENT_LEGISLATORS)
            parties = set()
            posts = {}
            person_cache = defaultdict(lambda: defaultdict(lambda: None))

            for person in people:
                name = person['name'].get('official_full')
                if name is None:
                    name = "{name[first]} {name[last]}".format(**person)

                if 'birthday' in person['bio']:
                    birth_date = person['bio']['birthday']

                who = person_cache[name][birth_date]
                has_term = False

                if who is None:
                    who = Person(name=name, birth_date=birth_date)
                    who.add_source(url=CURRENT_LEGISLATORS,
                                   note="unitedstates project on GitHub")

                for term in person.get('terms', []):
                    has_term = True
                    start_date = term['start']
                    end_date = term['end']
                    state = term['state']
                    type_ = term['type']
                    district = term.get('district', None)
                    party = term.get('party', None)

                    chamber = {
                        'rep': 'lower',
                        'sen': 'upper',
                    }[type_]

                    role = {
                        'rep': 'Representative',
                        'sen': 'Senator',
                    }[type_]

                    if type_ == "rep" and district is not None:
                        label = "%s for District %s in %s" % (role, district,
                                                              state)

                        if district == 0:
                            division_id = (
                                "ocd-division/country:us/state:{state}".format(
                                    state=state.lower()))
                        else:
                            division_id = (
                                "ocd-division/country:us/"
                                "state:{state}/cd:{district}".format(
                                    state=state.lower(), district=district))

                        post = posts.get(division_id)
                        if post is None:
                            post = Post(organization_id={
                                "rep": self.house,
                                "sen": self.senate
                            }[type_]._id,
                                        division_id=division_id,
                                        label=label,
                                        role=role)
                            posts[division_id] = post
                            yield post

                        membership = Membership(post_id=post._id,
                                                role=role,
                                                label=label,
                                                start_date=start_date,
                                                end_date=end_date,
                                                person_id=who._id,
                                                organization_id={
                                                    "rep": self.house,
                                                    "sen": self.senate,
                                                }[type_]._id)
                        yield membership

                    if type_ == "sen":

                        division_id = (
                            "ocd-division/country:us/state:{state}".format(
                                state=state.lower()))

                        label = "Senitor for %s" % (state)

                        post = posts.get(division_id)
                        if post is None:
                            post = Post(organization_id={
                                "rep": self.house,
                                "sen": self.senate
                            }[type_]._id,
                                        division_id=division_id,
                                        label=label,
                                        role=role)
                            posts[division_id] = post
                            yield post

                        membership = Membership(post_id=post._id,
                                                role=role,
                                                label=label,
                                                start_date=start_date,
                                                end_date=end_date,
                                                person_id=who._id,
                                                organization_id={
                                                    "rep": self.house,
                                                    "sen": self.senate,
                                                }[type_]._id)
                        yield membership

                    if party == "Democrat":
                        party = "Democratic"

                    if party:
                        membership = Membership(role='member',
                                                start_date=start_date,
                                                end_date=end_date,
                                                person_id=who._id,
                                                organization_id=make_pseudo_id(
                                                    classification="party",
                                                    name=party))
                        yield membership

                for key, value in person.get('id', {}).items():
                    if isinstance(value, list):
                        for v in value:
                            who.add_identifier(str(v), scheme=key)
                    else:
                        who.add_identifier(str(value), scheme=key)

                if has_term:
                    yield who
Example #14
0
def test_basic_invalid_membership():
    membership = Membership(33, "orga_id")
    with pytest.raises(ValueError):
        membership.validate()
Example #15
0
def test_basic_invalid_membership():
    membership = Membership(33, "orga_id")
    with pytest.raises(ValueError):
        membership.validate()
Example #16
0
def test_basic_membership():
    m = Membership('person', 'org')
    assert 'person' in str(m) and 'org' in str(m)