Ejemplo n.º 1
0
def import_bills(state, data_dir):
    data_dir = os.path.join(data_dir, state)
    pattern = os.path.join(data_dir, "bills", "*.json")
    for path in glob.iglob(pattern):
        with open(path) as f:
            data = prepare_obj(json.load(f))

        bill = db.bills.find_one(
            {"state": data["state"], "session": data["session"], "chamber": data["chamber"], "bill_id": data["bill_id"]}
        )

        for sponsor in data["sponsors"]:
            id = get_legislator_id(state, data["session"], None, sponsor["name"])
            sponsor["leg_id"] = id

        for vote in data["votes"]:
            for vtype in ("yes_votes", "no_votes", "other_votes"):
                svlist = []
                for svote in vote[vtype]:
                    id = get_legislator_id(state, data["session"], vote["chamber"], svote)
                    svlist.append({"name": svote, "leg_id": id})

                vote[vtype] = svlist

        if not bill:
            data["created_at"] = datetime.datetime.now()
            data["updated_at"] = data["created_at"]
            data["keywords"] = list(keywordize(data["title"]))
            insert_with_id(data)
        else:
            data["keywords"] = list(keywordize(data["title"]))
            update(bill, data, db.bills)
Ejemplo n.º 2
0
def import_legislator(data):
    # Rename 'role' -> 'type'
    for role in data['roles']:
        if 'role' in role:
            role['type'] = role['role']
            del role['role']

    cur_role = data['roles'][0]

    spec = {'state': data['state'],
            'term': cur_role['term'],
            'type': cur_role['type']}
    if 'district' in cur_role:
        spec['district'] = cur_role['district']
    if 'chamber' in cur_role:
        spec['chamber'] = cur_role['chamber']

    leg = db.legislators.find_one(
        {'state': data['state'],
         'full_name': data['full_name'],
         'roles': {'$elemMatch': spec}})

    if not leg:
        metadata = db.metadata.find_one({'_id': data['state']})

        term_names = [t['name'] for t in metadata['terms']]

        try:
            index = term_names.index(cur_role['term'])

            if index > 0:
                prev_term = term_names[index - 1]
                spec['term'] = prev_term
                prev_leg = db.legislators.find_one(
                    {'full_name': data['full_name'],
                     'roles': {'$elemMatch': spec}})

                if prev_leg:
                    update(prev_leg, data, db.legislators)
                    return
        except ValueError:
            print "Invalid term: %s" % cur_role['term']
            sys.exit(1)

        data['created_at'] = datetime.datetime.now()
        data['updated_at'] = datetime.datetime.now()

        insert_with_id(data)
    else:
        update(leg, data, db.legislators)

    ensure_indexes()
def import_legislator(data):
    # Rename 'role' -> 'type'
    for role in data['roles']:
        if 'role' in role:
            role['type'] = role['role']
            del role['role']

    cur_role = data['roles'][0]

    leg = db.legislators.find_one(
        {'state': data['state'],
         'full_name': data['full_name'],
         'roles': {'$elemMatch': {
                    'session': cur_role['session'],
                    'chamber': cur_role['chamber'],
                    'type': cur_role['type'],
                    'district': cur_role['district']}}})

    if not leg:
        metadata = db.metadata.find_one({'_id': data['state']})

        session_names = [s['name'] for s in metadata['sessions']]

        try:
            index = session_names.index(cur_role['session'])

            if index > 0:
                prev_session = session_names[index - 1]
                prev_leg = db.legislators.find_one(
                    {'state': data['state'],
                     'full_name': data['full_name'],
                     'roles': {'$elemMatch': {
                                'session': prev_session,
                                'chamber': cur_role['chamber'],
                                'type': cur_role['type'],
                                'district': cur_role['district']}}})

                if prev_leg:
                    update(prev_leg, data, db.legislators)
                    return
        except ValueError:
            print "Invalid session: %s" % cur_role['session']
            sys.exit(1)

        data['created_at'] = datetime.datetime.now()
        data['updated_at'] = datetime.datetime.now()

        insert_with_id(data)
    else:
        update(leg, data, db.legislators)
Ejemplo n.º 4
0
def test_update():
    dt = datetime.datetime.utcnow()
    obj1 = {'_type': 'bill', 'state': 'ex', 'field1': 'stuff',
            'field2': 'original', '_locked_fields': 'field2',
            'created_at': dt, 'updated_at': dt}

    id1 = utils.insert_with_id(obj1)
    obj1 = db.bills.find_one(id1)

    # Updating a bill with itself shouldn't cause 'updated_at' to be changed
    utils.update(obj1, obj1, db.bills)
    obj2 = db.bills.find_one({'_id': id1})
    assert obj2['created_at'] == obj2['updated_at']
    assert obj1['updated_at'] == obj2['updated_at']

    utils.update(obj1, {'_type': 'bill', 'field1': 'more stuff',
                        'field2': 'a change', 'state': 'ex'},
                 db.bills)
    obj2 = db.bills.find_one({'_id': id1})
    assert obj2['created_at'] != obj2['updated_at']
    assert obj1['updated_at'] != obj2['updated_at']
    assert obj2['field1'] == 'more stuff'

    # make sure locked fields don't get overwritten
    assert obj2['field2'] == 'original'
Ejemplo n.º 5
0
def import_legislator(data):
    # Rename 'role' -> 'type'
    for role in data["roles"]:
        if "role" in role:
            role["type"] = role["role"]
            del role["role"]

    cur_role = data["roles"][0]

    spec = {"state": data["state"], "term": cur_role["term"], "type": cur_role["type"]}
    if "district" in cur_role:
        spec["district"] = cur_role["district"]
    if "chamber" in cur_role:
        spec["chamber"] = cur_role["chamber"]

    leg = db.legislators.find_one(
        {"state": data["state"], "full_name": data["full_name"], "roles": {"$elemMatch": spec}}
    )

    if not leg:
        metadata = db.metadata.find_one({"_id": data["state"]})

        term_names = [t["name"] for t in metadata["terms"]]

        try:
            index = term_names.index(cur_role["term"])

            if index > 0:
                prev_term = term_names[index - 1]
                spec["term"] = prev_term
                prev_leg = db.legislators.find_one({"full_name": data["full_name"], "roles": {"$elemMatch": spec}})

                if prev_leg:
                    update(prev_leg, data, db.legislators)
                    return
        except ValueError:
            print "Invalid term: %s" % cur_role["term"]
            sys.exit(1)

        data["created_at"] = datetime.datetime.utcnow()
        data["updated_at"] = datetime.datetime.utcnow()

        insert_with_id(data)
    else:
        update(leg, data, db.legislators)

    ensure_indexes()
Ejemplo n.º 6
0
def test_insert_with_id():
    obj1 = {'full_name': 'a test legislator',
           '_type': 'person',
           'state': 'ex'}
    obj2 = {'full_name': 'another legislator',
            '_type': 'person',
            'state': 'ex'}

    id_re = r'^EXL\d{6,6}$'

    id1 = utils.insert_with_id(obj1)
    assert re.match(id_re, id1)
    assert db.legislators.find_one({'_id': id1})

    id2 = utils.insert_with_id(obj2)
    assert re.match(id_re, id2)
    assert id2 != id1
    assert db.legislators.find_one({'_id': id2})
Ejemplo n.º 7
0
def import_committees(state):
    db.committees.ensure_index([('state', pymongo.ASCENDING),
                                ('chamber', pymongo.ASCENDING)])
    db.committees.ensure_index([('state', pymongo.ASCENDING),
                                ('votesmart_id', pymongo.ASCENDING)])

    types = {'upper': 'S'}
    if state['lower_chamber_name'].startswith('House'):
        types['lower'] = 'H'
    else:
        types['lower'] = 'S'

    for chamber, typeId in types.items():
        for committee in votesmart.committee.getCommitteesByTypeState(
            typeId=typeId, stateId=state['_id'].upper()):

            parent_id = committee.parentId
            if parent_id == "-1":
                parent_id = None

            data = db.committees.find_one({
                    'state': state['_id'],
                    'votesmart_id': committee.committeeId})

            insert = False
            if not data:
                insert = True
                data = {}

            data.update({'state': state['_id'],
                         'votesmart_id': committee.committeeId,
                         'chamber': chamber,
                         'name': committee.name,
                         'parent_votesmart_id': parent_id,
                         '_type': 'committee'})

            if insert:
                insert_with_id(data)
            else:
                db.committees.save(data)

    import_committee_ids(state)
Ejemplo n.º 8
0
def import_legislator(data):
    data = prepare_obj(data)
    data['_scraped_name'] = data['full_name']

    # Rename 'role' -> 'type'
    for role in data['roles']:
        if 'role' in role:
            role['type'] = role['role']
            del role['role']

    cur_role = data['roles'][0]
    term = cur_role['term']
    prev_term = get_previous_term(data['state'], term)
    next_term = get_next_term(data['state'], term)

    spec = {'state': data['state'],
            'type': cur_role['type'],
            'term': {'$in': [term, prev_term, next_term]}}
    if 'district' in cur_role:
        spec['district'] = cur_role['district']
    if 'chamber' in cur_role:
        spec['chamber'] = cur_role['chamber']

    leg = db.legislators.find_one(
        {'state': data['state'],
         '_scraped_name': data['full_name'],
         'roles': {'$elemMatch': spec}})

    if leg:
        if 'old_roles' not in leg:
            leg['old_roles'] = {}

        if leg['roles'][0]['term'] == prev_term:
            # Move to old
            leg['old_roles'][leg['roles'][0]['term']] = leg['roles']
        elif leg['roles'][0]['term'] == next_term:
            leg['old_roles'][term] = data['roles']
            data['roles'] = leg['roles']

        update(leg, data, db.legislators)
    else:
        insert_with_id(data)
Ejemplo n.º 9
0
def import_committees(state, data_dir):
    data_dir = os.path.join(data_dir, state)
    pattern = os.path.join(data_dir, "committees", "*.json")

    meta = db.metadata.find_one({"_id": state})
    current_term = meta["terms"][-1]["name"]
    current_session = meta["terms"][-1]["sessions"][-1]

    paths = glob.glob(pattern)

    if not paths:
        # Not standalone committees
        for legislator in db.legislators.find({"roles": {"$elemMatch": {"term": current_term, "state": state}}}):

            for role in legislator["roles"]:
                if role["type"] == "committee member" and "committee_id" not in role:

                    spec = {"state": role["state"], "chamber": role["chamber"], "committee": role["committee"]}
                    if "subcommittee" in role:
                        spec["subcommittee"] = role["subcommittee"]

                    committee = db.committees.find_one(spec)

                    if not committee:
                        committee = spec
                        committee["_type"] = "committee"
                        committee["members"] = []
                        committee["sources"] = []
                        insert_with_id(committee)

                    for member in committee["members"]:
                        if member["leg_id"] == legislator["leg_id"]:
                            break
                    else:
                        committee["members"].append(
                            {"name": legislator["full_name"], "leg_id": legislator["leg_id"], "role": "member"}
                        )
                        db.committees.save(committee, safe=True)

                        role["committee_id"] = committee["_id"]

            db.legislators.save(legislator, safe=True)

    for path in paths:
        with open(path) as f:
            data = prepare_obj(json.load(f))

        spec = {"state": state, "chamber": data["chamber"], "committee": data["committee"]}
        if "subcommittee" in data:
            spec["subcommittee"] = data["subcommittee"]

        committee = db.committees.find_one(spec)

        if not committee:
            insert_with_id(data)
            committee = data
        else:
            update(committee, data, db.committees)

        for member in committee["members"]:
            if not member["name"]:
                continue

            (pre, first, last, suff) = name_tools.split(member["name"])

            leg_id = get_legislator_id(state, current_session, data["chamber"], member["name"])

            if not leg_id:
                print "No matches for %s" % member["name"].encode("ascii", "ignore")
                member["leg_id"] = None
                continue

            legislator = db.legislators.find_one({"_id": leg_id})

            member["leg_id"] = leg_id

            for role in legislator["roles"]:
                if (
                    role["type"] == "committee member"
                    and role["term"] == current_term
                    and role["committee_id"] == committee["_id"]
                ):
                    break
            else:
                new_role = {
                    "type": "committee member",
                    "committee": committee["committee"],
                    "term": current_term,
                    "chamber": committee["chamber"],
                    "committee_id": committee["_id"],
                    "state": state,
                }
                if "subcommittee" in committee:
                    new_role["subcommittee"] = committee["subcommittee"]
                legislator["roles"].append(new_role)
                legislator["updated_at"] = datetime.datetime.utcnow()
                db.legislators.save(legislator, safe=True)

        db.committees.save(committee, safe=True)

    print "imported %s committee files" % len(paths)

    link_parents(state)

    ensure_indexes()
Ejemplo n.º 10
0
def import_bills(state, data_dir):
    data_dir = os.path.join(data_dir, state)
    pattern = os.path.join(data_dir, 'bills', '*.json')

    meta = db.metadata.find_one({'_id': state})

    # Build a session to term mapping
    sessions = {}
    for term in meta['terms']:
        for session in term['sessions']:
            sessions[session] = term['name']

    for path in glob.iglob(pattern):
        with open(path) as f:
            data = prepare_obj(json.load(f))

        bill = db.bills.find_one({'state': data['state'],
                                  'session': data['session'],
                                  'chamber': data['chamber'],
                                  'bill_id': data['bill_id']})

        for sponsor in data['sponsors']:
            id = get_legislator_id(state, data['session'], None,
                                   sponsor['name'])
            sponsor['leg_id'] = id

        for vote in data['votes']:
            for vtype in ('yes_votes', 'no_votes', 'other_votes'):
                svlist = []
                for svote in vote[vtype]:
                    id = get_legislator_id(state, data['session'],
                                           vote['chamber'], svote)
                    svlist.append({'name': svote, 'leg_id': id})

                vote[vtype] = svlist

        data['_term'] = sessions[data['session']]

        # Merge any version titles into the alternate_titles list
        alt_titles = set(data['alternate_titles'])
        for version in data['versions']:
            if 'title' in version:
                alt_titles.add(version['title'])
            if '+short_title' in version:
                alt_titles.add(version['+short_title'])
        try:
            # Make sure the primary title isn't included in the
            # alternate title list
            alt_titles.remove(data['title'])
        except KeyError:
            pass
        data['alternate_titles'] = list(alt_titles)

        if not bill:
            data['created_at'] = datetime.datetime.now()
            data['updated_at'] = data['created_at']
            data['_keywords'] = list(bill_keywords(data))
            insert_with_id(data)
        else:
            data['_keywords'] = list(bill_keywords(data))
            update(bill, data, db.bills)

    populate_current_fields(state)
    ensure_indexes()
Ejemplo n.º 11
0
def import_committees(state, data_dir):
    data_dir = os.path.join(data_dir, state)
    pattern = os.path.join(data_dir, 'committees', '*.json')

    meta = db.metadata.find_one({'_id': state})
    current_term = meta['terms'][-1]['name']

    paths = glob.glob(pattern)

    if not paths:
        # Not standalone committees
        for legislator in db.legislators.find({
            'roles': {'$elemMatch': {'term': current_term,
                                     'state': state}}}):

            for role in legislator['roles']:
                if (role['type'] == 'committee member' and
                    'committee_id' not in role):

                    spec = {'state': role['state'],
                            'chamber': role['chamber'],
                            'committee': role['committee']}
                    if 'subcommittee' in role:
                        spec['subcommittee'] = role['subcommittee']

                    committee = db.committees.find_one(spec)

                    if not committee:
                        committee = spec
                        committee['_type'] = 'committee'
                        committee['members'] = []
                        insert_with_id(committee)

                    for member in committee['members']:
                        if member['leg_id'] == legislator['leg_id']:
                            break
                    else:
                        committee['members'].append(
                            {'name': legislator['full_name'],
                             'leg_id': legislator['leg_id'],
                             'role': 'member'})
                        db.committees.save(committee, safe=True)

                        role['committee_id'] = committee['_id']

            db.legislators.save(legislator, safe=True)

    for path in paths:
        with open(path) as f:
            data = prepare_obj(json.load(f))

        spec = {'state': state,
                'committee': data['committee']}
        if 'subcommittee' in data:
            spec['subcommittee'] = data['subcommittee']

        committee = db.committees.find_one(spec)

        if not committee:
            insert_with_id(data)
            committee = data
        else:
            update(committee, data, db.committees)

        for member in committee['members']:
            if not member['legislator']:
                continue

            (pre, first, last, suff) = name_tools.split(member['legislator'])

            found = db.legislators.find({
                    'first_name': first,
                    'last_name': last,
                    'roles': {'$elemMatch': {'term': current_term,
                                             'state': state}}})

            if found.count() > 1:
                print "Too many matches for %s" % member['legislator'].encode(
                    'ascii', 'ignore')
                continue
            elif found.count() == 0:
                print "No matches for %s" % member['legislator'].encode(
                    'ascii', 'ignore')
                continue

            legislator = found[0]

            member['leg_id'] = legislator['_id']

            for role in legislator['roles']:
                if (role['type'] == 'committee member' and
                    role['term'] == current_term and
                    role['committee_id'] == committee['_id']):
                    break
            else:
                new_role = {'type': 'committee member',
                            'committee': committee['committee'],
                            'term': current_term,
                            'chamber': committee['chamber'],
                            'committee_id': committee['_id'],
                            'state': state}
                if 'subcommittee' in committee:
                    new_role['subcommittee'] = committee['subcommittee']
                legislator['roles'].append(new_role)
                legislator['updated_at'] = datetime.datetime.now()
                db.legislators.save(legislator, safe=True)

        db.committees.save(committee, safe=True)

    ensure_indexes()
Ejemplo n.º 12
0
def test_activate_legislators():
    # Previous term
    leg1 = {
        "_type": "person",
        "state": "ex",
        "roles": [
            {
                "type": "member",
                "chamber": "upper",
                "state": "ex",
                "term": "2009-2010",
                "district": "1",
                "party": "Democrat",
                "start_date": None,
                "end_date": None,
            }
        ],
    }

    # Current term, no end date
    leg2 = {
        "_type": "person",
        "state": "ex",
        "roles": [
            {
                "type": "member",
                "chamber": "upper",
                "state": "ex",
                "term": "2011-2012",
                "district": "2",
                "party": "Democrat",
                "start_date": None,
                "end_date": None,
            }
        ],
    }

    # Current term, end date
    leg3 = {
        "_type": "person",
        "state": "ex",
        "roles": [
            {
                "type": "member",
                "chamber": "upper",
                "state": "ex",
                "term": "2011-2012",
                "district": "3",
                "party": "Democrat",
                "start_date": None,
                "end_date": datetime.datetime.now(),
            }
        ],
    }

    id1 = utils.insert_with_id(leg1)
    id2 = utils.insert_with_id(leg2)
    id3 = utils.insert_with_id(leg3)

    legislators.activate_legislators("ex", "2011-2012")
    assert "active" not in leg1
    assert "district" not in leg1
    assert "chamber" not in leg1
    assert "party" not in leg1

    leg2 = db.legislators.find_one({"_id": id2})
    assert leg2["active"] == True
    assert leg2["district"] == "2"
    assert leg2["chamber"] == "upper"
    assert leg2["party"] == "Democrat"

    leg3 = db.legislators.find_one({"_id": id3})
    assert "active" not in leg3
    assert "district" not in leg3
    assert "chamber" not in leg3
    assert "party" not in leg3
Ejemplo n.º 13
0
def test_deactivate_legislators():
    # Previous term
    leg1 = {
        "_type": "person",
        "state": "ex",
        "roles": [
            {
                "type": "member",
                "chamber": "upper",
                "state": "ex",
                "term": "2009-2010",
                "district": "1",
                "party": "Democrat",
                "start_date": None,
                "end_date": None,
            }
        ],
        "active": True,
        "district": "1",
        "chamber": "upper",
        "party": "Democrat",
    }
    leg1_roles = leg1["roles"]

    # Current term, no end date
    leg2 = {
        "_type": "person",
        "state": "ex",
        "roles": [
            {
                "type": "member",
                "chamber": "upper",
                "state": "ex",
                "term": "2011-2012",
                "district": "2",
                "party": "Democrat",
                "start_date": None,
                "end_date": None,
            }
        ],
        "active": True,
        "district": "2",
        "chamber": "upper",
        "party": "Democrat",
    }
    leg2_roles = leg2["roles"]

    id1 = utils.insert_with_id(leg1)
    id2 = utils.insert_with_id(leg2)

    legislators.deactivate_legislators("ex", "2011-2012")

    leg1 = db.legislators.find_one({"_id": id1})
    assert leg1["active"] == False
    assert "chamber" not in leg1
    assert "district" not in leg1
    assert "party" not in leg1
    assert leg1["roles"] == []
    assert leg1["old_roles"]["2009-2010"] == leg1_roles

    leg2 = db.legislators.find_one({"_id": id2})
    assert leg2["active"] == True
    assert leg2["chamber"] == "upper"
    assert leg2["district"] == "2"
    assert leg2["party"] == "Democrat"
    assert leg2["roles"] == leg2_roles
    assert "old_roles" not in leg2