for data_file in data_files:
        for legislator in data_file[0]:
            num_matches = 0
            # # this can't run unless we've already collected a bioguide for this person
            bioguide = legislator["id"].get("bioguide", None)
            # if we've limited this to just one bioguide, skip over everyone else
            if only_bioguide and (bioguide != only_bioguide):
                continue
            # if not in currently read chamber, skip
            chamber = legislator["terms"][len(legislator["terms"]) - 1]["type"]
            if chamber != read_file[1]:
                continue

            # only run for selected congress
            latest_congress = utils.congress_from_legislative_year(
                utils.legislative_year(parse_date(legislator["terms"][len(legislator["terms"]) - 1]["start"]))
            )
            if chamber == "sen":
                congresses = [latest_congress, latest_congress + 1, latest_congress + 2]
            else:
                congresses = [latest_congress]

            if int(congress) not in congresses:
                continue

            # pull data to match from yaml

            last_name_unicode = legislator["name"]["last"].upper().strip().replace("'", "")
            last_name = unicodedata.normalize("NFD", unicode(last_name_unicode)).encode("ascii", "ignore")
            state = utils.states[legislator["terms"][len(legislator["terms"]) - 1]["state"]].upper()[:7].strip()
            # select icpsr source data based on more recent chamber
Example #2
0
def run():

    # default to caching
    cache = utils.flags().get('cache', True)
    force = not cache

    only_bioguide = utils.flags().get('bioguide', None)
    congress = utils.flags().get('congress', None)

    data_files = []

    print("Loading %s..." % "legislators-current.yaml")
    legislators = load_data("legislators-current.yaml")
    data_files.append((legislators, "legislators-current.yaml"))
    print("Loading %s..." % "legislators-historical.yaml")
    legislators = load_data("legislators-historical.yaml")
    data_files.append((legislators, "legislators-historical.yaml"))

    # load member data from vote view
    if congress == None:
        raise Exception("the --congress flag is required")
    elif int(congress) < 10 and int(congress) > 0:
        url_senate = "https://voteview.com/static/data/out/members/S00%s_members.csv" % congress
        url_house = "https://voteview.com/static/data/out/members/H00%s_members.csv" % congress
    elif int(congress) < 100 and int(congress) >= 10:
        url_senate = "https://voteview.com/static/data/out/members/S0%s_members.csv" % congress
        url_house = "https://voteview.com/static/data/out/members/H0%s_members.csv" % congress
    elif int(congress) >= 100:
        url_senate = "https://voteview.com/static/data/out/members/S%s_members.csv" % congress
        url_house = "https://voteview.com/static/data/out/members/H%s_members.csv" % congress
    else:
        raise Exception("no data for congress " + congress)

    senate_destination = "icpsr/source/senate_rollcall%s.txt" % congress
    senate_data = utils.download(url_senate, senate_destination, force)

    house_destination = "icpsr/source/house_rollcall%s.txt" % congress
    house_data = utils.download(url_house, house_destination, force)

    error_log = csv.writer(
        open("cache/errors/mismatch/mismatch_%s.csv" % congress, "w"))
    error_log.writerow([
        "error_type", "matches", "icpsr_name", "icpsr_state", "is_territory",
        "old_id", "new_id"
    ])

    read_files = [("sen", senate_data), ("rep", house_data)]
    print("Running for congress " + congress)
    for read_file_chamber, read_file_content in read_files:
        for data_file in data_files:
            for legislator in data_file[0]:
                num_matches = 0
                write_id = ""
                # this can't run unless we've already collected a bioguide for this person
                bioguide = legislator["id"].get("bioguide", None)
                # if we've limited this to just one bioguide, skip over everyone else
                if only_bioguide and (bioguide != only_bioguide):
                    continue
                #if not in currently read chamber, skip
                chamber = legislator['terms'][len(legislator['terms']) -
                                              1]['type']
                if chamber != read_file_chamber:
                    continue

                #only run for selected congress
                latest_congress = utils.congress_from_legislative_year(
                    utils.legislative_year(
                        parse_date(
                            legislator['terms'][len(legislator['terms']) -
                                                1]['start'])))
                if chamber == "sen":
                    congresses = [
                        latest_congress, latest_congress + 1,
                        latest_congress + 2
                    ]
                else:
                    congresses = [latest_congress]

                if int(congress) not in congresses:
                    continue

                # pull data to match from yaml

                last_name = legislator['name']['last'].upper()
                state = utils.states[legislator['terms']
                                     [len(legislator['terms']) -
                                      1]['state']].upper()[:7].strip()

                # convert read_file_content str to file object, then parse as csv file
                content_as_file = StringIO(read_file_content)
                content_parsed = csv.reader(content_as_file, delimiter=',')

                # loop through congress members in read file, see if one matches the current legislator
                for icpsr_member in content_parsed:
                    # ensure unique match bassed of bioguide id
                    if bioguide == icpsr_member[10]:
                        num_matches += 1
                        write_id = int(icpsr_member[2])

                # skip if icpsr id is currently in data
                if "icpsr" in legislator["id"]:
                    if write_id == legislator["id"]["icpsr"] or write_id == "":
                        continue
                    elif write_id != legislator["id"][
                            "icpsr"] and write_id != "":
                        error_log.writerow([
                            "Incorrect_ID", "NA", last_name[:8], state, "NA",
                            legislator["id"]["icpsr"], write_id
                        ])
                        print("ID updated for %s" % last_name)

                if num_matches == 1:
                    legislator['id']['icpsr'] = int(write_id)
                else:
                    if state == 'GUAM' or state == 'PUERTO' or state == "VIRGIN" or state == "DISTRIC" or state == "AMERICA" or state == "NORTHER" or state == "PHILIPP":
                        print('error: non 1 match')
                        error_log.writerow([
                            "Non_1_match_number",
                            str(num_matches), last_name[:8], state, "Y", "NA",
                            "NA"
                        ])
                    else:
                        print(
                            str(num_matches) + " matches found for " +
                            last_name[:8] + ", " + state + " in congress " +
                            str(congress))
                        error_log.writerow([
                            "Non_1_match_number",
                            str(num_matches), last_name, state, "N", "NA", "NA"
                        ])

            save_data(data_file[0], data_file[1])
Example #3
0
def run():

    # default to caching
    cache = utils.flags().get('cache', True)
    force = not cache

    only_bioguide = utils.flags().get('bioguide', None)
    congress = utils.flags().get('congress', None)

    data_files = []

    print("Loading %s..." % "legislators-current.yaml")
    legislators = load_data("legislators-current.yaml")
    data_files.append((legislators, "legislators-current.yaml"))
    print("Loading %s..." % "legislators-historical.yaml")
    legislators = load_data("legislators-historical.yaml")
    data_files.append((legislators, "legislators-historical.yaml"))

    #load roll call data. Will need to be updated (possibly) for 114th+ congresses, since it is unclear what the URl format will be
    if congress == None:
        raise Exception("the --congress flag is required")
    elif congress == "113":
        url_senate = "http://amypond.sscnet.ucla.edu/rollcall/static/S113.ord"
        url_house = "http://amypond.sscnet.ucla.edu/rollcall/static/H113.ord"
    elif int(congress) < 10 and int(congress) > 0:
        url_senate = "ftp://voteview.com/dtaord/sen0%skh.ord" % congress
        url_house = "ftp://voteview.com/dtaord/hou0%skh.ord" % congress
    elif int(congress) < 113 and int(congress) >= 10:
        url_senate = "ftp://voteview.com/dtaord/sen%skh.ord" % congress
        url_house = "ftp://voteview.com/dtaord/hou%skh.ord" % congress
    else:
        raise Exception("no data for congress " + congress)

    senate_destination = "icpsr/source/senate_rollcall%s.txt" % congress
    senate_data = utils.download(url_senate, senate_destination, force)

    house_destination = "icpsr/source/house_rollcall%s.txt" % congress
    house_data = utils.download(url_house, house_destination, force)

    error_log = csv.writer(
        open("cache/errors/mismatch/mismatch_%s.csv" % congress, "wb"))
    error_log.writerow([
        "error_type", "matches", "icpsr_name", "icpsr_state", "is_territory",
        "old_id", "new_id"
    ])

    read_files = [(senate_data, "sen"), (house_data, "rep")]
    print("Running for congress " + congress)
    for read_file in read_files:
        for data_file in data_files:
            for legislator in data_file[0]:
                num_matches = 0
                # # this can't run unless we've already collected a bioguide for this person
                bioguide = legislator["id"].get("bioguide", None)
                # if we've limited this to just one bioguide, skip over everyone else
                if only_bioguide and (bioguide != only_bioguide):
                    continue
                #if not in currently read chamber, skip
                chamber = legislator['terms'][len(legislator['terms']) -
                                              1]['type']
                if chamber != read_file[1]:
                    continue

                #only run for selected congress
                latest_congress = utils.congress_from_legislative_year(
                    utils.legislative_year(
                        parse_date(
                            legislator['terms'][len(legislator['terms']) -
                                                1]['start'])))
                if chamber == "sen":
                    congresses = [
                        latest_congress, latest_congress + 1,
                        latest_congress + 2
                    ]
                else:
                    congresses = [latest_congress]

                if int(congress) not in congresses:
                    continue

                # pull data to match from yaml

                last_name_unicode = legislator['name']['last'].upper().strip(
                ).replace('\'', '')
                last_name = unicodedata.normalize(
                    'NFD', str(last_name_unicode)).encode('ascii', 'ignore')
                state = utils.states[legislator['terms']
                                     [len(legislator['terms']) -
                                      1]['state']].upper()[:7].strip()
                # select icpsr source data based on more recent chamber

                write_id = ""
                lines = read_file[0].split('\n')
                for line in lines:
                    # parse source data
                    icpsr_state = line[12:20].strip()
                    icpsr_name = line[21:].strip().strip(string.digits).strip()
                    icpsr_id = line[3:8].strip()

                    #ensure unique match
                    if icpsr_name[:8] == last_name[:8] and state == icpsr_state:
                        num_matches += 1
                        write_id = icpsr_id
                #skip if icpsr id is currently in data
                if "icpsr" in legislator["id"]:
                    if write_id == legislator["id"]["icpsr"] or write_id == "":
                        continue
                    elif write_id != legislator["id"][
                            "icpsr"] and write_id != "":
                        error_log.writerow([
                            "Incorrect_ID", "NA", last_name[:8], state, "NA",
                            legislator["id"]["icpsr"], write_id
                        ])
                        print("ID updated for %s" % last_name)
                if num_matches == 1:
                    legislator['id']['icpsr'] = int(write_id)
                else:
                    if state == 'GUAM' or state == 'PUERTO' or state == "VIRGIN" or state == "DISTRIC" or state == "AMERICA" or state == "NORTHER" or state == "PHILIPP":
                        error_log.writerow([
                            "Non_1_match_number",
                            str(num_matches), last_name[:8], state, "Y", "NA",
                            "NA"
                        ])
                    else:
                        print(
                            str(num_matches) + " matches found for " +
                            last_name[:8] + ", " + state + " in congress " +
                            str(congress))
                        error_log.writerow([
                            "Non_1_match_number",
                            str(num_matches), last_name, state, "N", "NA", "NA"
                        ])

            save_data(data_file[0], data_file[1])
def run():

    # default to caching
    cache = utils.flags().get('cache', True)
    force = not cache


    only_bioguide = utils.flags().get('bioguide', None)
    congress = utils.flags().get('congress',None)


    data_files = []

    print("Loading %s..." % "legislators-current.yaml")
    legislators = load_data("legislators-current.yaml")
    data_files.append((legislators,"legislators-current.yaml"))
    print("Loading %s..." % "legislators-historical.yaml")
    legislators = load_data("legislators-historical.yaml")
    data_files.append((legislators,"legislators-historical.yaml"))

    #load roll call data. Will need to be updated (possibly) for 114th+ congresses, since it is unclear what the URl format will be
    if congress == None:
        raise Exception("the --congress flag is required")
    elif congress == "113":
        url_senate = "http://amypond.sscnet.ucla.edu/rollcall/static/S113.ord"
        url_house = "http://amypond.sscnet.ucla.edu/rollcall/static/H113.ord"
    elif int(congress) <10 and int(congress) >0:
        url_senate = "ftp://voteview.com/dtaord/sen0%skh.ord" % congress
        url_house = "ftp://voteview.com/dtaord/hou0%skh.ord" % congress
    elif int(congress) < 113 and int(congress) >= 10:
        url_senate = "ftp://voteview.com/dtaord/sen%skh.ord" % congress
        url_house = "ftp://voteview.com/dtaord/hou%skh.ord" % congress
    else:
        raise Exception("no data for congress " + congress)

    senate_destination = "icpsr/source/senate_rollcall%s.txt" % congress
    senate_data = utils.download(url_senate, senate_destination, force)

    house_destination = "icpsr/source/house_rollcall%s.txt" % congress
    house_data = utils.download(url_house, house_destination, force)

    error_log = csv.writer(open("cache/errors/mismatch/mismatch_%s.csv" % congress, "wb"))
    error_log.writerow(["error_type","matches","icpsr_name","icpsr_state","is_territory","old_id","new_id"])



    read_files = [(senate_data,"sen"),(house_data,"rep")]
    print("Running for congress " + congress)
    for read_file in read_files:
        for data_file in data_files:
            for legislator in data_file[0]:
                num_matches = 0
                # # this can't run unless we've already collected a bioguide for this person
                bioguide = legislator["id"].get("bioguide", None)
                # if we've limited this to just one bioguide, skip over everyone else
                if only_bioguide and (bioguide != only_bioguide):
                    continue
                #if not in currently read chamber, skip
                chamber = legislator['terms'][len(legislator['terms'])-1]['type']
                if chamber != read_file[1]:
                    continue

                #only run for selected congress
                latest_congress = utils.congress_from_legislative_year(utils.legislative_year(parse_date(legislator['terms'][len(legislator['terms'])-1]['start'])))
                if chamber == "sen":
                    congresses = [latest_congress,latest_congress+1,latest_congress+2]
                else:
                    congresses =[latest_congress]

                if int(congress) not in congresses:
                    continue

                # pull data to match from yaml

                last_name_unicode = legislator['name']['last'].upper().strip().replace('\'','')
                last_name = unicodedata.normalize('NFD', str(last_name_unicode)).encode('ascii', 'ignore')
                state = utils.states[legislator['terms'][len(legislator['terms'])-1]['state']].upper()[:7].strip()
                # select icpsr source data based on more recent chamber

                write_id = ""
                lines = read_file[0].split('\n')
                for line in lines:
                    # parse source data
                    icpsr_state = line[12:20].strip()
                    icpsr_name = line[21:].strip().strip(string.digits).strip()
                    icpsr_id = line[3:8].strip()

                    #ensure unique match
                    if icpsr_name[:8] == last_name[:8] and state == icpsr_state:
                        num_matches += 1
                        write_id = icpsr_id
                #skip if icpsr id is currently in data
                if "icpsr" in legislator["id"]:
                    if write_id == legislator["id"]["icpsr"] or write_id == "":
                        continue
                    elif write_id != legislator["id"]["icpsr"] and write_id != "":
                        error_log.writerow(["Incorrect_ID","NA",last_name[:8],state,"NA",legislator["id"]["icpsr"],write_id])
                        print("ID updated for %s" % last_name)
                if num_matches == 1:
                    legislator['id']['icpsr'] = int(write_id)
                else:
                    if state == 'GUAM' or state == 'PUERTO' or state == "VIRGIN" or state == "DISTRIC" or state == "AMERICA" or state == "NORTHER" or state == "PHILIPP":
                        error_log.writerow(["Non_1_match_number",str(num_matches),last_name[:8],state,"Y","NA","NA"])
                    else:
                        print(str(num_matches) + " matches found for "+ last_name[:8] + ", " + state + " in congress " + str(congress))
                        error_log.writerow(["Non_1_match_number",str(num_matches),last_name,state,"N","NA","NA"])


            save_data(data_file[0], data_file[1])