def memberLookup(qDict, maxResults=50, distinct=0, api="Web"): # Setup so that the bottle call to this API doesn't need to know parameters we accept explicitly name = qDict["name"] if "name" in qDict else "" icpsr = qDict["icpsr"] if "icpsr" in qDict else "" state_abbrev = qDict["state_abbrev"] if "state_abbrev" in qDict else "" congress = qDict["congress"] if "congress" in qDict else "" chamber = qDict["chamber"] if "chamber" in qDict else "" party_code = qDict["party_code"] if "party_code" in qDict else "" bioguide_id = qDict["bioguide_id"] if "bioguide_id" in qDict else "" district_code = qDict["district_code"] if "district_code" in qDict else "" id = qDict["id"] if "id" in qDict else "" speaker = qDict["speaker"] if "speaker" in qDict else "" freshman = qDict["freshman"] if "freshman" in qDict else "" idIn = qDict["idIn"] if "idIn" in qDict else [] biography = qDict["biography"] if "biography" in qDict else "" if api == "R": maxResults = 5000 # Check to make sure there's a query if not name and not icpsr and not state_abbrev and not congress and not district_code and not chamber and not id and not party_code and not bioguide_id and not speaker and not idIn and not freshman and not biography: return({'errormessage': 'No search terms provided'}) # Fold search query into dict searchQuery = {} if api=="districtLookup": searchQuery["id"] = {"$in": qDict["idIn"]} if icpsr: try: icpsr = int(icpsr) searchQuery["icpsr"] = icpsr except: try: if icpsr[0]=="M": for r in db.voteview_members.find({'id': icpsr}, {'icpsr': 1, '_id': 0}): searchQuery["icpsr"] = r["icpsr"] break except: return({"errormessage": "Invalid ICPSR number supplied."}) if id: try: if id.upper().startswith("MH") or id.upper().startswith("MS"): searchQuery["id"] = id else: return({"errormessage": "Invalid ID supplied1."}) except: return({"errormessage": "Invalid ID supplied2."}) if state_abbrev: state = str(state_abbrev) if len(state) == 2 or state.upper() == "USA": searchQuery["state_abbrev"] = state.upper() # States are all stored upper-case else: searchQuery["state_abbrev"] = stateNameToAbbrev(state.upper()) if biography: searchQuery["biography"] = {"$regex": biography, "$options": i} if congress: try: if isinstance(congress, int): # already a number searchQuery["congress"] = congress elif not " " in congress: # congress is just a number congress = int(congress) searchQuery["congress"] = congress elif "[" in congress and "]" in congress and "to" in congress: # congress is a range valText = congress[1:-1] min, maxC = valText.split(" to ") searchQuery["congress"] = {} if len(min): searchQuery["congress"]["$gte"] = int(min) # From min if len(maxC): searchQuery["congress"]["$lte"] = int(maxC) # To max else: # congress is a series of integers, use $in vals = [int(val) for val in congress.split(" ")] searchQuery["congress"] = {} searchQuery["congress"]["$in"] = vals except: print traceback.format_exc() return({"errormessage": "Invalid congress ID supplied."}) if name: if ", " in name: # Last, First last, rest = name.split(", ",1) searchQuery["$text"] = {"$search": rest+" "+last} else: searchQuery["$text"] = {"$search": name} if speaker: searchQuery["served_as_speaker"] = 1 if freshman: try: maxCongress = json.load(open("static/config.json","r"))["maxCongress"] except: try: maxCongress = json.load(open("../static/config.json","r"))["maxCongress"] except: maxCongress = 116 searchQuery["congresses.0.0"] = maxCongress if party_code: if isinstance(party_code, dict): searchQuery["party_code"] = party_code else: searchQuery["party_code"] = int(party_code) if bioguide_id: searchQuery["bioguide_id"] = bioguide_id if district_code and state_abbrev: searchQuery["district_code"] = district_code if chamber: chamber = chamber.capitalize() if chamber=="Senate" or chamber=="House" or chamber=="President": searchQuery["chamber"] = chamber else: return({"errormessage": "Invalid chamber provided. Please select House or Senate."}) response = [] errormessage = "" i = 0 # Field return specifications, allows us to return less than all our data to searches. if api=="Web_PI": fieldSet = {"nominate.dim1": 1, "party_code": 1, "district_code": 1, "icpsr": 1, "chamber":1, "nvotes_yea_nay": 1, "nvotes_against_party": 1, "nvotes_abs": 1, "_id": 0} elif api=="Web_FP_Search": fieldSet = {"bioname": 1, "party_code": 1, "icpsr": 1, "state_abbrev": 1, "congress": 1, "_id": 0, "congresses": 1, "chamber": 1, "bioguide_id": 1} elif api == "Check_Party_Switch": fieldSet = {"icpsr": 1, "_id": 0} elif api=="Web_Congress": if chamber: fieldName = "elected_"+chamber.lower() fieldSet = {"bioname": 1, "party_code": 1, "icpsr": 1, "state_abbrev": 1, "congress": 1, "_id": 0, "bioImgURL": 1, "minElected": 1, "nominate.dim1": 1, "nominate.dim2": 1, "congresses": 1, fieldName: 1} else: fieldSet = {"bioname": 1, "party_code": 1, "icpsr": 1, "state_abbrev": 1, "congress": 1, "_id": 0, "bioImgURL": 1, "minElected": 1, "nominate.dim1": 1, "nominate.dim2": 1, "congresses": 1, "state_abbrev": 1, "elected_senate": 1, "elected_house": 1} elif api=="Web_Party": fieldSet = {"bioname": 1, "party_code": 1, "icpsr": 1, "state_abbrev": 1, "congress": 1, "_id": 0, "bioImgURL": 1, "minElected": 1, "nominate.dim1": 1, "nominate.dim2": 1, "congresses": 1, "chamber": 1} elif api=="R": fieldSet = {"bioname": 1, "party_code": 1, "icpsr": 1, "state_abbrev": 1, "congress": 1, "id": 1, "_id": 0, "nominate.dim1": 1, "nominate.dim2": 1, "nominate.geo_mean_probability": 1, "cqlabel": 1, "district_code": 1, "chamber": 1, "congresses": 1} elif api=="exportCSV" or api == "exportORD": fieldSet = {"bioname": 1, "party_code": 1, "icpsr": 1, "state_abbrev": 1, "congress": 1, "id": 1, "_id": 0, "nominate": 1, "district_code": 1, "chamber": 1, "state_name_trunc": 1, "last_means": 1, "occupancy": 1, "name": 1} elif api=="districtLookup": fieldSet = {"bioname": 1, "party_code": 1, "icpsr": 1, "state_abbrev": 1, "congress": 1, "id": 1, "nominate.dim1": 1, "nominate.dim2": 1, "district_code": 1, "_id": 0, "chamber": 1, "congresses": 1} else: fieldSet = {"_id": 0, "personid": 0} if "$text" in searchQuery: fieldSet["score"] = {"$meta": "textScore"} res = db.voteview_members.find(searchQuery, fieldSet) # Try to induce regex if a name search fails? hypotheticalCount = res.count() if "$text" in searchQuery and hypotheticalCount==0: print "No results from a name search, fall back to regex" del searchQuery["$text"] searchQuery["bioname"] = {'$regex': name, '$options': 'i'} res = db.voteview_members.find(searchQuery, fieldSet) if "$text" in searchQuery: sortedRes = res.sort([('score', {'$meta': 'textScore'})]) elif api=="exportORD": db.voteview_members.ensure_index([('state_abbrev', 1), ('district_code', 1), ('icpsr', 1)], name="ordIndex") sortedRes = res.sort([('state_abbrev', 1), ('district_code', 1), ('icpsr', 1)]) else: sortedRes = res.sort('congress', -1) if sortedRes.count()>1000 and api != "R" and api!= "Web_Party": return({"errormessage": "Too many results found."}) elif sortedRes.count()>5000 and api!= "Web_Party": return({"errormessage": "Too many results found."}) currentICPSRs = [] for m in sortedRes: if m["icpsr"] in currentICPSRs and distinct==1: continue else: currentICPSRs.append(m["icpsr"]) newM = m if "state_abbrev" in newM: newM["state"] = stateName(newM["state_abbrev"]) if api=="exportORD": newM["state_icpsr"] = stateIcpsr(newM["state_abbrev"]) if "district_code" in newM and "state_abbrev" in newM: newM["cqlabel"] = cqlabel(newM["state_abbrev"], newM["district_code"]) if "party_code" in newM: newM["party_name"] = partyName(newM["party_code"]) if api not in ["exportORD", "exportCSV", "R"]: newM["party_noun"] = noun(newM["party_code"]) newM["party_color"] = partyColor(newM["party_code"]) newM["party_short_name"] = shortName(newM["party_code"]) # Check if an image exists. if os.path.isfile("/var/www/voteview/static/img/bios/"+str(newM["icpsr"]).zfill(6)+".jpg"): newM["bioImgURL"] = str(newM["icpsr"]).zfill(6)+".jpg" else: newM["bioImgURL"] = "silhouette.png" if api in ["exportCSV", "exportORD"]: if 'bioname' in newM: newM['bioname'] = newM['bioname'].encode('utf-8') if "nominate" in newM: for k,v in newM["nominate"].iteritems(): if k == 'log_likelihood': newM[k] = round(v, 5) else: newM[k] = round(v, 3) del newM["nominate"] try: newM["seo_name"] = slugify(newM["bioname"]) except: pass response.append(newM) i=i+1 if i>=maxResults: break if len(response)>maxResults and maxResults>1: # For regular searches get mad if we have more than max results. errormessage = "Capping number of responses at "+str(maxResults)+"." if len(response)==0: return({'errormessage': 'No members found matching your search query.', 'query': qDict}) elif errormessage: return({'errormessage': errormessage, 'results': response}) else: return({'results': response})
def downloadAPI(rollcall_id, apitype="Web", voterId=0): starttime = time.time() # Setup API version response webexportapis = ["Web", "Web_Person", "exportJSON", "exportCSV"] if apitype in webexportapis: apiVersion = "Web 2016-10" elif apitype=="R": apiVersion = "R 2016-10" if not rollcall_id or len(rollcall_id)==0: response = {'errormessage': 'No rollcall id specified.', 'apitype': apiVersion} return response # Split multiple ID requests into list if type(rollcall_id)==type([""]): rollcall_ids = rollcall_id elif "," in rollcall_id: rollcall_ids = [x.strip() for x in rollcall_id.split(",")] else: rollcall_ids = [rollcall_id] # Abuse filter maxVotes = 100 if apitype in ["exportJSON", "exportCSV"]: maxVotes = 500 if len(rollcall_ids)>maxVotes: response = {'errormessage': 'API abuse. Too many votes.', 'apitype': apiVersion} return response rollcall_results = [] # Stores top level rollcall results, one per vote errormessage = "" # String storing error text errormeta = [] # List storing failed IDs found = {} for rid in rollcall_ids: found[rid] = 0 setupTime = time.time() # Do we need to fold in members? peopleIds = [] memberSet = [] if apitype=="Web_Person": # I need to fold in one specific member needPeople=-1 # Just one specific member peopleIds = [voterId] elif apitype=="exportCSV": needPeople=0 # No members at all else: needPeople=1 peopleIds = db.voteview_rollcalls.distinct("votes.icpsr", {"id": {"$in": rollcall_ids}}) # All relevant members congresses = [] for rollcall_id in rollcall_ids: try: congresses.append(int(rollcall_id[2:5])) except: pass memberTime1 = time.time() # Now fetch the members memberSet = [] if len(peopleIds): memberFields = {"icpsr":1, "nominate":1, "bioname":1, "party_code":1, "state_abbrev":1, "chamber":1, "district_code": 1, "congress": 1, "id":1} members = db.voteview_members.find({"icpsr": {"$in": peopleIds}, "congress": {"$in": congresses}}, memberFields) for m in members: memberSet.append(m) memberTime2 = time.time() # Now iterate through the rollcalls fieldSetNeed = {"votes": 1, "nominate": 1, "id": 1, "codes": 1, "key_flags": 1, "yea_count": 1, "nay_count": 1, "congress": 1, "chamber": 1, "rollnumber": 1, "date": 1, "vote_desc": 1, "vote_document_text": 1, "description": 1, "shortdescription": 1, "short_description": 1, "vote_question": 1, "question": 1, "party_vote_counts": 1, 'vote_result': 1, 'vote_title':1, 'vote_question_text':1, 'amendment_author':1} rollcalls = db.voteview_rollcalls.find({'id': {'$in': rollcall_ids}}, fieldSetNeed).sort('id') for rollcall in rollcalls: result = [] # Hold new votes output, start blank try: # If we need some people, let's iterate through the voters and fill them out if needPeople!=0: metaMembers = [m for m in memberSet if m["congress"] == rollcall["congress"]] for v in rollcall["votes"]: newV = {} # Only add the person if they're in our validated list of people we want. if v["icpsr"] in peopleIds: #print "In here" newV.update(v) # Do the match from the member list try: memberMap = next((m for m in metaMembers if m["icpsr"]==v["icpsr"]), None) if memberMap is None: print v["icpsr"], "Error! We don't have a member with this icpsr. Skipping" except: print v["icpsr"], "Error! We don't have a member with this icpsr. Skipping" continue # Now assemble the matching. if apitype=="Web" or apitype=="Web_Person" or apitype=="exportJSON": newV["vote"] = _get_yeanayabs(newV["cast_code"]) del newV["cast_code"] # We are not returning cast code. try: newV["x"] = memberMap["nominate"]["dim1"] except: pass try: newV["y"] = memberMap["nominate"]["dim2"] except: pass if "prob" in newV: newV["prob"] = int(round(newV["prob"])) newV["name"] = memberMap["bioname"] newV["party"] = partyName(memberMap["party_code"]) newV["party_short_name"] = shortName(memberMap["party_code"]) newV["party_code"] = memberMap["party_code"] newV["state_abbrev"] = memberMap["state_abbrev"] if memberMap["state_abbrev"] == "USA": newV["district"] = "POTUS" elif memberMap["district_code"] > 70: newV["district"] = "%s00" % memberMap["state_abbrev"] elif memberMap["district_code"] and memberMap["district_code"] <= 70: newV["district"] = "%s%02d" % (memberMap["state_abbrev"], memberMap["district_code"]) else: newV["district"] = "" # And for the R API elif apitype=="R" or apitype=="exportXLS": try: del newV["prob"] except: pass if "nominate" in memberMap and "dim1" in memberMap["nominate"]: newV["dim1"] = memberMap["nominate"]["dim1"] newV["dim2"] = memberMap["nominate"]["dim2"] newV["id"] = memberMap["id"] newV["name"] = memberMap["bioname"] newV["party_code"] = memberMap["party_code"] newV["state_abbrev"] = memberMap["state_abbrev"] newV["cqlabel"] = cqlabel(memberMap["state_abbrev"], memberMap["district_code"]) newV['district_code'] = memberMap['district_code'] # Append the new voter to the list of voters. result.append(newV) else: continue # Top level nominate metadata # Debug code to delete nominate data so we can regenerate it. if "nominate" in rollcall and "slope" in rollcall["nominate"]: del rollcall["nominate"]["slope"] if "nominate" in rollcall and "intercept" in rollcall["nominate"]: del rollcall["nominate"]["intercept"] if "nominate" in rollcall and "x" in rollcall["nominate"]: del rollcall["nominate"]["x"] if "nominate" in rollcall and "y" in rollcall["nominate"]: del rollcall["nominate"]["y"] # Generate other nominate fields if "nominate" in rollcall and "mid" in rollcall["nominate"] and "spread" in rollcall["nominate"] and rollcall["nominate"]["spread"][0] is not None: rollcall["nominate"]["slope"], rollcall["nominate"]["intercept"], rollcall["nominate"]["x"], rollcall["nominate"]["y"] = add_endpoints(rollcall["nominate"]["mid"], rollcall["nominate"]["spread"]) # Ensure everything has at least some nominate field. elif "nominate" not in rollcall: rollcall["nominate"] = {} # Flatten nominate for the R API. if apitype in webexportapis: nominate = rollcall['nominate'] checkNom = ['classified', 'pre', 'log_likelihood'] for f in checkNom: if f not in nominate: nominate[f] = '' elif apitype=="R": if "nominate" in rollcall: nominate = {"mid1": rollcall["nominate"]["mid"][0], "mid2": rollcall["nominate"]["mid"][1], "spread1": rollcall["nominate"]["spread"][0], "spread2": rollcall["nominate"]["spread"][1], "nomslope": rollcall['nominate']['slope'], 'nomintercept': rollcall['nominate']['intercept']} else: nominate = {} # Top level rollcall item. found[rollcall["id"]] = 1 # Get the best available description. description = waterfallText(rollcall) # Truncate the description for the R API. if apitype=="R" or apitype=="exportCSV": if len(description)<=255: pass else: baseDesc = description[0:254] rest = description[255:] try: cutoff = rest.index(". ") if cutoff<255: baseDesc = baseDesc + rest[0:cutoff] else: baseDesc = baseDesc + rest[0:255]+"..." except: if len(rest)<255: baseDesc = baseDesc+rest else: baseDesc = baseDesc + rest[0:255]+"..." description = baseDesc # Get the best available question question = waterfallQuestion(rollcall) if 'vote_result' not in rollcall: rollcall['vote_result'] = None # Collapse codes for R if apitype == "exportCSV" or apitype == "exportXLS": codeFields = {"Clausen1": "" , "Issue1": "", "Issue2": "", "Peltzman1": "", "Peltzman2": ""} if 'codes' in rollcall: for key, value in rollcall["codes"].iteritems(): if len(value) == 1: codeFields[key + '1'] = value[0] else: codeFields[key + '1'] = value[0] codeFields[key + '2'] = value[1] elif apitype in webexportapis: if "codes" in rollcall: codes = rollcall["codes"] else: codes = {} elif apitype=="R": if "codes" in rollcall: codes = rollcall["codes"] for key, value in codes.iteritems(): codes[key] = '; '.join(value) else: codes = {} # Pre-allocate keyvote flags. if not "key_flags" in rollcall: rollcall["key_flags"] = [] # Output object: z = {'id': rollcall['id'], 'chamber': rollcall['chamber'], 'congress': rollcall['congress'], 'date': rollcall['date'], 'rollnumber': rollcall['rollnumber'], 'yea': rollcall["yea_count"], 'nay': rollcall["nay_count"], 'vote_result': rollcall['vote_result']} if apitype == "exportCSV" or apitype == "exportXLS": z.update({k:v for k,v in codeFields.iteritems()}) z.update({'keyvote': ''.join(rollcall['key_flags']), 'spread.dim1': nominate['spread'][0], 'spread.dim2': nominate['spread'][1], 'mid.dim1': nominate['mid'][0], 'mid.dim2': nominate['mid'][1], 'slope': nominate['slope'], 'intercept': nominate['intercept'], 'log_likelihood': nominate['log_likelihood'], 'classified': nominate['classified'], 'pre': nominate['pre'], 'description': description.encode('utf-8')}) if apitype != "exportCSV": z.update({'key_flags': rollcall["key_flags"], 'votes': result, 'codes': codes, 'nominate': nominate, 'description': description, 'question': question}) # Get other people's results from the party results aggregate. if apitype=="Web_Person": z["party_vote_counts"] = rollcall["party_vote_counts"] rollcall_results.append(z) except: # Invalid vote id print traceback.format_exc() errormessage = "Invalid Rollcall ID specified." errormeta.append(str(rollcall["id"])) if rollcalls.count() != len(rollcall_ids): errormessage = "Invalid Rollcall ID specified." for voteID, num in found.iteritems(): if num==0: errormeta.append(str(voteID)) endtime = time.time() #print round(setupTime-starttime,2), round(memberTime1-setupTime,2), round(memberTime2-memberTime1,2), round(endtime-memberTime2,2) response = {} if len(rollcall_results): response["rollcalls"] = rollcall_results if len(errormessage): response["errormessage"] = errormessage if len(errormeta): response["errormeta"] = errormeta response["apitype"] = apiVersion response["elapsedTime"] = round(endtime - starttime,3) return response
def downloadAPI(rollcall_id, apitype="Web", voterId=0): starttime = time.time() # Setup API version response webexportapis = ["Web", "Web_Person", "exportJSON", "exportCSV"] if apitype in webexportapis: apiVersion = "Web 2016-10" elif apitype == "R": apiVersion = "R 2016-10" if not rollcall_id or len(rollcall_id) == 0: response = {'errormessage': 'No rollcall id specified.', 'apitype': apiVersion} return response # Split multiple ID requests into list if type(rollcall_id) == type([""]): rollcall_ids = rollcall_id elif "," in rollcall_id: rollcall_ids = [x.strip() for x in rollcall_id.split(",")] else: rollcall_ids = [rollcall_id] # Abuse filter maxVotes = 100 if apitype in ["exportJSON", "exportCSV"]: maxVotes = 500 if len(rollcall_ids) > maxVotes: response = {'errormessage': 'API abuse. Too many votes.', 'apitype': apiVersion} return response rollcall_results = [] # Stores top level rollcall results, one per vote errormessage = "" # String storing error text errormeta = [] # List storing failed IDs found = {} for rid in rollcall_ids: found[rid] = 0 setupTime = time.time() # Do we need to fold in members? peopleIds = [] memberSet = [] if apitype == "Web_Person": # I need to fold in one specific member needPeople = -1 # Just one specific member peopleIds = [voterId] elif apitype == "exportCSV": needPeople = 0 # No members at all else: needPeople = 1 peopleIds = db.voteview_rollcalls.distinct( "votes.icpsr", {"id": {"$in": rollcall_ids}}) # All relevant members congresses = [] for rollcall_id in rollcall_ids: try: congresses.append(int(rollcall_id[2:5])) except: pass memberTime1 = time.time() # Now fetch the members memberSet = [] if len(peopleIds): memberFields = {"icpsr": 1, "nominate": 1, "bioname": 1, "party_code": 1, "state_abbrev": 1, "chamber": 1, "district_code": 1, "congress": 1, "id": 1} members = db.voteview_members.find( {"icpsr": {"$in": peopleIds}, "congress": {"$in": congresses}}, memberFields) for m in members: memberSet.append(m) memberTime2 = time.time() # Now iterate through the rollcalls fieldsNeeded = [ 'party_vote_counts', 'vote_title', 'vote_desc', 'key_flags', 'yea_count', 'sponsor', 'bill_number', 'id', 'description', 'tie_breaker', 'votes', 'codes', 'dtl_desc', 'question', 'vote_description', 'short_description', 'nay_count', 'congress', 'vote_question_text', 'rollnumber', 'date', 'vote_document_text', 'nominate', 'amendment_author', 'chamber', 'vote_result', 'shortdescription', 'vote_question', 'tie_breaker', 'cg_summary', 'cg_official_titles', 'cg_short_titles_for_portions', 'dtl_sources', 'congress_url', 'clerk_rollnumber', ] rollcalls = ( db.voteview_rollcalls .find( {'id': {'$in': rollcall_ids}}, {field: 1 for field in fieldsNeeded} ) .sort('id') .batch_size(10) ) for rollcall in rollcalls: result = [] # Hold new votes output, start blank try: # If we need some people, let's iterate through the voters and fill # them out if needPeople != 0: metaMembers = [m for m in memberSet if m[ "congress"] == rollcall["congress"]] for v in rollcall["votes"]: newV = {} # Only add the person if they're in our validated list of # people we want. if v["icpsr"] in peopleIds: # print "In here" newV.update(v) # Do the match from the member list try: memberMap = next((m for m in metaMembers if m[ "icpsr"] == v["icpsr"]), None) if memberMap is None: print v["icpsr"], "Error! We don't have a member with this icpsr. Skipping" except: print v["icpsr"], "Error! We don't have a member with this icpsr. Skipping" continue # Now assemble the matching. if apitype == "Web" or apitype == "Web_Person" or apitype == "exportJSON": newV["vote"] = _get_yeanayabs(newV["cast_code"]) # We are not returning cast code. del newV["cast_code"] try: newV["x"] = memberMap["nominate"]["dim1"] except: pass try: newV["y"] = memberMap["nominate"]["dim2"] except: pass if "prob" in newV: try: newV["prob"] = int(round(newV["prob"])) except: newV["prob"] = 0 newV["name"] = memberMap["bioname"] try: newV["seo_name"] = slugify(newV["name"]) except: print "error can't slugify" print traceback.format_exc() pass newV["party"] = partyName(memberMap["party_code"]) newV["party_short_name"] = shortName( memberMap["party_code"]) newV["party_code"] = memberMap["party_code"] newV["state_abbrev"] = memberMap["state_abbrev"] if os.path.isfile("./static/img/bios/" + str(memberMap["icpsr"]).zfill(6) + ".jpg") or os.path.isfile("../static/img/bios/" + str(memberMap["icpsr"]).zfill(6) + ".jpg"): newV["img"] = str( memberMap["icpsr"]).zfill(6) + ".jpg" else: newV["img"] = "silhouette.png" if memberMap["state_abbrev"] == "USA": newV["district"] = "POTUS" elif memberMap["district_code"] > 70: newV["district"] = "%s00" % memberMap[ "state_abbrev"] elif memberMap["district_code"] and memberMap["district_code"] <= 70: newV["district"] = "%s%02d" % ( memberMap["state_abbrev"], memberMap["district_code"]) else: newV["district"] = "" # And for the R API elif apitype == "R" or apitype == "exportXLS": try: del newV["prob"] except: pass if "nominate" in memberMap and "dim1" in memberMap["nominate"]: newV["dim1"] = memberMap["nominate"]["dim1"] newV["dim2"] = memberMap["nominate"]["dim2"] newV["id"] = memberMap["id"] newV["name"] = memberMap["bioname"] newV["party_code"] = memberMap["party_code"] newV["state_abbrev"] = memberMap["state_abbrev"] newV["cqlabel"] = cqlabel( memberMap["state_abbrev"], memberMap["district_code"]) newV['district_code'] = memberMap['district_code'] # Append the new voter to the list of voters. result.append(newV) else: continue # Sort by ideology, and then identify the median and pivots if apitype == "Web": median = [] pivotCopy = [ x for x in result if "x" in x and x["x"] is not None] pivotCopy = sorted(pivotCopy, key=lambda x: x["x"]) if len(pivotCopy) % 2: median = [ pivotCopy[int(math.ceil(len(pivotCopy) / 2)) - 1]["icpsr"]] else: median = [pivotCopy[ (len(pivotCopy) / 2) - 1]["icpsr"], pivotCopy[(len(pivotCopy) / 2)]["icpsr"]] # Filibuster pivot fbPivot = [] if rollcall["chamber"] == "Senate" and rollcall["congress"] >= 94 and len(pivotCopy) >= 60: fbPivot = [pivotCopy[59]["icpsr"], pivotCopy[len(pivotCopy) - 60]["icpsr"]] # Veto Override pivot numVotes = len( [x for x in result if "vote" in x and x["vote"] != "Abs"]) voPivotNum = int( math.ceil(float(numVotes) * float(2) / float(3))) voPivot = [pivotCopy[voPivotNum]["icpsr"], pivotCopy[ len(pivotCopy) - voPivotNum - 1]["icpsr"]] for i in xrange(len(result)): if result[i]["icpsr"] in median: result[i]["flags"] = "median" if result[i]["icpsr"] in fbPivot: result[i]["flags"] = "fbPivot" if result[i]["icpsr"] in voPivot: result[i]["flags"] = "voPivot" # Top level nominate metadata # Debug code to delete nominate data so we can regenerate it. if "nominate" in rollcall and "slope" in rollcall["nominate"]: del rollcall["nominate"]["slope"] if "nominate" in rollcall and "intercept" in rollcall["nominate"]: del rollcall["nominate"]["intercept"] if "nominate" in rollcall and "x" in rollcall["nominate"]: del rollcall["nominate"]["x"] if "nominate" in rollcall and "y" in rollcall["nominate"]: del rollcall["nominate"]["y"] # Generate other nominate fields if "nominate" in rollcall and "mid" in rollcall["nominate"] and "spread" in rollcall["nominate"] and rollcall["nominate"]["spread"][0] is not None: rollcall["nominate"]["slope"], rollcall["nominate"]["intercept"], rollcall["nominate"]["x"], rollcall[ "nominate"]["y"] = add_endpoints(rollcall["nominate"]["mid"], rollcall["nominate"]["spread"]) # Ensure everything has at least some nominate field. elif "nominate" not in rollcall: rollcall["nominate"] = {} # Flatten nominate for the R API. if apitype in webexportapis: nominate = rollcall['nominate'] checkNom = ['classified', 'pre', 'log_likelihood', 'geo_mean_probability'] for f in checkNom: if f not in nominate: nominate[f] = '' elif apitype == "R": if "nominate" in rollcall: nominate = {"mid1": rollcall["nominate"]["mid"][0], "mid2": rollcall["nominate"]["mid"][1], "spread1": rollcall["nominate"]["spread"][0], "spread2": rollcall["nominate"]["spread"][1], "nomslope": rollcall['nominate']['slope'], 'nomintercept': rollcall['nominate']['intercept']} else: nominate = {} # Top level rollcall item. found[rollcall["id"]] = 1 # Get the best available description. description = waterfallText(rollcall) # Truncate the description for the R API. if apitype == "R" or apitype == "exportCSV": if len(description) <= 255: pass else: baseDesc = description[0:254] rest = description[255:] try: cutoff = rest.index(". ") if cutoff < 255: baseDesc = baseDesc + rest[0:cutoff] else: baseDesc = baseDesc + rest[0:255] + "..." except: if len(rest) < 255: baseDesc = baseDesc + rest else: baseDesc = baseDesc + rest[0:255] + "..." description = baseDesc # Get the best available question question = waterfallQuestion(rollcall) if 'vote_result' not in rollcall: rollcall['vote_result'] = None # Collapse codes for R if apitype == "exportCSV" or apitype == "exportXLS": codeFields = {"Clausen1": "", "Issue1": "", "Issue2": "", "Peltzman1": "", "Peltzman2": ""} if 'codes' in rollcall: for key, value in rollcall["codes"].iteritems(): if len(value) == 1: codeFields[key + '1'] = value[0] else: codeFields[key + '1'] = value[0] codeFields[key + '2'] = value[1] elif apitype in webexportapis: if "codes" in rollcall: codes = rollcall["codes"] else: codes = {} elif apitype == "R": if "codes" in rollcall: codes = rollcall["codes"] for key, value in codes.iteritems(): codes[key] = '; '.join(value) else: codes = {} # Pre-allocate keyvote flags. if not "key_flags" in rollcall: rollcall["key_flags"] = [] # Output object: z = {key: rollcall.get(key) for key in fieldsNeeded if key in rollcall} if "sponsor" in rollcall: z["sponsor"] = rollcall["sponsor"] if "bill_number" in rollcall: z["bill_number"] = rollcall["bill_number"] if "tie_breaker" in rollcall: z["tie_breaker"] = rollcall["tie_breaker"] if apitype == "exportCSV" or apitype == "exportXLS": z.update({k: v for k, v in codeFields.iteritems()}) z.update({'keyvote': ''.join(rollcall['key_flags']), 'spread.dim1': nominate['spread'][0], 'spread.dim2': nominate['spread'][1], 'mid.dim1': nominate['mid'][0], 'mid.dim2': nominate['mid'][1], 'slope': nominate['slope'], 'intercept': nominate['intercept'], 'log_likelihood': round(nominate['log_likelihood'], 5), 'classified': nominate['classified'], 'pre': nominate['pre'], 'question': None if not question else question.encode('utf-8'), 'description': description.encode('utf-8'), 'geo_mean_probability': None if nominate['geo_mean_probability'] == '' else round(nominate['geo_mean_probability'], 3)}) if apitype != "exportCSV": z.update({'key_flags': rollcall["key_flags"], 'votes': result, 'codes': codes, 'nominate': nominate, 'description': description, 'question': question}) # Get other people's results from the party results aggregate. if apitype == "Web_Person": z["party_vote_counts"] = rollcall["party_vote_counts"] rollcall_results.append(z) except: # Invalid vote id print traceback.format_exc() errormessage = "Invalid Rollcall ID specified." errormeta.append(str(rollcall["id"])) if rollcalls.count() != len(rollcall_ids): errormessage = "Invalid Rollcall ID specified." for voteID, num in found.iteritems(): if num == 0: errormeta.append(str(voteID)) endtime = time.time() # print round(setupTime-starttime,2), round(memberTime1-setupTime,2), # round(memberTime2-memberTime1,2), round(endtime-memberTime2,2) response = {} if len(rollcall_results): response["rollcalls"] = rollcall_results if len(errormessage): response["errormessage"] = errormessage if len(errormeta): response["errormeta"] = errormeta response["apitype"] = apiVersion response["elapsedTime"] = round(endtime - starttime, 3) return response