def memberLookup(qDict, maxResults=50, distinct=0, api="Web"):
# Setup so that the bottle call to this API doesn't need to know parameters we accept explicitly
name = qDict["name"] if "name" in qDict else ""
icpsr = qDict["icpsr"] if "icpsr" in qDict else ""
state_abbrev = qDict["state_abbrev"] if "state_abbrev" in qDict else ""
congress = qDict["congress"] if "congress" in qDict else ""
chamber = qDict["chamber"] if "chamber" in qDict else ""
party_code = qDict["party_code"] if "party_code" in qDict else ""
bioguide_id = qDict["bioguide_id"] if "bioguide_id" in qDict else ""
district_code = qDict["district_code"] if "district_code" in qDict else ""
id = qDict["id"] if "id" in qDict else ""
speaker = qDict["speaker"] if "speaker" in qDict else ""
freshman = qDict["freshman"] if "freshman" in qDict else ""
idIn = qDict["idIn"] if "idIn" in qDict else []
biography = qDict["biography"] if "biography" in qDict else ""
if api == "R":
maxResults = 5000
# Check to make sure there's a query
if not name and not icpsr and not state_abbrev and not congress and not district_code and not chamber and not id and not party_code and not bioguide_id and not speaker and not idIn and not freshman and not biography:
return({'errormessage': 'No search terms provided'})
# Fold search query into dict
searchQuery = {}
if api=="districtLookup":
searchQuery["id"] = {"$in": qDict["idIn"]}
if icpsr:
try:
icpsr = int(icpsr)
searchQuery["icpsr"] = icpsr
except:
try:
if icpsr[0]=="M":
for r in db.voteview_members.find({'id': icpsr}, {'icpsr': 1, '_id': 0}):
searchQuery["icpsr"] = r["icpsr"]
break
except:
return({"errormessage": "Invalid ICPSR number supplied."})
if id:
try:
if id.upper().startswith("MH") or id.upper().startswith("MS"):
searchQuery["id"] = id
else:
return({"errormessage": "Invalid ID supplied1."})
except:
return({"errormessage": "Invalid ID supplied2."})
if state_abbrev:
state = str(state_abbrev)
if len(state) == 2 or state.upper() == "USA":
searchQuery["state_abbrev"] = state.upper() # States are all stored upper-case
else:
searchQuery["state_abbrev"] = stateNameToAbbrev(state.upper())
if biography:
searchQuery["biography"] = {"$regex": biography, "$options": i}
if congress:
try:
if isinstance(congress, int): # already a number
searchQuery["congress"] = congress
elif not " " in congress: # congress is just a number
congress = int(congress)
searchQuery["congress"] = congress
elif "[" in congress and "]" in congress and "to" in congress: # congress is a range
valText = congress[1:-1]
min, maxC = valText.split(" to ")
searchQuery["congress"] = {}
if len(min):
searchQuery["congress"]["$gte"] = int(min) # From min
if len(maxC):
searchQuery["congress"]["$lte"] = int(maxC) # To max
else: # congress is a series of integers, use $in
vals = [int(val) for val in congress.split(" ")]
searchQuery["congress"] = {}
searchQuery["congress"]["$in"] = vals
except:
print traceback.format_exc()
return({"errormessage": "Invalid congress ID supplied."})
if name:
if ", " in name: # Last, First
last, rest = name.split(", ",1)
searchQuery["$text"] = {"$search": rest+" "+last}
else:
searchQuery["$text"] = {"$search": name}
if speaker:
searchQuery["served_as_speaker"] = 1
if freshman:
try:
maxCongress = json.load(open("static/config.json","r"))["maxCongress"]
except:
try:
maxCongress = json.load(open("../static/config.json","r"))["maxCongress"]
except:
maxCongress = 116
searchQuery["congresses.0.0"] = maxCongress
if party_code:
if isinstance(party_code, dict):
searchQuery["party_code"] = party_code
else:
searchQuery["party_code"] = int(party_code)
if bioguide_id:
searchQuery["bioguide_id"] = bioguide_id
if district_code and state_abbrev:
searchQuery["district_code"] = district_code
if chamber:
chamber = chamber.capitalize()
if chamber=="Senate" or chamber=="House" or chamber=="President":
searchQuery["chamber"] = chamber
else:
return({"errormessage": "Invalid chamber provided. Please select House or Senate."})
response = []
errormessage = ""
i = 0
# Field return specifications, allows us to return less than all our data to searches.
if api=="Web_PI":
fieldSet = {"nominate.dim1": 1, "party_code": 1, "district_code": 1, "icpsr": 1, "chamber":1, "nvotes_yea_nay": 1, "nvotes_against_party": 1, "nvotes_abs": 1, "_id": 0}
elif api=="Web_FP_Search":
fieldSet = {"bioname": 1, "party_code": 1, "icpsr": 1, "state_abbrev": 1, "congress": 1, "_id": 0, "congresses": 1, "chamber": 1, "bioguide_id": 1}
elif api == "Check_Party_Switch":
fieldSet = {"icpsr": 1, "_id": 0}
elif api=="Web_Congress":
if chamber:
fieldName = "elected_"+chamber.lower()
fieldSet = {"bioname": 1, "party_code": 1, "icpsr": 1, "state_abbrev": 1, "congress": 1, "_id": 0, "bioImgURL": 1, "minElected": 1, "nominate.dim1": 1, "nominate.dim2": 1, "congresses": 1, fieldName: 1}
else:
fieldSet = {"bioname": 1, "party_code": 1, "icpsr": 1, "state_abbrev": 1, "congress": 1, "_id": 0, "bioImgURL": 1, "minElected": 1, "nominate.dim1": 1, "nominate.dim2": 1, "congresses": 1, "state_abbrev": 1, "elected_senate": 1, "elected_house": 1}
elif api=="Web_Party":
fieldSet = {"bioname": 1, "party_code": 1, "icpsr": 1, "state_abbrev": 1, "congress": 1, "_id": 0, "bioImgURL": 1, "minElected": 1, "nominate.dim1": 1, "nominate.dim2": 1, "congresses": 1, "chamber": 1}
elif api=="R":
fieldSet = {"bioname": 1, "party_code": 1, "icpsr": 1, "state_abbrev": 1, "congress": 1, "id": 1, "_id": 0, "nominate.dim1": 1, "nominate.dim2": 1, "nominate.geo_mean_probability": 1, "cqlabel": 1, "district_code": 1, "chamber": 1, "congresses": 1}
elif api=="exportCSV" or api == "exportORD":
fieldSet = {"bioname": 1, "party_code": 1, "icpsr": 1, "state_abbrev": 1, "congress": 1, "id": 1, "_id": 0, "nominate": 1, "district_code": 1, "chamber": 1, "state_name_trunc": 1, "last_means": 1, "occupancy": 1, "name": 1}
elif api=="districtLookup":
fieldSet = {"bioname": 1, "party_code": 1, "icpsr": 1, "state_abbrev": 1, "congress": 1, "id": 1, "nominate.dim1": 1, "nominate.dim2": 1, "district_code": 1, "_id": 0, "chamber": 1, "congresses": 1}
else:
fieldSet = {"_id": 0, "personid": 0}
if "$text" in searchQuery:
fieldSet["score"] = {"$meta": "textScore"}
res = db.voteview_members.find(searchQuery, fieldSet)
# Try to induce regex if a name search fails?
hypotheticalCount = res.count()
if "$text" in searchQuery and hypotheticalCount==0:
print "No results from a name search, fall back to regex"
del searchQuery["$text"]
searchQuery["bioname"] = {'$regex': name, '$options': 'i'}
res = db.voteview_members.find(searchQuery, fieldSet)
if "$text" in searchQuery:
sortedRes = res.sort([('score', {'$meta': 'textScore'})])
elif api=="exportORD":
db.voteview_members.ensure_index([('state_abbrev', 1), ('district_code', 1), ('icpsr', 1)], name="ordIndex")
sortedRes = res.sort([('state_abbrev', 1), ('district_code', 1), ('icpsr', 1)])
else:
sortedRes = res.sort('congress', -1)
if sortedRes.count()>1000 and api != "R" and api!= "Web_Party":
return({"errormessage": "Too many results found."})
elif sortedRes.count()>5000 and api!= "Web_Party":
return({"errormessage": "Too many results found."})
currentICPSRs = []
for m in sortedRes:
if m["icpsr"] in currentICPSRs and distinct==1:
continue
else:
currentICPSRs.append(m["icpsr"])
newM = m
if "state_abbrev" in newM:
newM["state"] = stateName(newM["state_abbrev"])
if api=="exportORD":
newM["state_icpsr"] = stateIcpsr(newM["state_abbrev"])
if "district_code" in newM and "state_abbrev" in newM:
newM["cqlabel"] = cqlabel(newM["state_abbrev"], newM["district_code"])
if "party_code" in newM:
newM["party_name"] = partyName(newM["party_code"])
if api not in ["exportORD", "exportCSV", "R"]:
newM["party_noun"] = noun(newM["party_code"])
newM["party_color"] = partyColor(newM["party_code"])
newM["party_short_name"] = shortName(newM["party_code"])
# Check if an image exists.
if os.path.isfile("/var/www/voteview/static/img/bios/"+str(newM["icpsr"]).zfill(6)+".jpg"):
newM["bioImgURL"] = str(newM["icpsr"]).zfill(6)+".jpg"
else:
newM["bioImgURL"] = "silhouette.png"
if api in ["exportCSV", "exportORD"]:
if 'bioname' in newM:
newM['bioname'] = newM['bioname'].encode('utf-8')
if "nominate" in newM:
for k,v in newM["nominate"].iteritems():
if k == 'log_likelihood':
newM[k] = round(v, 5)
else:
newM[k] = round(v, 3)
del newM["nominate"]
try:
newM["seo_name"] = slugify(newM["bioname"])
except:
pass
response.append(newM)
i=i+1
if i>=maxResults:
break
if len(response)>maxResults and maxResults>1: # For regular searches get mad if we have more than max results.
errormessage = "Capping number of responses at "+str(maxResults)+"."
if len(response)==0:
return({'errormessage': 'No members found matching your search query.', 'query': qDict})
elif errormessage:
return({'errormessage': errormessage, 'results': response})
else:
return({'results': response})
def downloadAPI(rollcall_id, apitype="Web", voterId=0):
starttime = time.time()
# Setup API version response
webexportapis = ["Web", "Web_Person", "exportJSON", "exportCSV"]
if apitype in webexportapis:
apiVersion = "Web 2016-10"
elif apitype=="R":
apiVersion = "R 2016-10"
if not rollcall_id or len(rollcall_id)==0:
response = {'errormessage': 'No rollcall id specified.', 'apitype': apiVersion}
return response
# Split multiple ID requests into list
if type(rollcall_id)==type([""]):
rollcall_ids = rollcall_id
elif "," in rollcall_id:
rollcall_ids = [x.strip() for x in rollcall_id.split(",")]
else:
rollcall_ids = [rollcall_id]
# Abuse filter
maxVotes = 100
if apitype in ["exportJSON", "exportCSV"]:
maxVotes = 500
if len(rollcall_ids)>maxVotes:
response = {'errormessage': 'API abuse. Too many votes.', 'apitype': apiVersion}
return response
rollcall_results = [] # Stores top level rollcall results, one per vote
errormessage = "" # String storing error text
errormeta = [] # List storing failed IDs
found = {}
for rid in rollcall_ids:
found[rid] = 0
setupTime = time.time()
# Do we need to fold in members?
peopleIds = []
memberSet = []
if apitype=="Web_Person": # I need to fold in one specific member
needPeople=-1 # Just one specific member
peopleIds = [voterId]
elif apitype=="exportCSV":
needPeople=0 # No members at all
else:
needPeople=1
peopleIds = db.voteview_rollcalls.distinct("votes.icpsr", {"id": {"$in": rollcall_ids}}) # All relevant members
congresses = []
for rollcall_id in rollcall_ids:
try:
congresses.append(int(rollcall_id[2:5]))
except:
pass
memberTime1 = time.time()
# Now fetch the members
memberSet = []
if len(peopleIds):
memberFields = {"icpsr":1, "nominate":1, "bioname":1, "party_code":1, "state_abbrev":1, "chamber":1, "district_code": 1, "congress": 1, "id":1}
members = db.voteview_members.find({"icpsr": {"$in": peopleIds}, "congress": {"$in": congresses}}, memberFields)
for m in members:
memberSet.append(m)
memberTime2 = time.time()
# Now iterate through the rollcalls
fieldSetNeed = {"votes": 1, "nominate": 1, "id": 1, "codes": 1, "key_flags": 1, "yea_count": 1, "nay_count": 1, "congress": 1, "chamber": 1, "rollnumber": 1, "date": 1, "vote_desc": 1, "vote_document_text": 1, "description": 1, "shortdescription": 1, "short_description": 1, "vote_question": 1, "question": 1, "party_vote_counts": 1, 'vote_result': 1, 'vote_title':1, 'vote_question_text':1, 'amendment_author':1}
rollcalls = db.voteview_rollcalls.find({'id': {'$in': rollcall_ids}}, fieldSetNeed).sort('id')
for rollcall in rollcalls:
result = [] # Hold new votes output, start blank
try:
# If we need some people, let's iterate through the voters and fill them out
if needPeople!=0:
metaMembers = [m for m in memberSet if m["congress"] == rollcall["congress"]]
for v in rollcall["votes"]:
newV = {}
# Only add the person if they're in our validated list of people we want.
if v["icpsr"] in peopleIds:
#print "In here"
newV.update(v)
# Do the match from the member list
try:
memberMap = next((m for m in metaMembers if m["icpsr"]==v["icpsr"]), None)
if memberMap is None:
print v["icpsr"], "Error! We don't have a member with this icpsr. Skipping"
except:
print v["icpsr"], "Error! We don't have a member with this icpsr. Skipping"
continue
# Now assemble the matching.
if apitype=="Web" or apitype=="Web_Person" or apitype=="exportJSON":
newV["vote"] = _get_yeanayabs(newV["cast_code"])
del newV["cast_code"] # We are not returning cast code.
try:
newV["x"] = memberMap["nominate"]["dim1"]
except:
pass
try:
newV["y"] = memberMap["nominate"]["dim2"]
except:
pass
if "prob" in newV:
newV["prob"] = int(round(newV["prob"]))
newV["name"] = memberMap["bioname"]
newV["party"] = partyName(memberMap["party_code"])
newV["party_short_name"] = shortName(memberMap["party_code"])
newV["party_code"] = memberMap["party_code"]
newV["state_abbrev"] = memberMap["state_abbrev"]
if memberMap["state_abbrev"] == "USA":
newV["district"] = "POTUS"
elif memberMap["district_code"] > 70:
newV["district"] = "%s00" % memberMap["state_abbrev"]
elif memberMap["district_code"] and memberMap["district_code"] <= 70:
newV["district"] = "%s%02d" % (memberMap["state_abbrev"], memberMap["district_code"])
else:
newV["district"] = ""
# And for the R API
elif apitype=="R" or apitype=="exportXLS":
try:
del newV["prob"]
except:
pass
if "nominate" in memberMap and "dim1" in memberMap["nominate"]:
newV["dim1"] = memberMap["nominate"]["dim1"]
newV["dim2"] = memberMap["nominate"]["dim2"]
newV["id"] = memberMap["id"]
newV["name"] = memberMap["bioname"]
newV["party_code"] = memberMap["party_code"]
newV["state_abbrev"] = memberMap["state_abbrev"]
newV["cqlabel"] = cqlabel(memberMap["state_abbrev"], memberMap["district_code"])
newV['district_code'] = memberMap['district_code']
# Append the new voter to the list of voters.
result.append(newV)
else:
continue
# Top level nominate metadata
# Debug code to delete nominate data so we can regenerate it.
if "nominate" in rollcall and "slope" in rollcall["nominate"]:
del rollcall["nominate"]["slope"]
if "nominate" in rollcall and "intercept" in rollcall["nominate"]:
del rollcall["nominate"]["intercept"]
if "nominate" in rollcall and "x" in rollcall["nominate"]:
del rollcall["nominate"]["x"]
if "nominate" in rollcall and "y" in rollcall["nominate"]:
del rollcall["nominate"]["y"]
# Generate other nominate fields
if "nominate" in rollcall and "mid" in rollcall["nominate"] and "spread" in rollcall["nominate"] and rollcall["nominate"]["spread"][0] is not None:
rollcall["nominate"]["slope"], rollcall["nominate"]["intercept"], rollcall["nominate"]["x"], rollcall["nominate"]["y"] = add_endpoints(rollcall["nominate"]["mid"], rollcall["nominate"]["spread"])
# Ensure everything has at least some nominate field.
elif "nominate" not in rollcall:
rollcall["nominate"] = {}
# Flatten nominate for the R API.
if apitype in webexportapis:
nominate = rollcall['nominate']
checkNom = ['classified', 'pre', 'log_likelihood']
for f in checkNom:
if f not in nominate:
nominate[f] = ''
elif apitype=="R":
if "nominate" in rollcall:
nominate = {"mid1": rollcall["nominate"]["mid"][0], "mid2": rollcall["nominate"]["mid"][1],
"spread1": rollcall["nominate"]["spread"][0], "spread2": rollcall["nominate"]["spread"][1],
"nomslope": rollcall['nominate']['slope'], 'nomintercept': rollcall['nominate']['intercept']}
else:
nominate = {}
# Top level rollcall item.
found[rollcall["id"]] = 1
# Get the best available description.
description = waterfallText(rollcall)
# Truncate the description for the R API.
if apitype=="R" or apitype=="exportCSV":
if len(description)<=255:
pass
else:
baseDesc = description[0:254]
rest = description[255:]
try:
cutoff = rest.index(". ")
if cutoff<255:
baseDesc = baseDesc + rest[0:cutoff]
else:
baseDesc = baseDesc + rest[0:255]+"..."
except:
if len(rest)<255:
baseDesc = baseDesc+rest
else:
baseDesc = baseDesc + rest[0:255]+"..."
description = baseDesc
# Get the best available question
question = waterfallQuestion(rollcall)
if 'vote_result' not in rollcall:
rollcall['vote_result'] = None
# Collapse codes for R
if apitype == "exportCSV" or apitype == "exportXLS":
codeFields = {"Clausen1": "" , "Issue1": "", "Issue2": "", "Peltzman1": "", "Peltzman2": ""}
if 'codes' in rollcall:
for key, value in rollcall["codes"].iteritems():
if len(value) == 1:
codeFields[key + '1'] = value[0]
else:
codeFields[key + '1'] = value[0]
codeFields[key + '2'] = value[1]
elif apitype in webexportapis:
if "codes" in rollcall:
codes = rollcall["codes"]
else:
codes = {}
elif apitype=="R":
if "codes" in rollcall:
codes = rollcall["codes"]
for key, value in codes.iteritems():
codes[key] = '; '.join(value)
else:
codes = {}
# Pre-allocate keyvote flags.
if not "key_flags" in rollcall:
rollcall["key_flags"] = []
# Output object:
z = {'id': rollcall['id'], 'chamber': rollcall['chamber'], 'congress': rollcall['congress'], 'date': rollcall['date'],
'rollnumber': rollcall['rollnumber'], 'yea': rollcall["yea_count"],
'nay': rollcall["nay_count"], 'vote_result': rollcall['vote_result']}
if apitype == "exportCSV" or apitype == "exportXLS":
z.update({k:v for k,v in codeFields.iteritems()})
z.update({'keyvote': ''.join(rollcall['key_flags']),
'spread.dim1': nominate['spread'][0], 'spread.dim2': nominate['spread'][1],
'mid.dim1': nominate['mid'][0], 'mid.dim2': nominate['mid'][1],
'slope': nominate['slope'], 'intercept': nominate['intercept'],
'log_likelihood': nominate['log_likelihood'], 'classified': nominate['classified'], 'pre': nominate['pre'], 'description': description.encode('utf-8')})
if apitype != "exportCSV":
z.update({'key_flags': rollcall["key_flags"], 'votes': result, 'codes': codes, 'nominate': nominate, 'description': description, 'question': question})
# Get other people's results from the party results aggregate.
if apitype=="Web_Person":
z["party_vote_counts"] = rollcall["party_vote_counts"]
rollcall_results.append(z)
except: # Invalid vote id
print traceback.format_exc()
errormessage = "Invalid Rollcall ID specified."
errormeta.append(str(rollcall["id"]))
if rollcalls.count() != len(rollcall_ids):
errormessage = "Invalid Rollcall ID specified."
for voteID, num in found.iteritems():
if num==0:
errormeta.append(str(voteID))
endtime = time.time()
#print round(setupTime-starttime,2), round(memberTime1-setupTime,2), round(memberTime2-memberTime1,2), round(endtime-memberTime2,2)
response = {}
if len(rollcall_results):
response["rollcalls"] = rollcall_results
if len(errormessage):
response["errormessage"] = errormessage
if len(errormeta):
response["errormeta"] = errormeta
response["apitype"] = apiVersion
response["elapsedTime"] = round(endtime - starttime,3)
return response
def downloadAPI(rollcall_id, apitype="Web", voterId=0):
starttime = time.time()
# Setup API version response
webexportapis = ["Web", "Web_Person", "exportJSON", "exportCSV"]
if apitype in webexportapis:
apiVersion = "Web 2016-10"
elif apitype == "R":
apiVersion = "R 2016-10"
if not rollcall_id or len(rollcall_id) == 0:
response = {'errormessage': 'No rollcall id specified.',
'apitype': apiVersion}
return response
# Split multiple ID requests into list
if type(rollcall_id) == type([""]):
rollcall_ids = rollcall_id
elif "," in rollcall_id:
rollcall_ids = [x.strip() for x in rollcall_id.split(",")]
else:
rollcall_ids = [rollcall_id]
# Abuse filter
maxVotes = 100
if apitype in ["exportJSON", "exportCSV"]:
maxVotes = 500
if len(rollcall_ids) > maxVotes:
response = {'errormessage': 'API abuse. Too many votes.',
'apitype': apiVersion}
return response
rollcall_results = [] # Stores top level rollcall results, one per vote
errormessage = "" # String storing error text
errormeta = [] # List storing failed IDs
found = {}
for rid in rollcall_ids:
found[rid] = 0
setupTime = time.time()
# Do we need to fold in members?
peopleIds = []
memberSet = []
if apitype == "Web_Person": # I need to fold in one specific member
needPeople = -1 # Just one specific member
peopleIds = [voterId]
elif apitype == "exportCSV":
needPeople = 0 # No members at all
else:
needPeople = 1
peopleIds = db.voteview_rollcalls.distinct(
"votes.icpsr", {"id": {"$in": rollcall_ids}}) # All relevant members
congresses = []
for rollcall_id in rollcall_ids:
try:
congresses.append(int(rollcall_id[2:5]))
except:
pass
memberTime1 = time.time()
# Now fetch the members
memberSet = []
if len(peopleIds):
memberFields = {"icpsr": 1, "nominate": 1, "bioname": 1, "party_code": 1,
"state_abbrev": 1, "chamber": 1, "district_code": 1, "congress": 1, "id": 1}
members = db.voteview_members.find(
{"icpsr": {"$in": peopleIds}, "congress": {"$in": congresses}}, memberFields)
for m in members:
memberSet.append(m)
memberTime2 = time.time()
# Now iterate through the rollcalls
fieldsNeeded = [
'party_vote_counts', 'vote_title',
'vote_desc', 'key_flags', 'yea_count', 'sponsor',
'bill_number', 'id', 'description', 'tie_breaker', 'votes',
'codes', 'dtl_desc', 'question', 'vote_description',
'short_description', 'nay_count', 'congress',
'vote_question_text', 'rollnumber', 'date',
'vote_document_text', 'nominate', 'amendment_author',
'chamber', 'vote_result', 'shortdescription', 'vote_question',
'tie_breaker', 'cg_summary', 'cg_official_titles',
'cg_short_titles_for_portions', 'dtl_sources', 'congress_url',
'clerk_rollnumber',
]
rollcalls = (
db.voteview_rollcalls
.find(
{'id': {'$in': rollcall_ids}},
{field: 1 for field in fieldsNeeded}
)
.sort('id')
.batch_size(10)
)
for rollcall in rollcalls:
result = [] # Hold new votes output, start blank
try:
# If we need some people, let's iterate through the voters and fill
# them out
if needPeople != 0:
metaMembers = [m for m in memberSet if m[
"congress"] == rollcall["congress"]]
for v in rollcall["votes"]:
newV = {}
# Only add the person if they're in our validated list of
# people we want.
if v["icpsr"] in peopleIds:
# print "In here"
newV.update(v)
# Do the match from the member list
try:
memberMap = next((m for m in metaMembers if m[
"icpsr"] == v["icpsr"]), None)
if memberMap is None:
print v["icpsr"], "Error! We don't have a member with this icpsr. Skipping"
except:
print v["icpsr"], "Error! We don't have a member with this icpsr. Skipping"
continue
# Now assemble the matching.
if apitype == "Web" or apitype == "Web_Person" or apitype == "exportJSON":
newV["vote"] = _get_yeanayabs(newV["cast_code"])
# We are not returning cast code.
del newV["cast_code"]
try:
newV["x"] = memberMap["nominate"]["dim1"]
except:
pass
try:
newV["y"] = memberMap["nominate"]["dim2"]
except:
pass
if "prob" in newV:
try:
newV["prob"] = int(round(newV["prob"]))
except:
newV["prob"] = 0
newV["name"] = memberMap["bioname"]
try:
newV["seo_name"] = slugify(newV["name"])
except:
print "error can't slugify"
print traceback.format_exc()
pass
newV["party"] = partyName(memberMap["party_code"])
newV["party_short_name"] = shortName(
memberMap["party_code"])
newV["party_code"] = memberMap["party_code"]
newV["state_abbrev"] = memberMap["state_abbrev"]
if os.path.isfile("./static/img/bios/" + str(memberMap["icpsr"]).zfill(6) + ".jpg") or os.path.isfile("../static/img/bios/" + str(memberMap["icpsr"]).zfill(6) + ".jpg"):
newV["img"] = str(
memberMap["icpsr"]).zfill(6) + ".jpg"
else:
newV["img"] = "silhouette.png"
if memberMap["state_abbrev"] == "USA":
newV["district"] = "POTUS"
elif memberMap["district_code"] > 70:
newV["district"] = "%s00" % memberMap[
"state_abbrev"]
elif memberMap["district_code"] and memberMap["district_code"] <= 70:
newV["district"] = "%s%02d" % (
memberMap["state_abbrev"], memberMap["district_code"])
else:
newV["district"] = ""
# And for the R API
elif apitype == "R" or apitype == "exportXLS":
try:
del newV["prob"]
except:
pass
if "nominate" in memberMap and "dim1" in memberMap["nominate"]:
newV["dim1"] = memberMap["nominate"]["dim1"]
newV["dim2"] = memberMap["nominate"]["dim2"]
newV["id"] = memberMap["id"]
newV["name"] = memberMap["bioname"]
newV["party_code"] = memberMap["party_code"]
newV["state_abbrev"] = memberMap["state_abbrev"]
newV["cqlabel"] = cqlabel(
memberMap["state_abbrev"], memberMap["district_code"])
newV['district_code'] = memberMap['district_code']
# Append the new voter to the list of voters.
result.append(newV)
else:
continue
# Sort by ideology, and then identify the median and pivots
if apitype == "Web":
median = []
pivotCopy = [
x for x in result if "x" in x and x["x"] is not None]
pivotCopy = sorted(pivotCopy, key=lambda x: x["x"])
if len(pivotCopy) % 2:
median = [
pivotCopy[int(math.ceil(len(pivotCopy) / 2)) - 1]["icpsr"]]
else:
median = [pivotCopy[
(len(pivotCopy) / 2) - 1]["icpsr"], pivotCopy[(len(pivotCopy) / 2)]["icpsr"]]
# Filibuster pivot
fbPivot = []
if rollcall["chamber"] == "Senate" and rollcall["congress"] >= 94 and len(pivotCopy) >= 60:
fbPivot = [pivotCopy[59]["icpsr"],
pivotCopy[len(pivotCopy) - 60]["icpsr"]]
# Veto Override pivot
numVotes = len(
[x for x in result if "vote" in x and x["vote"] != "Abs"])
voPivotNum = int(
math.ceil(float(numVotes) * float(2) / float(3)))
voPivot = [pivotCopy[voPivotNum]["icpsr"], pivotCopy[
len(pivotCopy) - voPivotNum - 1]["icpsr"]]
for i in xrange(len(result)):
if result[i]["icpsr"] in median:
result[i]["flags"] = "median"
if result[i]["icpsr"] in fbPivot:
result[i]["flags"] = "fbPivot"
if result[i]["icpsr"] in voPivot:
result[i]["flags"] = "voPivot"
# Top level nominate metadata
# Debug code to delete nominate data so we can regenerate it.
if "nominate" in rollcall and "slope" in rollcall["nominate"]:
del rollcall["nominate"]["slope"]
if "nominate" in rollcall and "intercept" in rollcall["nominate"]:
del rollcall["nominate"]["intercept"]
if "nominate" in rollcall and "x" in rollcall["nominate"]:
del rollcall["nominate"]["x"]
if "nominate" in rollcall and "y" in rollcall["nominate"]:
del rollcall["nominate"]["y"]
# Generate other nominate fields
if "nominate" in rollcall and "mid" in rollcall["nominate"] and "spread" in rollcall["nominate"] and rollcall["nominate"]["spread"][0] is not None:
rollcall["nominate"]["slope"], rollcall["nominate"]["intercept"], rollcall["nominate"]["x"], rollcall[
"nominate"]["y"] = add_endpoints(rollcall["nominate"]["mid"], rollcall["nominate"]["spread"])
# Ensure everything has at least some nominate field.
elif "nominate" not in rollcall:
rollcall["nominate"] = {}
# Flatten nominate for the R API.
if apitype in webexportapis:
nominate = rollcall['nominate']
checkNom = ['classified', 'pre',
'log_likelihood', 'geo_mean_probability']
for f in checkNom:
if f not in nominate:
nominate[f] = ''
elif apitype == "R":
if "nominate" in rollcall:
nominate = {"mid1": rollcall["nominate"]["mid"][0], "mid2": rollcall["nominate"]["mid"][1],
"spread1": rollcall["nominate"]["spread"][0], "spread2": rollcall["nominate"]["spread"][1],
"nomslope": rollcall['nominate']['slope'], 'nomintercept': rollcall['nominate']['intercept']}
else:
nominate = {}
# Top level rollcall item.
found[rollcall["id"]] = 1
# Get the best available description.
description = waterfallText(rollcall)
# Truncate the description for the R API.
if apitype == "R" or apitype == "exportCSV":
if len(description) <= 255:
pass
else:
baseDesc = description[0:254]
rest = description[255:]
try:
cutoff = rest.index(". ")
if cutoff < 255:
baseDesc = baseDesc + rest[0:cutoff]
else:
baseDesc = baseDesc + rest[0:255] + "..."
except:
if len(rest) < 255:
baseDesc = baseDesc + rest
else:
baseDesc = baseDesc + rest[0:255] + "..."
description = baseDesc
# Get the best available question
question = waterfallQuestion(rollcall)
if 'vote_result' not in rollcall:
rollcall['vote_result'] = None
# Collapse codes for R
if apitype == "exportCSV" or apitype == "exportXLS":
codeFields = {"Clausen1": "", "Issue1": "",
"Issue2": "", "Peltzman1": "", "Peltzman2": ""}
if 'codes' in rollcall:
for key, value in rollcall["codes"].iteritems():
if len(value) == 1:
codeFields[key + '1'] = value[0]
else:
codeFields[key + '1'] = value[0]
codeFields[key + '2'] = value[1]
elif apitype in webexportapis:
if "codes" in rollcall:
codes = rollcall["codes"]
else:
codes = {}
elif apitype == "R":
if "codes" in rollcall:
codes = rollcall["codes"]
for key, value in codes.iteritems():
codes[key] = '; '.join(value)
else:
codes = {}
# Pre-allocate keyvote flags.
if not "key_flags" in rollcall:
rollcall["key_flags"] = []
# Output object:
z = {key: rollcall.get(key)
for key in fieldsNeeded if key in rollcall}
if "sponsor" in rollcall:
z["sponsor"] = rollcall["sponsor"]
if "bill_number" in rollcall:
z["bill_number"] = rollcall["bill_number"]
if "tie_breaker" in rollcall:
z["tie_breaker"] = rollcall["tie_breaker"]
if apitype == "exportCSV" or apitype == "exportXLS":
z.update({k: v for k, v in codeFields.iteritems()})
z.update({'keyvote': ''.join(rollcall['key_flags']),
'spread.dim1': nominate['spread'][0], 'spread.dim2': nominate['spread'][1],
'mid.dim1': nominate['mid'][0], 'mid.dim2': nominate['mid'][1],
'slope': nominate['slope'], 'intercept': nominate['intercept'],
'log_likelihood': round(nominate['log_likelihood'], 5),
'classified': nominate['classified'], 'pre': nominate['pre'],
'question': None if not question else question.encode('utf-8'),
'description': description.encode('utf-8'),
'geo_mean_probability': None if nominate['geo_mean_probability'] == '' else round(nominate['geo_mean_probability'], 3)})
if apitype != "exportCSV":
z.update({'key_flags': rollcall["key_flags"], 'votes': result, 'codes': codes,
'nominate': nominate, 'description': description, 'question': question})
# Get other people's results from the party results aggregate.
if apitype == "Web_Person":
z["party_vote_counts"] = rollcall["party_vote_counts"]
rollcall_results.append(z)
except: # Invalid vote id
print traceback.format_exc()
errormessage = "Invalid Rollcall ID specified."
errormeta.append(str(rollcall["id"]))
if rollcalls.count() != len(rollcall_ids):
errormessage = "Invalid Rollcall ID specified."
for voteID, num in found.iteritems():
if num == 0:
errormeta.append(str(voteID))
endtime = time.time()
# print round(setupTime-starttime,2), round(memberTime1-setupTime,2),
# round(memberTime2-memberTime1,2), round(endtime-memberTime2,2)
response = {}
if len(rollcall_results):
response["rollcalls"] = rollcall_results
if len(errormessage):
response["errormessage"] = errormessage
if len(errormeta):
response["errormeta"] = errormeta
response["apitype"] = apiVersion
response["elapsedTime"] = round(endtime - starttime, 3)
return response
