def save_signature(i, pid, uid):
where = 'petition_id=$pid AND user_id=$uid'
signed = db.select('signatory', where=where, vars=locals())
share_with = (i.get('share_with', 'off') == 'on' and 'N') or 'A'
update_user_details(i)
if not signed:
referrer = get_referrer(pid, uid)
signid = db.insert('signatory',
user_id=uid,
share_with=share_with,
petition_id=pid,
comment=i.get('comment'),
referrer=referrer)
helpers.set_msg(
"Thanks for your signing! Why don't you tell your friends about it now?"
)
return signid
else:
db.update('signatory',
where='user_id=$uid and petition_id=$pid',
comment=i.get('comment'),
deleted=None,
vars=locals())
helpers.set_msg(
"Your signature has been changed. Why don't you tell your friends about it now?"
)
return 'old_%s' % signed[0].id
def POST_password(self, uid):
user = db.select('users',
what='password',
where='id=$uid',
vars=locals())[0]
form = get_password_form(user)
set_passwd_form = 'curr_password' not in [
inp.name for inp in list(form.inputs)
]
i = web.input()
if form.validates(i):
if set_passwd_form or auth.check_password(user, i.curr_password):
enc_password = auth.encrypt_password(i.password)
db.update('users',
password=enc_password,
verified=True,
where='id=$uid',
vars=locals())
helpers.set_msg('Password %s.' %
('saved' if set_passwd_form else 'changed'))
raise web.seeother('/%s/preferences' % uid)
else:
helpers.set_msg('Invalid Password', 'error')
form.note = 'Current Password invalid.'
form.valid = False
return self.GET(uid, password_form=form)
def main():
assert os.path.exists(ALMANAC_DIR), ALMANAC_DIR
files = glob.glob(ALMANAC_DIR + 'people/*/rep_*.htm') + \
glob.glob(ALMANAC_DIR + 'people/*/*s[12].htm')
files.sort()
for fn in files:
district = web.storage()
demog = None
dist = web.lstrips(web.rstrips(fn.split('/')[-1], '.htm'), 'rep_')
diststate = dist[0:2].upper()
distnum = dist[-2:]
distname = tools.fixdist(diststate + '-' + distnum)
d = almanac.scrape_person(fn)
load_election_results(d, distname)
if 'demographics' in d:
demog = d['demographics']
elif distname[-2:] == '00' or '-' not in distname: # if -00 then this district is the same as the state.
#print "Using state file for:", distname
statefile = ALMANAC_DIR + 'states/%s/index.html' % diststate.lower()
demog = almanac.scrape_state(statefile).get('state')
demog_to_dist(demog, district)
district.almanac = 'http://' + d['filename'][d['filename'].find('nationaljournal.com'):]
#print 'district:', distname, pformat(district)
db.update('district', where='name=$distname', vars=locals(), **district)
def save_contacts(email, contacts, provider):
#Even if the user is not logged-in, but has an account with us, let him import contacts
user_id = helpers.get_loggedin_userid()
if not user_id:
user = db.select('users',
what='id',
where='email=$email',
vars=locals())
if user: user_id = user[0].id
if user_id:
for c in contacts:
cname, cemail = c['name'], c['email']
vars = dict(user_id=user_id,
uemail=email,
cemail=cemail,
cname=cname,
provider=provider)
e = db.select(
'contacts',
where='user_id=$user_id and uemail=$uemail and cemail=$cemail',
vars=vars)
if not e: n = db.insert('contacts', seqname=False, **vars)
elif cname:
db.update(
'contacts',
cname=cname,
where=
'user_id=$user_id and uemail=$uemail and cemail=$cemail',
vars=vars)
def save_signature(i, pid, uid):
where = "petition_id=$pid AND user_id=$uid"
signed = db.select("signatory", where=where, vars=locals())
share_with = (i.get("share_with", "off") == "on" and "N") or "A"
update_user_details(i)
if not signed:
referrer = get_referrer(pid, uid)
signid = db.insert(
"signatory",
user_id=uid,
share_with=share_with,
petition_id=pid,
comment=i.get("comment"),
referrer=referrer,
)
helpers.set_msg("Thanks for your signing! Why don't you tell your friends about it now?")
return signid
else:
db.update(
"signatory",
where="user_id=$uid and petition_id=$pid",
comment=i.get("comment"),
deleted=None,
vars=locals(),
)
helpers.set_msg("Your signature has been changed. Why don't you tell your friends about it now?")
return "old_%s" % signed[0].id
def loadbill(fn, maplightid=None):
bill = xmltramp.load(fn)
d = bill2dict(bill)
d.maplightid = maplightid
try:
bill_id = d.id
db.insert('bill', seqname=False, **d)
except IntegrityError:
bill_id = d.pop('id')
db.update('bill', where="id=" + web.sqlquote(bill_id), **d)
positions = {}
for vote in bill.actions['vote':]:
if not vote().get('roll'): continue
rolldoc = '/us/%s/rolls/%s%s-%s.xml' % (
d.session, vote('where'), vote('datetime')[:4], vote('roll'))
roll = xmltramp.load(GOVTRACK_CRAWL + rolldoc)
for voter in roll['voter':]:
positions[govtrackp(voter('id'))] = fixvote(voter('vote'))
if None in positions: del positions[None]
with db.transaction():
for p, v in positions.iteritems():
db.insert('position', seqname=False,
bill_id=bill_id, politician_id=p, vote=v)
def load_new_pols():
with db.transaction():
db.update('congress', where="current_member='t'", current_member=False)
for polid, pol in new_pols.items():
district = pol['district_id']
create_or_update(polid, district)
update_congress(polid, district)
def GET(self, n):
res = db.select('pastes', where='id=$n', vars=locals())
db.update('pastes', where='id=$n',vars=locals(), lastview=now_time())
if res:
return render.view(res[0],n)
else:
return web.notfound()
def load_new_pols():
with db.transaction():
db.update('congress', where="current_member='t'", current_member=False)
for polid, pol in new_pols.items():
district = pol['district_id']
create_or_update(polid, district)
update_congress(polid, district)
def loadbill(fn, maplightid=None):
bill = xmltramp.load(fn)
d = bill2dict(bill)
d.maplightid = maplightid
try:
bill_id = d.id
db.insert('bill', seqname=False, **d)
except IntegrityError:
bill_id = d.pop('id')
db.update('bill', where="id=" + web.sqlquote(bill_id), **d)
positions = {}
for vote in bill.actions['vote':]:
if not vote().get('roll'): continue
rolldoc = '/us/%s/rolls/%s%s-%s.xml' % (
d.session, vote('where'), vote('datetime')[:4], vote('roll'))
roll = xmltramp.load(GOVTRACK_CRAWL + rolldoc)
for voter in roll['voter':]:
positions[govtrackp(voter('id'))] = fixvote(voter('vote'))
if None in positions: del positions[None]
with db.transaction():
db.delete('position', where='bill_id=$bill_id', vars=locals())
for p, v in positions.iteritems():
db.insert('position',
seqname=False,
bill_id=bill_id,
politician_id=p,
vote=v)
def POST_unsign(self, pid):
i = web.input()
now = datetime.now()
db.update("signatory", deleted=now, where="petition_id=$pid and user_id=$i.user_id", vars=locals())
msg = "Your signature has been removed for this petition."
helpers.set_msg(msg)
raise web.seeother("/%s" % pid)
def load_election_results(d, distname):
(year, votes, vote_pct) = (0,'0','0')
if 'name' not in d:
print "No name for the congress person for: ",distname
return
pname = d['name'].lower()
if 'electionresults' not in d:
print "No election results for %s repsenting %s." % (d['name'],distname)
return
for e in d['electionresults']:
if 'candidate' in e and 'primary' not in e['election'] and \
pname.replace(' ','') in e['candidate'].lower().replace(' ',''):
if int(e['election'][0:4]) > year:
(year,votes) = (int(e['election'][0:4]), e['totalvotes'])
if 'percent' in e: vote_pct = e['percent']
#print year, votes, vote_pct, d['name'], distname
if year:
pol=db.select('politician', what='id',
where="district_id='"+distname+"' AND "+web.sqlors('lastname ilike ',pname.split(' ')),
vars=locals()).list()
if pol and len(pol)==1:
polid=pol[0].id
db.update('politician', where='id=$polid',
n_vote_received=votes.replace(',','').replace('Unopposed','0'),
pct_vote_received=vote_pct.replace('%',''),
last_elected_year=year, vars=locals());
else: print "Couldn't find an id for %s representing %s." % (d['name'], distname)
else: print "Didn't find a recent election for %s representing %s." %(d['name'], distname) #, pformat(d['electionresults'])
def main():
assert os.path.exists(ALMANAC_DIR), ALMANAC_DIR
files = glob.glob(ALMANAC_DIR + '*/people/*/rep_*.htm') + \
glob.glob(ALMANAC_DIR + '*/people/*/*s[12].htm')
files.sort()
for fn in files:
district = web.storage()
demog = None
dist = web.lstrips(web.rstrips(fn.split('/')[-1], '.htm'), 'rep_')
diststate = dist[0:2].upper()
distnum = dist[-2:]
distname = tools.fixdist(diststate + '-' + distnum)
d = almanac.scrape_person(fn)
load_election_results(d, distname)
if ALMANAC_DIR + '2008' in fn:
if 'demographics' in d:
demog = d['demographics']
elif distname[-2:] == '00' or '-' not in distname: # if -00 then this district is the same as the state.
#print "Using state file for:", distname
statefile = ALMANAC_DIR + '2008/states/%s/index.html' % diststate.lower()
demog = almanac.scrape_state(statefile).get('state')
demog_to_dist(demog, district)
district.almanac = 'http://' + d['filename'][d['filename'].find('nationaljournal.com'):]
#print 'district:', distname, pformat(district)
db.update('district', where='name=$distname', vars=locals(), **district)
def load():
for district in shapes.parse():
outline = json.dumps({'type': 'MultiPolygon', 'coordinates': district['shapes']})
if district['district']:
district = tools.unfips(district['state_fipscode']) + '-' + district['district']
else:
district = tools.unfips(district['state_fipscode'])
db.update('district', where='name=$district', outline=outline, vars=locals())
def save_and_send_msg(self, i, wyrform, pform=None):
self.set_pol(i)
msg_id = self.save_msg(i.ptitle, i.msg)
self.set_msg_id(msg_id)
msg_sent = self.send_msg(i, wyrform, pform)
if msg_sent == True:
db.update('wyr', where='id=$msg_id', sent=True, vars=locals())
return msg_sent
def calculate_per_capita():
""" """
print "Pre getting all populations per district..."
pop_dist = {}
for d in schema.District.select(where='est_population is not null'):
pop_dist[d.name] = d.est_population
print "Calculate the per-capita impact of each earmark..."
pc = {}
for e in db.select('earmark', what='final_amt, id', order='id asc'):
done_states = set()
amount = float(e.final_amt or 0)
pop = 0
sponsors = db.query(
"select district_id, state_id, id from politician, district, earmark_sponsor where politician.district_id = district.name and earmark_sponsor.politician_id = politician.id and earmark_id=$e.id",
vars=locals()).list()
if not sponsors: continue
# Get the population for each district sponsoring
for p in sponsors:
if p.district_id != p.state_id:
done_states.add(p.state_id)
pop += pop_dist.get(p.district_id, 0)
# Get the population for state sponsoring unless a district has from
# within state also sponsors.
for p in sponsors:
if p.district_id == p.state_id:
if p.state_id in done_states: continue
done_states.add(p.state_id)
pop += pop_dist.get(p.district_id, 0)
if not pop:
pc[e.id] = 0.0
else:
pc[e.id] = amount / pop
#print e.id, pc[e.id], amount, pop
print "Aggregating per-capita impact to districts..."
for d in schema.District.select():
if d.name == d.state_id: continue # Don't set for states.
congress_people = set()
senators = db.select('curr_politician',
where='district_id = $d.state.code',
vars=locals())
if senators:
congress_people.update(p.id for p in senators)
politician = db.select('curr_politician',
where='district_id = $d.name',
vars=locals())
if politician:
congress_people.update(p.id for p in politician)
ems = db.select('earmark_sponsor',
what='distinct(earmark_id)',
where=web.sqlors('politician_id=', congress_people))
empc = sum(
map(lambda x: pc.get(x, 0.0), set(e.earmark_id for e in ems)))
#print d.name, empc
db.update('district',
where='name=$d.name',
earmark_per_capita=empc,
vars=locals())
def POST_delete(self, pid):
now = datetime.now()
title = db.select('petition',
what='title',
where='id=$pid',
vars=locals())[0].title
db.update('petition', where='id=$pid', deleted=now, vars=locals())
helpers.set_msg('Petition "%s" deleted' % (title))
raise web.seeother('/')
def load():
outdb = {}
done = set()
with db.transaction():
db.delete('earmark_sponsor', '1=1')
db.delete('earmark', '1=1')
for e in earmarks.parse_file(earmarks.EARMARK_FILE):
de = dict(e)
de['id'] = web.intget(de['id'])
if not de['id'] or de['id'] in done: continue # missing the ID? come on!
if isinstance(de['house_request'], basestring): continue # CLASSIFIED
for k in de: de[k] = cleanrow(de[k])
for x in ['house_member', 'house_state', 'house_party', 'senate_member', 'senate_state', 'senate_party', 'district']:
de.pop(x)
de['recipient_stem'] = tools.stemcorpname(de['intended_recipient'])
try:
db.insert('earmark', seqname=False, **de)
except:
pprint(de)
raise
done.add(de['id'])
reps_not_found = set()
for e in earmarks.parse_file(earmarks.EARMARK_FILE):
for rawRequest, chamber in zip([e.house_request, e.senate_request],[e.house_member, e.senate_member]):
for rep in chamber:
if rep.lower() not in lastname2rep:
#@@ should work on improving quality
reps_not_found.add(rep)
else:
rep = lastname2rep[rep.lower()]
if e.id in done:
try:
db.insert('earmark_sponsor', seqname=False, earmark_id=e.id, politician_id=rep)
except:
print "Couldn't add %s as sponsor to earmark %d" %(rep, e.id)
outdb.setdefault(rep, {
'amt_earmark_requested': 0,
'n_earmark_requested': 0,
'n_earmark_received': 0,
'amt_earmark_received': 0
})
outdb[rep]['n_earmark_requested'] += 1
requested = rawRequest or e.final_amt
if not isinstance(requested, float):
requested = e.final_amt
if requested:
outdb[rep]['amt_earmark_requested'] += requested
if isinstance(e.final_amt, float) and e.final_amt:
outdb[rep]['n_earmark_received'] += 1
outdb[rep]['amt_earmark_received'] += e.final_amt
print "Did not find",len(reps_not_found),"reps:", pformat(reps_not_found)
for rep, d in outdb.iteritems():
db.update('politician', where='id=$rep', vars=locals(), **d)
def load_votesmart():
# Candidates from votesmart
for district, cands in votesmart.candidates():
district = tools.fix_district_name(district)
for pol in cands:
vs_id = pol['candidateId']
wd = get_wd_id(vs_id)
if not wd: continue
polid = wd['watchdog_id']
pol_cand = filter_dict(cand_mapping, pol)
if not db.select('politician', where='id=$polid', vars=locals()):
db.insert('politician',
seqname=False,
id=polid,
**unidecode(
filter_dict(schema.Politician.columns.keys(),
pol_cand)))
else:
# @@ Should probably check that we really want to do this, but
# it apears as though the data has two entries for current
# members (the second having more info filled out).
db.update('politician',
where='id=$polid',
vars=locals(),
**unidecode(
filter_dict(schema.Politician.columns.keys(),
pol_cand)))
if not db.select(
'congress',
where="politician_id=$polid AND congress_num='-1'",
vars=locals()):
db.insert('congress',
seqname=False,
congress_num='-1',
politician_id=polid,
district_id=district,
party=pol_cand['party'])
# Bios from votesmart
for vs_id, p in votesmart.bios():
pol = p['candidate']
if pol['gender']:
pol['gender'] = pol['gender'][0]
if pol['education']:
pol['education'] = pol['education'].replace('\r\n', '\n')
wd = get_wd_id(vs_id)
if not wd: continue
polid = wd['watchdog_id']
pol_people = filter_dict(schema.Politician.columns.keys(),
filter_dict(bios_mapping, pol))
if db.select('politician', where='votesmartid=$vs_id', vars=locals()):
db.update('politician',
where='votesmartid=$vs_id',
vars=locals(),
**unidecode(pol_people))
def POST_unsign(self, pid):
i = web.input()
now = datetime.now()
db.update('signatory',
deleted=now,
where='petition_id=$pid and user_id=$i.user_id',
vars=locals())
msg = 'Your signature has been removed for this petition.'
helpers.set_msg(msg)
raise web.seeother('/%s' % pid)
def create_or_update(polid, dist):
if not db.select('district', where='name=$dist', vars=locals()):
db.insert('district', seqname=False, name=dist, state_id=dist[:2])
if db.select('politician', where='id=$polid', vars=locals()):
db.update('politician', where='id=$polid', district_id=dist, last_elected_year='2008', vars=locals())
else:
first, last = id.split('_', 1)
first, last = first.title(), last.title()
db.insert('politician', seqname=False, id=polid, firstname=first, lastname=last, last_elected_year='2008', district_id=dist)
def POST(self):
i = web.input()
form = forms.passwordform()
if form.validates(i):
password = encrypt_password(i.password)
db.update('users', password=password, verified=True, where='email=$i.email', vars=locals())
helpers.set_msg('Login with your new password.')
raise web.seeother('/login')
else:
return self.GET(form)
def main():
districts = json.load(file(DISTRICT_TABLE))
for dist in districts.iterkeys():
try:
d = file(GMAPDATA + '/%s-marker.js' % dist.replace('-0', '').replace('-', '')).read()
x = r_center.findall(d)[0]
db.update('district', where='name=$dist', vars=locals(),
center_lat = x[0], center_lng = x[1], zoom_level=x[2])
except IOError: # file not found
continue
def POST_unsign(self, pid):
i = web.input()
now = datetime.now()
db.update('signatory',
deleted=now,
where='petition_id=$pid and user_id=$i.user_id',
vars=locals())
msg = 'Your signature has been removed for this petition.'
helpers.set_msg(msg)
raise web.seeother('/%s' % pid)
def POST(self):
i = web.input()
form = forms.passwordform()
if form.validates(i):
password = encrypt_password(i.password)
db.update('users', password=password, verified=True, where='email=$i.email', vars=locals())
helpers.set_login_cookie(i.email)
helpers.set_msg('Password stored')
raise web.seeother('/c/', absolute=True)
else:
return self.GET(form)
def send_to_congress(i, wyrform, signid=None):
from utils.writerep import write_your_rep
pform, wyrform = forms.petitionform(), (wyrform or forms.wyrform())
pform.fill(i), wyrform.fill(i)
wyr = write_your_rep()
wyr.set_pol(i)
if not signid: signid = get_signid(i.pid)
wyr.set_msg_id(signid, petition=True)
msg_sent = wyr.send_msg(i, wyrform, pform)
sent_status = msg_sent and 'S' or 'D' # Sent or Due for sending
db.update('signatory', where='id=$signid', sent_to_congress=sent_status, vars=locals())
def update_user_details(i):
user = helpers.get_user_by_email(i.get("email"))
userid = user and user.id
i["zip5"] = i.get("zipcode")
i["phone"] = web.numify(i.get("phone"))
details = ["prefix", "lname", "fname", "addr1", "addr2", "city", "zip5", "zip4", "phone", "state"]
d = {}
for (k, v) in i.items():
if v and (k in details):
d[k] = v
db.update("users", where="id=$userid", vars=locals(), **d)
def update_user_details(i, uid=None):
if not uid:
user = helpers.get_user_by_email(i.get('email'))
uid = user and user.id
i['phone'] = web.numify(i.get('phone'))
details = ['prefix', 'lname', 'fname', 'addr1', 'addr2', 'city', 'zip5', 'zip4', 'phone', 'state']
d = {}
for (k, v) in i.items():
if v and (k in details):
d[k] = v
db.update('users', where='id=$uid', vars=locals(), **d)
def POST(self, uid):
if web.input('m', _method='GET'):
return self.POST_password(uid)
form = forms.userinfo()
i = web.input()
if form.validates(i):
i.pop('submit')
db.update('users', where='id=$uid', vars=locals(), **i)
helpers.set_msg('User information updated.')
raise web.seeother('/%s' % uid)
else:
return self.GET(uid, info_form=form)
def main():
for dist, canl in cans.iteritems():
dist=dist.replace("-SEN1","").replace("-SEN2","") ## JT: Hack
for can in canl:
wid = tools.getWatchdogID(dist,can['lastName'])
if wid:
bio = bios[can['candidateId']]['candidate']
db.update('politician', where='id = $wid', vars=locals(),
votesmartid=can['candidateId'],
nickname=can['nickName'],
birthplace=bio['birthPlace'],
education=bio['education'].replace('\r\n', '\n')
)
def main():
districts = json.load(file(DISTRICT_TABLE))
states = json.load(file(STATE_TABLE))
for table in [districts, states]:
for dist in table.iterkeys():
dname = GMAPDATA + '/%s[_-]marker*.js' % dist.replace('-0', '').replace('-', '')
dname = glob(dname)
if not dname: continue # File not found
dname = dname[0]
d = file(dname).read()
x = r_center.findall(d)[0]
db.update('district', where='name=$dist', vars=locals(),
center_lat = x[0], center_lng = x[1], zoom_level=x[2])
def main():
for pol in voteview.parse():
if not pol.get('district_id'): continue #@@
if not tools.districtp(pol.district_id) and pol.district_id.endswith('01'):
pol.district_id = pol.district_id.split('-')[0] + '-' + '00'
watchdog_id = tools.getWatchdogID(pol.district_id, pol.last_name)
if watchdog_id:
db.update('politician', where='id=$watchdog_id', vars=locals(),
icpsrid = pol.icpsr_id,
nominate = pol.dim1,
predictability = 1 - (pol.n_errs / float(pol.n_votes)))
def save_contacts(email, contacts, provider):
user_id = helpers.get_loggedin_userid()
for c in contacts:
cname, cemail = c['name'], c['email']
vars = dict(user_id=user_id, uemail=email, cemail=cemail,
cname=cname, provider=provider)
e = db.select('contacts',
where='user_id=$user_id and uemail=$uemail and cemail=$cemail',
vars=vars)
if not e: n = db.insert('contacts', seqname=False, **vars)
else: db.update('contacts', cname=cname,
where='user_id=$user_id and uemail=$uemail and cemail=$cemail',
vars=vars)
def load_all():
for code, state in items('states'):
if 'aka' in state: state.pop('aka')
db.insert('state', seqname=False, code=code, **unidecode(state))
for name, district in items('districts'):
db.insert('district', seqname=False, name=name, **unidecode(district))
db.update('congress', where="current_member='t'", current_member=False)
for polid, pol in items('politicians'):
db.insert('politician', seqname=False, id=polid, **unidecode(pol))
db.insert('congress', seqname=False, politician_id=polid,
congress_num=current_session, district_id=pol['district_id'],
current_member=True)
def POST(self, uid):
i = web.input('m', _method='GET')
if i.m == 'password':
return self.POST_password(uid)
form = forms.userinfo()
i = web.input(_method='POST')
if form.validates(i):
if 'submit' in i: i.pop('submit')
db.update('users', where='id=$uid', vars=locals(), **i)
helpers.set_msg('User information updated.')
raise web.seeother('/%s/preferences' % uid)
else:
return self.GET(uid, info_form=form)
def POST_password(self, uid):
user = db.select('users', what='password', where='id=$uid', vars=locals())[0]
form = get_password_form(user)
i = web.input()
if form.validates(i):
if ('curr_password' not in form) or auth.check_password(user, i.curr_password):
enc_password = auth.encrypt_password(i.password)
db.update('users', password=enc_password, verified=True, where='id=$uid', vars=locals())
helpers.set_msg('Password saved.')
else:
helpers.set_msg('Invalid Password', 'error')
raise web.seeother('/%s' % uid)
else:
return self.GET(uid, password_form=form)
def POST_edit(self, pid):
i = web.input()
tocongress = i.get('tocongress', 'off') == 'on'
pform = forms.petitionform()
pform.inputs = filter(lambda i: i.name != 'pid', pform.inputs)
wyrform = forms.wyrform()
i.email = helpers.get_loggedin_email()
wyr_valid = (not(tocongress) or wyrform.validates(i))
if not pform.validates(i) or not wyr_valid:
title = "Edit petition"
return render.petitionform(pform, wyrform, title, target='/c/%s?m=edit' % (pid))
db.update('petition', where='id=$pid', title=i.ptitle, description=i.msg, to_congress=tocongress, vars=locals())
update_user_details(i)
raise web.seeother('/%s' % pid)
def load_all():
for code, state in items('states/index'):
if 'aka' in state: state.pop('aka')
db.insert('state', seqname=False, code=code, **unidecode(state))
for name, district in items('districts/index'):
db.insert('district', seqname=False, name=name, **unidecode(district))
for fn in ['almanac', 'shapes', 'centers']:
for name, district in items('districts/' + fn):
if 'interest_group_rating' in district:
district.pop('interest_group_rating')
db.update('district',
where='name = $name',
vars=locals(),
**unidecode(district))
district_to_pol = {}
for polid, pol in items('politicians/index'):
db.insert('politician', seqname=False, id=polid, **unidecode(pol))
district_to_pol[pol['district']] = polid
for fn in ['govtrack', 'voteview', 'earmarks']:
for polid, pol in items('politicians/' + fn):
db.update('politician',
where='id = $polid',
vars=locals(),
**unidecode(pol))
groupname_to_id = {}
for groupname, longname in items('interest_groups'):
gid = db.insert('interest_group', groupname=groupname, longname=longname)
groupname_to_id[groupname] = gid
for name, district in items('districts/almanac'):
if name not in district_to_pol: continue #@@ desynchronized data!
if 'interest_group_rating' in district:
for year, groups in district['interest_group_rating'].items():
for groupname, rating in groups.items():
try:
group_id = db.select('interest_group', what='id', where='groupname=$groupname', vars=locals())[0].id
except:
group_id = db.insert('interest_group', groupname=groupname)
db.insert('interest_group_rating',
politician_id=district_to_pol[name],
year=year,
group_id=groupname_to_id[groupname],
rating=rating)
def update_user_details(i, uid=None):
if not uid:
user = helpers.get_user_by_email(i.get('email'))
uid = user and user.id
i['phone'] = web.numify(i.get('phone'))
details = [
'prefix', 'lname', 'fname', 'addr1', 'addr2', 'city', 'zip5', 'zip4',
'phone', 'state'
]
d = {}
for (k, v) in i.items():
if v and (k in details):
d[k] = v
db.update('users', where='id=$uid', vars=locals(), **d)
def POST(self):
i = web.input()
form = forms.passwordform()
if form.validates(i):
password = encrypt_password(i.password)
db.update('users',
password=password,
verified=True,
where='email=$i.email',
vars=locals())
helpers.set_login_cookie(i.email)
helpers.set_msg('Password stored')
raise web.seeother('/c/', absolute=True)
else:
return self.GET(form)
def POST(self, uid):
i = web.input("m", _method="GET")
if i.m == "password":
return self.POST_password(uid)
form = forms.userinfo()
i = web.input(_method="POST")
if form.validates(i):
if "submit" in i:
i.pop("submit")
db.update("users", where="id=$uid", vars=locals(), **i)
helpers.set_msg("User information updated.")
raise web.seeother("/%s/preferences" % uid)
else:
return self.GET(uid, info_form=form)
def save_petition(p):
p.pid = p.pid.replace(' ', '_')
with db.transaction():
try:
owner = db.select('users', where='email=$p.email', vars=locals())[0]
except:
owner_id = db.insert('users', email=p.email, verified=True)
else:
if not owner.verified: db.update('users', where='email=$p.email', verified=True, vars=locals())
owner_id = owner.id
db.insert('petition', seqname=False, id=p.pid, title=p.ptitle, description=p.pdescription,
owner_id=owner_id)
#make the owner of the petition sign for it (??)
db.insert('signatory', seqname=False, user_id=owner_id, share_with='E', petition_id=p.pid)
def load():
for district in shapes.parse():
outline = json.dumps({
'type': 'MultiPolygon',
'coordinates': district['shapes']
})
if district['district']:
district = tools.unfips(
district['state_fipscode']) + '-' + district['district']
else:
district = tools.unfips(district['state_fipscode'])
db.update('district',
where='name=$district',
outline=outline,
vars=locals())
def load():
out = {}
pols = db.select('politician')
#json.load(file('../data/load/politicians/govtrack.json'))
for pol in pols:
options = [
(
'../data/crawl/house/photos/%s.jpg' % pol['bioguideid'],
(bioguide_u(pol['bioguideid']), 'Congressional Biographical Directory')
),
(
'../data/crawl/govtrack/photos/%s.jpg' % pol['govtrackid'],
govtrackcredit(pol['govtrackid'])
),
(
'../data/crawl/opensecrets/photos/%s.jpg' % pol.get('opensecretsid'),
(opensecrets_u(pol.get('opensecretsid')), 'Center for Responsive Politics')
),
(
'../data/crawl/votesmart/photos/%s.jpg' % pol.get('votesmartid'),
(votesmart_u(pol.get('votesmartid')), 'Project Vote Smart')
),
(
'../data/crawl/votesmart/photos/%s.JPG' % pol.get('votesmartid'),
(votesmart_u(pol.get('votesmartid')), 'Project Vote Smart')
)
]
maxsize = 0
currentfn = None
currentcredit = None
for fn, source in options:
try:
if os.stat(fn).st_size > maxsize:
maxsize = os.stat(fn).st_size
currentfn = fn
currentcredit = source
except OSError: # file does not exist
pass
if currentfn:
db.update('politician', where='id=$pol.id', vars=locals(),
photo_path=currentfn[2:],
photo_credit_url=currentcredit[0],
photo_credit_text=currentcredit[1]
)
def loadroll(fn):
roll = web.storage()
roll.id = fn.split('/')[-1].split('.')[0]
vote = xmltramp.load(fn)
if vote['bill':]:
b = vote.bill
roll.bill_id = 'us/%s/%s%s' % (b('session'), b('type'), b('number'))
else:
roll.bill_id = None
roll.type = str(vote.type)
roll.question = str(vote.question)
roll.required = str(vote.required)
roll.result = str(vote.result)
try:
db.insert('roll', seqname=False, **roll)
except IntegrityError:
if not db.update('roll',
where="id=" + web.sqlquote(roll.id),
bill_id=roll.bill_id):
print "\nMissing bill:", roll.bill_id
raise NotDone
with db.transaction():
db.delete('vote', where="roll_id=$roll.id", vars=locals())
for voter in vote['voter':]:
rep = govtrackp(voter('id'))
if rep:
db.insert('vote',
seqname=False,
politician_id=rep,
roll_id=roll.id,
vote=fixvote(voter('vote')))
else:
pass #@@!--check again after load_everyone
def main():
for pol in voteview.parse():
if not pol.get('district_id'): continue #@@
if not tools.districtp(
pol.district_id) and pol.district_id.endswith('01'):
pol.district_id = pol.district_id.split('-')[0] + '-' + '00'
watchdog_id = tools.getWatchdogID(pol.district_id, pol.last_name)
if watchdog_id:
db.update('politician',
where='id=$watchdog_id',
vars=locals(),
icpsrid=pol.icpsr_id,
nominate=pol.dim1,
predictability=1 - (pol.n_errs / float(pol.n_votes)))
def load_all():
for code, state in items('states'):
if 'aka' in state: state.pop('aka')
db.insert('state', seqname=False, code=code, **unidecode(state))
for name, district in items('districts'):
db.insert('district', seqname=False, name=name, **unidecode(district))
db.update('congress', where="current_member='t'", current_member=False)
for polid, pol in items('politicians'):
db.insert('politician', seqname=False, id=polid, **unidecode(pol))
db.insert('congress',
seqname=False,
politician_id=polid,
congress_num=current_session,
district_id=pol['district_id'],
current_member=True)
def POST_edit(self, pid):
i = web.input()
tocongress = i.get('tocongress', 'off') == 'on'
pf = forms.petitionform()
pf.inputs = filter(lambda i: i.name != 'pid', pf.inputs)
wf = forms.wyrform()
i.email = helpers.get_loggedin_email()
wyr_valid = (not (tocongress) or wf.validates(i))
if not pf.validates(i) or not wyr_valid:
return render.petitionform(pf, wf, is_new=False, is_draft=is_draft)
p = dict(title=i.ptitle, description=i.msg, to_congress=tocongress)
if 'publish' in i: p['published'] = datetime.now()
db.update('petition', where='id=$pid', vars=locals(), **p)
if 'publish' in i:
create_first_signature(i, i.email)
update_user_details(i)
raise web.seeother('/%s' % pid)
def main():
watchdog_map = combine()
with db.transaction():
#db.delete('congress', where='1=1')
#db.delete('politician', where='1=1')
for polid, pol in watchdog_map.items():
roles = pol.pop('roles')
# Load the Politician table
if db.select('politician', where='id=$polid',
vars=locals()): #pol.get('current_member'):
if roles and pol.current_member:
pol.officeurl = roles[-1].get('url')
pol.party = roles[-1].get('party')
db.update('politician',
where='id=$polid',
vars=locals(),
**unidecode(
filter_dict(schema.Politician.columns.keys(),
pol)))
else:
db.insert('politician',
seqname=False,
id=polid,
**unidecode(
filter_dict(schema.Politician.columns.keys(),
pol)))
# Load the Congress table
done = set()
for r in roles:
repr = r.state
if r.type == 'rep' and int(r.district) >= 0:
repr = '%s-%02d' % (r.state, int(r.district))
for term in cong_term_lookup(r.startdate, r.enddate):
#if not db.select('congress', where='politician_id=$polid AND congress_num=$term AND district_id=$repr', vars=locals()):
if (polid, repr, term) not in done:
db.insert('congress',
seqname=False,
party=r.party,
congress_num=term,
politician_id=polid,
current_member=pol.current_member,
district_id=repr)
done.add((polid, repr, term))
def load():
out = {}
pols = db.select('politician')
#json.load(file('../data/load/politicians/govtrack.json'))
for pol in pols:
options = [
('../data/crawl/house/photos/%s.jpg' % pol['bioguideid'],
(bioguide_u(pol['bioguideid']),
'Congressional Biographical Directory')),
('../data/crawl/govtrack/photos/%s.jpg' % pol['govtrackid'],
govtrackcredit(pol['govtrackid'])),
('../data/crawl/opensecrets/photos/%s.jpg' %
pol.get('opensecretsid'),
(opensecrets_u(pol.get('opensecretsid')),
'Center for Responsive Politics')),
('../data/crawl/votesmart/photos/%s.jpg' % pol.get('votesmartid'),
(votesmart_u(pol.get('votesmartid')), 'Project Vote Smart')),
('../data/crawl/votesmart/photos/%s.JPG' % pol.get('votesmartid'),
(votesmart_u(pol.get('votesmartid')), 'Project Vote Smart'))
]
maxsize = 0
currentfn = None
currentcredit = None
for fn, source in options:
try:
if os.stat(fn).st_size > maxsize:
maxsize = os.stat(fn).st_size
currentfn = fn
currentcredit = source
except OSError: # file does not exist
pass
if currentfn:
db.update('politician',
where='id=$pol.id',
vars=locals(),
photo_path=currentfn[2:],
photo_credit_url=currentcredit[0],
photo_credit_text=currentcredit[1])
def create_or_update(polid, dist):
if not db.select('district', where='name=$dist', vars=locals()):
db.insert('district', seqname=False, name=dist, state_id=dist[:2])
if db.select('politician', where='id=$polid', vars=locals()):
db.update('politician',
where='id=$polid',
district_id=dist,
last_elected_year='2008',
vars=locals())
else:
first, last = id.split('_', 1)
first, last = first.title(), last.title()
db.insert('politician',
seqname=False,
id=polid,
firstname=first,
lastname=last,
last_elected_year='2008',
district_id=dist)
def load_into_db(pname, distname, electionresults, recent_election_year):
#load the details of the winner in recent election results into `politician` table
#and all the details of elections in district `distname` into the `past_elections` table
with db.transaction():
for r in electionresults:
candidate_id = r.candidate.split('(')[0].strip().lower()
if r.year == recent_election_year and r.type == 'Gen' and pname in candidate_id:
polid = db.update('politician',
where="district_id=$distname AND %s" % (web.sqlors('lastname ilike ', pname.split(' '))),
n_vote_received=r.votes,
pct_vote_received=r.vote_pct,
last_elected_year=r.year, vars=locals())
candidate_id = candidate_id.replace(' ', '_')
if not db.select('past_elections',
where='politician_id=$candidate_id and district_id=$distname '
'and year=$r.year and type=$r.type',
vars=locals()):
db.insert('past_elections', seqname=False, politician_id=candidate_id, district_id=distname,
votes_received=r.votes, pct_votes_received=r.vote_pct, type=r.type,
year=r.year, expenditure=r.expenditure)
def update_msg_status(msgid, status):
"""updates status of the message with id `msgid` with status.
"""
db.update('messages', sent=status, where='id=$msgid', vars=locals())
def load():
outdb = {}
done = set()
with db.transaction():
db.delete('earmark_sponsor', '1=1')
db.delete('earmark', '1=1')
for e in earmarks.parse_file(earmarks.EARMARK_FILE):
de = dict(e)
de['id'] = web.intget(de['id'])
if not de['id'] or de['id'] in done:
continue # missing the ID? come on!
if isinstance(de['house_request'], basestring):
continue # CLASSIFIED
for k in de:
de[k] = cleanrow(de[k])
for x in [
'house_member', 'house_state', 'house_party',
'senate_member', 'senate_state', 'senate_party', 'district'
]:
de.pop(x)
de['recipient_stem'] = tools.stemcorpname(de['intended_recipient'])
try:
db.insert('earmark', seqname=False, **de)
except:
pprint(de)
raise
done.add(de['id'])
reps_not_found = set()
for e in earmarks.parse_file(earmarks.EARMARK_FILE):
for rawRequest, chamber in zip([e.house_request, e.senate_request],
[e.house_member, e.senate_member]):
for rep in chamber:
if rep.lower() not in lastname2rep:
#@@ should work on improving quality
reps_not_found.add(rep)
else:
rep = lastname2rep[rep.lower()]
if e.id in done:
try:
db.insert('earmark_sponsor',
seqname=False,
earmark_id=e.id,
politician_id=rep)
except:
print "Couldn't add %s as sponsor to earmark %d" % (
rep, e.id)
outdb.setdefault(
rep, {
'amt_earmark_requested': 0,
'n_earmark_requested': 0,
'n_earmark_received': 0,
'amt_earmark_received': 0
})
outdb[rep]['n_earmark_requested'] += 1
requested = rawRequest or e.final_amt
if not isinstance(requested, float):
requested = e.final_amt
if requested:
outdb[rep]['amt_earmark_requested'] += requested
if isinstance(e.final_amt, float) and e.final_amt:
outdb[rep]['n_earmark_received'] += 1
outdb[rep]['amt_earmark_received'] += e.final_amt
print "Did not find", len(reps_not_found), "reps:", pformat(reps_not_found)
for rep, d in outdb.iteritems():
db.update('politician', where='id=$rep', vars=locals(), **d)
"""
Load data from opensecrets.
"""
from parse import opensecrets
import tools
from settings import db
for can in opensecrets.parse_all():
p = tools.opensecretsp(can.get('opensecretsid'))
if p and can:
db.update(
'politician',
where='id = $p',
vars=locals(),
pct_pac_business=can.business_pac / float(can.total),
#pct_badmoney = can.badmoney/float(can.total)
)
def main():
with db.transaction():
db.update('politician', votesmartid=None, where='1=1')
db.delete('congress', where="congress_num='-1'")
load_votesmart()
from parse import punch
import tools
from settings import db
for can in punch.parse_all():
d = tools.districtp(can.district)
if d and can.name.split(',')[0].lower() in d:
db.update('politician',
where='id = $d',
vars=locals(),
chips2008=can.chips2008,
progressive2008=can.progressive2008,
progressiveall=can.progressiveall)
