def person_cosponsors(request, pk): # Load the cosponsors. from bill.models import Cosponsor person = get_object_or_404(Person, pk=pk) cosponsors = Cosponsor.objects.filter(bill__sponsor=person, withdrawn=None)\ .prefetch_related("bill", "bill__terms", "person", "person__roles") # Pre-fetch all of the top-terms. from bill.models import BillTerm top_terms = set(BillTerm.get_top_term_ids()) # Aggregate. total = 0 from collections import defaultdict ret = defaultdict(lambda: { "total": 0, "by_issue": defaultdict(lambda: 0), }) for cosp in cosponsors: total += 1 ret[cosp.person]["total"] += 1 for t in cosp.bill.terms.all(): if t.id in top_terms: ret[cosp.person]["by_issue"][t] += 1 if "first_date" not in ret[ cosp.person] or cosp.joined < ret[cosp.person]["first_date"]: ret[cosp.person]["first_date"] = cosp.joined if "last_date" not in ret[ cosp.person] or cosp.joined > ret[cosp.person]["last_date"]: ret[cosp.person]["last_date"] = cosp.joined # Sort. for info in ret.values(): info['by_issue'] = sorted(info['by_issue'].items(), key=lambda kv: kv[1], reverse=True) ret = sorted(ret.items(), key=lambda kv: (kv[1]['total'], kv[1]['last_date'], kv[0].sortname), reverse=True) # Total bills, date range. from bill.models import Bill total_bills = Bill.objects.filter(sponsor=person).count() date_range = (None, None) if len(ret) > 0: date_range = (min(r["first_date"] for p, r in ret), max(r["last_date"] for p, r in ret)) return { "person": person, "cosponsors": ret, "total": total, "total_bills": total_bills, "date_range": date_range, }
def person_cosponsors(request, pk): # Load the cosponsors. from bill.models import Cosponsor person = get_object_or_404(Person, pk=pk) cosponsors = Cosponsor.objects.filter(bill__sponsor=person, withdrawn=None)\ .prefetch_related("bill", "bill__terms", "person", "person__roles") # Pre-fetch all of the top-terms. from bill.models import BillTerm top_terms = set(BillTerm.get_top_term_ids()) # Aggregate. total = 0 from collections import defaultdict ret = defaultdict(lambda : { "total": 0, "by_issue": defaultdict(lambda : 0), }) for cosp in cosponsors: total += 1 ret[cosp.person]["total"] += 1 for t in cosp.bill.terms.all(): if t.id in top_terms: ret[cosp.person]["by_issue"][t] += 1 if "first_date" not in ret[cosp.person] or cosp.joined < ret[cosp.person]["first_date"]: ret[cosp.person]["first_date"] = cosp.joined if "last_date" not in ret[cosp.person] or cosp.joined > ret[cosp.person]["last_date"]: ret[cosp.person]["last_date"] = cosp.joined # Sort. for info in ret.values(): info['by_issue'] = sorted(info['by_issue'].items(), key = lambda kv : kv[1], reverse=True) ret = sorted(ret.items(), key = lambda kv : (kv[1]['total'], kv[1]['last_date'], kv[0].sortname), reverse=True) # Total bills, date range. from bill.models import Bill total_bills = Bill.objects.filter(sponsor=person).count() date_range = (None, None) if len(ret) > 0: date_range = (min(r["first_date"] for p, r in ret), max(r["last_date"] for p, r in ret)) return { "person": person, "cosponsors": ret, "total": total, "total_bills": total_bills, "date_range": date_range, }
def main(options): """ Process bill terms and bills """ # Terms term_processor = TermProcessor() terms_parsed = set() # Cache existing terms. There aren't so many. existing_terms = {} for term in BillTerm.objects.all(): existing_terms[(int(term.term_type), term.name)] = term log.info('Processing old bill terms') TERMS_FILE = 'bill/liv.xml' tree = etree.parse(TERMS_FILE) for node in tree.xpath('/liv/top-term'): term = term_processor.process(BillTerm(), node) term.term_type = TermType.old try: # No need to update an existing term because there are no other attributes. term = existing_terms[(int(term.term_type), term.name)] terms_parsed.add(term.id) except: log.debug("Created %s" % term) term.save() term.subterms.clear() for subnode in node.xpath('./term'): subterm = term_processor.process(BillTerm(), subnode) subterm.term_type = TermType.old try: # No need to update an existing term because there are no other attributes. subterm = existing_terms[(int(subterm.term_type), subterm.name)] term.subterms.add(subterm) terms_parsed.add(subterm.id) except: try: log.debug("Created %s" % subterm) subterm.save() term.subterms.add(subterm) existing_terms[(int(subterm.term_type), subterm.name)] = subterm terms_parsed.add(subterm.id) except IntegrityError: log.error('Duplicated term %s' % term_processor.display_node(subnode)) log.info('Processing new bill terms') for FILE in ('bill/liv111.xml', 'bill/crsnet.xml'): tree = etree.parse(FILE) for node in tree.xpath('/liv/top-term'): term = term_processor.process(BillTerm(), node) term.term_type = TermType.new try: # No need to update an existing term because there are no other attributes. term = existing_terms[(int(term.term_type), term.name)] terms_parsed.add(term.id) except: log.debug("Created %s" % term) term.save() term.subterms.clear() for subnode in node.xpath('./term'): subterm = term_processor.process(BillTerm(), subnode) subterm.term_type = TermType.new try: # No need to update an existing term because there are no other attributes. subterm = existing_terms[(int(subterm.term_type), subterm.name)] terms_parsed.add(subterm.id) term.subterms.add(subterm) except: try: log.debug("Created %s" % term) subterm.save() term.subterms.add(subterm) existing_terms[(int(subterm.term_type), subterm.name)] = subterm terms_parsed.add(subterm.id) except IntegrityError: log.error('Duplicated term %s' % term_processor.display_node(subnode)) for term in existing_terms.values(): if not term.id in terms_parsed: log.debug("Deleted %s" % term) term.delete() # Bills bill_index = None if not options.disable_indexing: from bill.search_indexes import BillIndex bill_index = BillIndex() if options.congress: files = glob.glob(settings.CONGRESS_DATA_PATH + '/%s/bills/*/*/data.xml' % options.congress) log.info('Parsing unitedstates/congress bills of only congress#%s' % options.congress) else: files = glob.glob(settings.CONGRESS_DATA_PATH + '/*/bills/*/*/data.xml') if options.filter: files = [f for f in files if re.match(options.filter, f)] log.info('Processing bills: %d files' % len(files)) total = len(files) progress = Progress(total=total, name='files', step=100) bill_processor = BillProcessor() seen_bill_ids = [] for fname in files: progress.tick() # With indexing or events enabled, if the bill metadata file hasn't changed check # the bill's latest text file for changes so we can create a text-is-available # event and so we can index the bill's text. if (not options.congress or int(options.congress) > 42) and ( bill_index and not options.disable_events ) and not File.objects.is_changed(fname) and not options.force: m = re.match( re.escape(settings.CONGRESS_DATA_PATH) + r'/(?P<congress>\d+)/bills/(?P<bill_type>[a-z]+)/(?P<bill_type_2>[a-z]+)(?P<number>\d+)/data.xml', fname) try: b = Bill.objects.get(congress=int(m.group("congress")), bill_type=BillType.by_slug( m.group("bill_type")), number=m.group("number")) seen_bill_ids.append(b.id) # Update the index/events for any bill with recently changed text textfile = get_bill_text_metadata(b, None) if not textfile: if b.congress >= 103 and b.introduced_date < ( datetime.now() - timedelta(days=14)).date(): print("No bill text?", fname, b.introduced_date) continue textfile = textfile["text_file"] if os.path.exists(textfile) and File.objects.is_changed( textfile): b.update_index(bill_index) # index the full text b.create_events() # events for new bill text documents File.objects.save_file(textfile) continue except Bill.DoesNotExist: print("Unchanged metadata file but bill doesn't exist:", fname) pass # just parse as normal if options.slow: time.sleep(1) tree = etree.parse(fname) for node in tree.xpath('/bill'): try: bill = bill_processor.process(Bill(), node) except: print(fname) raise seen_bill_ids.append(bill.id) # don't delete me later # So far this is just for American Memory bills. if node.xpath("string(source/@url)"): bill.source_link = str(node.xpath("string(source/@url)")) else: bill.source_link = None actions = [] for axn in tree.xpath("actions/*[@state]"): if axn.xpath("string(@state)") == "REFERRED": continue # we don't track this state actions.append(( repr( bill_processor.parse_datetime( axn.xpath("string(@datetime)"))), BillStatus.by_xml_code(axn.xpath("string(@state)")), axn.xpath("string(text)"), etree.tostring(axn, encoding=str), )) bill.sliplawpubpriv = None bill.sliplawnum = None for axn in tree.xpath("actions/enacted"): bill.sliplawpubpriv = "PUB" if axn.get( "type") == "public" else "PRI" bill.sliplawnum = int(axn.get("number").split("-")[1]) bill.major_actions = actions try: bill.save() except: print(bill) raise if bill_index: bill.update_index(bill_index) if not options.disable_events: bill.create_events() File.objects.save_file(fname) # delete bill objects that are no longer represented on disk.... this is too dangerous. if options.congress and not options.filter: # this doesn't work because seen_bill_ids is too big for sqlite! for b in Bill.objects.filter(congress=options.congress).exclude( id__in=seen_bill_ids): print("Bill is no longer on disk: ", b.id, b) # The rest is for current only... if options.congress and int(options.congress) != settings.CURRENT_CONGRESS: return # Find what might be coming up this week. load_docs_house_gov(options, bill_index) load_senate_floor_schedule(options, bill_index)
def build_info(): global pronunciation_guide if re.match(r"\d", pk): person = get_object_or_404(Person, pk=pk) else: # support bioguide IDs for me person = get_object_or_404(Person, bioguideid=pk) # current role role = person.get_current_role() if role: active_role = True else: active_role = False try: role = person.roles.order_by('-enddate')[0] except IndexError: role = None # photo photo_url, photo_credit = person.get_photo() # analysis analysis_data = analysis.load_data(person) try: # Get session stats for the previous year. has_session_stats = person.get_session_stats( str(datetime.now().year - 1)) except: # Not everyone has current stats, obviously. They may have stats # corresponding to their most recent role. Since stats are a # session behind, even-year stats might not correspond to # a legislator's most recent role, which is why I hard-coded # the current session stats above. has_session_stats = False if role: try: has_session_stats = role.get_most_recent_session_stats() except: pass links = [] if role.current: if role.website: links.append(("%s's Official Website" % person.lastname, role.website, "fa fa-external-link")) if person.twitterid: links.append(("@" + person.twitterid, "http://twitter.com/" + person.twitterid, "fa fa-twitter")) if person.osid: links.append( ("OpenSecrets", "http://www.opensecrets.org/politicians/summary.php?cid=" + person.osid, "fa fa-money")) if person.pvsid: links.append( ("VoteSmart", "http://votesmart.org/candidate/" + person.pvsid, "fa fa-th-list")) if person.bioguideid: links.append( ("Bioguide", "http://bioguide.congress.gov/scripts/biodisplay.pl?index=" + person.bioguideid, "fa fa-user")) if person.cspanid: links.append(("C-SPAN", "http://www.c-spanvideo.org/person/" + str(person.cspanid), "fa fa-youtube-play")) # Get a break down of the top terms this person's sponsored bills fall into, # looking only at the most recent five years of bills. from bill.models import BillTerm most_recent_bill = person.sponsored_bills.order_by( "-introduced_date").first() bills_by_subject_counts = list(person.sponsored_bills.filter( terms__id__in=BillTerm.get_top_term_ids(), introduced_date__gt=(most_recent_bill.introduced_date if most_recent_bill else datetime.now())-timedelta(days=5*365.25))\ .values("terms")\ .annotate(count=Count('id')).order_by('-count')\ .filter(count__gt=1)\ [0:8]) terms = BillTerm.objects.in_bulk(item["terms"] for item in bills_by_subject_counts) total_count = sum(item["count"] for item in bills_by_subject_counts) while len(bills_by_subject_counts) > 2 and bills_by_subject_counts[-1][ "count"] < bills_by_subject_counts[0]["count"] / 8: bills_by_subject_counts.pop(-1) for item in bills_by_subject_counts: item["term"] = terms[item["terms"]] item["pct"] = int(round(float(item["count"]) / total_count * 100)) del item["terms"] # Missed vote explanations from ProPublica try: vote_explanations = http_rest_json( "https://projects.propublica.org/explanations/api/members/%s.json" % person.bioguideid) except: # squash all errors vote_explanations = {} # Misconduct - load and filter this person's entries, keeping original order. # Choose 'Alleged misconduct', 'Misconduct', 'Misconduct/alleged misconduct' as appropriate. from website.views import load_misconduct_data misconduct = [ m for m in load_misconduct_data() if m["person"] == person ] misconduct_any_alleged = (len([m for m in misconduct if m["alleged"]]) > 0) misconduct_any_not_alleged = (len( [m for m in misconduct if not m["alleged"]]) > 0) # Load pronunciation from guide. Turn into a mapping from GovTrack IDs to data. if pronunciation_guide is None: import rtyaml if not hasattr(settings, 'PRONUNCIATION_DATABASE_PATH'): # debugging pronunciation_guide = {} else: pronunciation_guide = { p["id"]["govtrack"]: p for p in rtyaml.load( open(settings.PRONUNCIATION_DATABASE_PATH)) } # Get this person's entry. pronunciation = pronunciation_guide.get(person.id) # TODO: Validate that the 'name' in the guide matches the name we're actually displaying. if pronunciation and not pronunciation.get("key"): # Show a key to the letters used in the pronunciation guide. Break apart the name # into words which we'll show in columns. pronunciation["key"] = [] for namepart in pronunciation["respell"].split(" // "): for nameword in namepart.split(" "): # Parse out the symbols actually used in the guide. Sweep from left to right chopping # off valid respelling letter combinations, chopping off the longest one where possible. pronunciation["key"].append([]) i = 0 while i < len(nameword): for s in sorted(pronunciation_guide_key, key=lambda s: -len(s)): if nameword[i:i + len(s)] in (s, s.upper()): pronunciation["key"][-1].append( (nameword[i:i + len(s)], pronunciation_guide_key[s])) i += len(s) break else: # respelling did not match any valid symbol, should be an error but we don't # want to issue an Oops! for this break # Get their enacted bills. enacted_bills_src_qs = person.sponsored_bills.exclude( original_intent_replaced=True).order_by('-current_status_date') return { 'person': person, 'role': role, 'active_role': active_role, 'active_congressional_role': active_role and role.role_type in (RoleType.senator, RoleType.representative), 'pronunciation': pronunciation, 'photo': photo_url, 'photo_credit': photo_credit, 'links': links, 'analysis_data': analysis_data, 'enacted_bills': [ b for b in enacted_bills_src_qs if b.was_enacted_ex( cache_related_bills_qs=enacted_bills_src_qs) ], 'recent_bills': person.sponsored_bills.all().order_by('-introduced_date')[0:7], 'committeeassignments': get_committee_assignments(person), 'feed': person.get_feed(), 'has_session_stats': has_session_stats, 'bill_subject_areas': bills_by_subject_counts, 'vote_explanations': vote_explanations, 'key_votes': load_key_votes(person), 'misconduct': misconduct, 'misconduct_any_alleged': misconduct_any_alleged, 'misconduct_any_not_alleged': misconduct_any_not_alleged, }
def main(options): """ Process bill terms and bills """ # Terms term_processor = TermProcessor() terms_parsed = set() # Cache existing terms. There aren't so many. existing_terms = {} for term in BillTerm.objects.all(): existing_terms[(int(term.term_type), term.name)] = term log.info('Processing old bill terms') TERMS_FILE = 'data/us/liv.xml' tree = etree.parse(TERMS_FILE) for node in tree.xpath('/liv/top-term'): term = term_processor.process(BillTerm(), node) term.term_type = TermType.old try: # No need to update an existing term because there are no other attributes. term = existing_terms[(int(term.term_type), term.name)] terms_parsed.add(term.id) except: log.debug("Created %s" % term) term.save() term.subterms.clear() for subnode in node.xpath('./term'): subterm = term_processor.process(BillTerm(), subnode) subterm.term_type = TermType.old try: # No need to update an existing term because there are no other attributes. subterm = existing_terms[(int(subterm.term_type), subterm.name)] term.subterms.add(subterm) terms_parsed.add(subterm.id) except: try: log.debug("Created %s" % subterm) subterm.save() term.subterms.add(subterm) existing_terms[(int(subterm.term_type), subterm.name)] = subterm terms_parsed.add(subterm.id) except IntegrityError: log.error('Duplicated term %s' % term_processor.display_node(subnode)) log.info('Processing new bill terms') for FILE in ('data/us/liv111.xml', 'data/us/crsnet.xml'): tree = etree.parse(FILE) for node in tree.xpath('/liv/top-term'): term = term_processor.process(BillTerm(), node) term.term_type = TermType.new try: # No need to update an existing term because there are no other attributes. term = existing_terms[(int(term.term_type), term.name)] terms_parsed.add(term.id) except: log.debug("Created %s" % term) term.save() term.subterms.clear() for subnode in node.xpath('./term'): subterm = term_processor.process(BillTerm(), subnode) subterm.term_type = TermType.new try: # No need to update an existing term because there are no other attributes. subterm = existing_terms[(int(subterm.term_type), subterm.name)] terms_parsed.add(subterm.id) term.subterms.add(subterm) except: try: log.debug("Created %s" % term) subterm.save() term.subterms.add(subterm) existing_terms[(int(subterm.term_type), subterm.name)] = subterm terms_parsed.add(subterm.id) except IntegrityError: log.error('Duplicated term %s' % term_processor.display_node(subnode)) for term in existing_terms.values(): if not term.id in terms_parsed: log.debug("Deleted %s" % term) term.delete() # Bills bill_index = None if not options.disable_indexing: from bill.search_indexes import BillIndex bill_index = BillIndex() if options.congress and int(options.congress) <= 42: files = glob.glob('data/congress/%s/bills/*/*/*.xml' % options.congress) log.info('Parsing unitedstates/congress bills of only congress#%s' % options.congress) elif options.congress: files = glob.glob('data/us/%s/bills/*.xml' % options.congress) log.info('Parsing bills of only congress#%s' % options.congress) else: files = glob.glob('data/us/*/bills/*.xml') if options.filter: files = [f for f in files if re.match(options.filter, f)] log.info('Processing bills: %d files' % len(files)) total = len(files) progress = Progress(total=total, name='files', step=100) bill_processor = BillProcessor() seen_bill_ids = [] for fname in files: progress.tick() # With indexing or events enabled, if the bill metadata file hasn't changed check # the bill's latest text file for changes so we can create a text-is-available # event and so we can index the bill's text. if (not options.congress or options.congress > 42) and ( bill_index and not options.disable_events ) and not File.objects.is_changed(fname) and not options.force: m = re.search(r"/(\d+)/bills/([a-z]+)(\d+)\.xml$", fname) try: b = Bill.objects.get(congress=m.group(1), bill_type=BillType.by_xml_code( m.group(2)), number=m.group(3)) seen_bill_ids.append(b.id) # Update the index/events for any bill with recently changed text textfile = get_bill_text_metadata(b, None) if not textfile: if b.congress >= 103 and b.introduced_date < ( datetime.now() - timedelta(days=14)).date(): print "No bill text?", fname, b.introduced_date continue textfile = textfile["text_file"] if os.path.exists(textfile) and File.objects.is_changed( textfile): bill_index.update_object( b, using="bill") # index the full text b.create_events() # events for new bill text documents File.objects.save_file(textfile) continue except Bill.DoesNotExist: print "Unchanged metadata file but bill doesn't exist:", fname pass # just parse as normal if options.slow: time.sleep(1) tree = etree.parse(fname) for node in tree.xpath('/bill'): try: bill = bill_processor.process(Bill(), node) except: print fname raise seen_bill_ids.append(bill.id) # don't delete me later if bill.congress >= 93: bill.source = "thomas-congproj" elif bill.congress >= 82: bill.source = "statutesatlarge" if bill.current_status == BillStatus.enacted_signed: bill.current_status = BillStatus.enacted_unknown elif bill.congress <= 42: bill.source = "americanmemory" else: raise ValueError() # So far this is just for American Memory bills. if node.xpath("string(source/@url)"): bill.source_link = unicode(node.xpath("string(source/@url)")) else: bill.source_link = None actions = [] for axn in tree.xpath("actions/*[@state]"): actions.append(( repr( bill_processor.parse_datetime( axn.xpath("string(@datetime)"))), BillStatus.by_xml_code(axn.xpath("string(@state)")), axn.xpath("string(text)"), etree.tostring(axn), )) bill.sliplawpubpriv = None bill.sliplawnum = None for axn in tree.xpath("actions/enacted"): bill.sliplawpubpriv = "PUB" if axn.get( "type") == "public" else "PRI" bill.sliplawnum = int(axn.get("number").split("-")[1]) bill.major_actions = actions try: bill.save() except: print bill raise if bill_index: bill_index.update_object(bill, using="bill") if not options.disable_events: bill.create_events() File.objects.save_file(fname) # delete bill objects that are no longer represented on disk.... this is too dangerous. if options.congress and not options.filter: # this doesn't work because seen_bill_ids is too big for sqlite! for b in Bill.objects.filter(congress=options.congress).exclude( id__in=seen_bill_ids): print "Bill is no longer on disk: ", b.id, b # The rest is for current only... if options.congress and int(options.congress) != CURRENT_CONGRESS: return # Parse docs.house.gov for what might be coming up this week. import iso8601 dhg_html = urllib.urlopen("http://docs.house.gov/floor/").read() m = re.search(r"class=\"downloadXML\" href=\"(Download.aspx\?file=.*?)\"", dhg_html) if not m: log.error( 'No docs.house.gov download link found at http://docs.house.gov.') else: def bt_re(bt): return re.escape(bt[1]).replace(r"\.", r"\.?\s*") try: dhg = etree.parse( urllib.urlopen("http://docs.house.gov/floor/" + m.group(1))).getroot() except: print "http://docs.house.gov/floor/" + m.group(1) raise # iso8601.parse_date(dhg.get("week-date")+"T00:00:00").date() for item in dhg.xpath("category/floor-items/floor-item"): billname = item.xpath("legis-num")[0].text if billname is None: continue # weird but OK m = re.match( r"\s*(?:Concur in the Senate Amendment to |Senate Amendment to )?(" + "|".join(bt_re(bt) for bt in BillType) + r")(\d+)\s*(\[Conference Report\]\s*)?$", billname, re.I) if not m: if not billname.strip().endswith(" __"): log.error( 'Could not parse legis-num "%s" in docs.house.gov.' % billname) else: for bt in BillType: if re.match(bt_re(bt) + "$", m.group(1), re.I): try: bill = Bill.objects.get(congress=CURRENT_CONGRESS, bill_type=bt[0], number=m.group(2)) bill.docs_house_gov_postdate = iso8601.parse_date( item.get("add-date")).replace(tzinfo=None) bill.save() if bill_index: bill_index.update_object(bill, using="bill") if not options.disable_events: bill.create_events() except Bill.DoesNotExist: log.error( 'Could not find bill "%s" in docs.house.gov.' % billname) break else: log.error( 'Could not parse legis-num bill type "%s" in docs.house.gov.' % m.group(1)) # Parse Senate.gov's "Floor Schedule" blurb for coming up tomorrow. now = datetime.now() sfs = urllib.urlopen( "http://www.senate.gov/pagelayout/legislative/d_three_sections_with_teasers/calendars.htm" ).read() try: sfs = re.search(r"Floor Schedule([\w\W]*)Previous Meeting", sfs).group(1) for congress, bill_type, number in re.findall( r"http://hdl.loc.gov/loc.uscongress/legislation.(\d+)([a-z]+)(\d+)", sfs): bill_type = BillType.by_slug(bill_type) bill = Bill.objects.get(congress=congress, bill_type=bill_type, number=number) if bill.senate_floor_schedule_postdate == None or now - bill.senate_floor_schedule_postdate > timedelta( days=7): bill.senate_floor_schedule_postdate = now bill.save() if bill_index: bill_index.update_object(bill, using="bill") if not options.disable_events: bill.create_events() except Exception as e: log.error('Could not parse Senate Floor Schedule: ' + repr(e))
def build_info(): if re.match(r"\d", pk): person = get_object_or_404(Person, pk=pk) else: # support bioguide IDs for me person = get_object_or_404(Person, bioguideid=pk) # current role role = person.get_current_role() if role: active_role = True else: active_role = False try: role = person.roles.order_by('-enddate')[0] except IndexError: role = None # photo photo_url, photo_credit = person.get_photo() # analysis analysis_data = analysis.load_data(person) try: has_session_stats = person.get_session_stats('2016') except: # Not everyone has current stats, obviously. They may have stats # corresponding to their most recent role. Since stats are a # session behind, even-year stats might not correspond to # a legislator's most recent role, which is why I hard-coded # the current session stats above. has_session_stats = False if role: try: has_session_stats = role.get_most_recent_session_stats() except: pass links = [] if role.current: if role.website: links.append(("%s's Official Website" % person.lastname, role.website, "fa fa-external-link")) if person.twitterid: links.append(("@" + person.twitterid, "http://twitter.com/" + person.twitterid, "fa fa-twitter")) if person.osid: links.append(("OpenSecrets", "http://www.opensecrets.org/politicians/summary.php?cid=" + person.osid, "fa fa-money")) if person.pvsid: links.append(("VoteSmart", "http://votesmart.org/candidate/" + person.pvsid, "fa fa-th-list")) if person.bioguideid: links.append(("Bioguide", "http://bioguide.congress.gov/scripts/biodisplay.pl?index=" + person.bioguideid, "fa fa-user")) if person.cspanid: links.append(("C-SPAN", "http://www.c-spanvideo.org/person/" + str(person.cspanid), "fa fa-youtube-play")) # Get a break down of the top terms this person's sponsored bills fall into, # looking only at the most recent five years of bills. from bill.models import BillTerm from datetime import datetime, timedelta most_recent_bill = person.sponsored_bills.order_by("-introduced_date").first() bills_by_subject_counts = list(person.sponsored_bills.filter( terms__id__in=BillTerm.get_top_term_ids(), introduced_date__gt=(most_recent_bill.introduced_date if most_recent_bill else datetime.now())-timedelta(days=5*365.25))\ .values("terms")\ .annotate(count=Count('id')).order_by('-count')\ .filter(count__gt=1)\ [0:8]) terms = BillTerm.objects.in_bulk(item["terms"] for item in bills_by_subject_counts) total_count = sum(item["count"] for item in bills_by_subject_counts) while len(bills_by_subject_counts) > 2 and bills_by_subject_counts[-1]["count"] < bills_by_subject_counts[0]["count"]/8: bills_by_subject_counts.pop(-1) for item in bills_by_subject_counts: item["term"] = terms[item["terms"]] item["pct"] = int(round(float(item["count"]) / total_count * 100)) del item["terms"] # Missed vote explanations from ProPublica try: vote_explanations = http_rest_json("https://projects.propublica.org/explanations/api/members/%s.json" % person.bioguideid) except: # squash all errors vote_explanations = { } return {'person': person, 'role': role, 'active_role': active_role, 'active_congressional_role': active_role and role.role_type in (RoleType.senator, RoleType.representative), 'photo': photo_url, 'photo_credit': photo_credit, 'links': links, 'analysis_data': analysis_data, 'enacted_bills': [b for b in person.sponsored_bills.order_by('-current_status_date') if b.was_enacted_ex()], 'recent_bills': person.sponsored_bills.all().order_by('-introduced_date')[0:7], 'committeeassignments': get_committee_assignments(person), 'feed': person.get_feed(), 'has_session_stats': has_session_stats, 'bill_subject_areas': bills_by_subject_counts, 'vote_explanations': vote_explanations, 'key_votes': load_key_votes(person), }
def build_info(): if re.match(r"\d", pk): person = get_object_or_404(Person, pk=pk) else: # support bioguide IDs for me person = get_object_or_404(Person, bioguideid=pk) # current role role = person.get_current_role() if role: active_role = True else: active_role = False try: role = person.roles.order_by('-enddate')[0] except IndexError: role = None # photo photo_url, photo_credit = person.get_photo() # analysis analysis_data = analysis.load_data(person) try: has_session_stats = person.get_session_stats('2015') except: # Not everyone has 2014 stats, obviously. They may have stats # corresponding to their most recent role. Since stats are a # session behind, even-year stats might not correspond to # a legislator's most recent role, which is why I hard-coded # the current session stats above. has_session_stats = False if role: try: has_session_stats = role.get_most_recent_session_stats() except: pass links = [] if role.current: if role.website: links.append(("%s's Official Website" % person.lastname, role.website, "fa fa-external-link")) if person.twitterid: links.append(("@" + person.twitterid, "http://twitter.com/" + person.twitterid, "fa fa-twitter")) if person.osid: links.append(("OpenSecrets", "http://www.opensecrets.org/politicians/summary.php?cid=" + person.osid, "fa fa-money")) if person.pvsid: links.append(("VoteSmart", "http://votesmart.org/candidate/" + person.pvsid, "fa fa-th-list")) if person.bioguideid: links.append(("Bioguide", "http://bioguide.congress.gov/scripts/biodisplay.pl?index=" + person.bioguideid, "fa fa-user")) if person.cspanid: links.append(("C-SPAN", "http://www.c-spanvideo.org/person/" + str(person.cspanid), "fa fa-youtube-play")) # Get a break down of the top terms this person's sponsored bills fall into, # looking only at the most recent five years of bills. from bill.models import BillTerm from datetime import datetime, timedelta most_recent_bill = person.sponsored_bills.order_by("-introduced_date").first() bills_by_subject_counts = list(person.sponsored_bills.filter( terms__id__in=BillTerm.get_top_term_ids(), introduced_date__gt=(most_recent_bill.introduced_date if most_recent_bill else datetime.now())-timedelta(days=5*365.25))\ .values("terms")\ .annotate(count=Count('id')).order_by('-count')\ .filter(count__gt=1)\ [0:8]) terms = BillTerm.objects.in_bulk(item["terms"] for item in bills_by_subject_counts) total_count = sum(item["count"] for item in bills_by_subject_counts) while len(bills_by_subject_counts) > 2 and bills_by_subject_counts[-1]["count"] < bills_by_subject_counts[0]["count"]/8: bills_by_subject_counts.pop(-1) for item in bills_by_subject_counts: item["term"] = terms[item["terms"]] item["pct"] = int(round(float(item["count"]) / total_count * 100)) del item["terms"] # Missed vote explanations from ProPublica try: vote_explanations = http_rest_json("https://projects.propublica.org/explanations/api/members/%s.json" % person.bioguideid) except: # squash all errors vote_explanations = { } return {'person': person, 'role': role, 'active_role': active_role, 'active_congressional_role': active_role and role.role_type in (RoleType.senator, RoleType.representative), 'photo': photo_url, 'photo_credit': photo_credit, 'links': links, 'analysis_data': analysis_data, 'recent_bills': person.sponsored_bills.all().order_by('-introduced_date')[0:7], 'committeeassignments': get_committee_assignments(person), 'feed': person.get_feed(), 'cities': get_district_cities("%s-%02d" % (role.state.lower(), role.district)) if role and role.district else None, 'has_session_stats': has_session_stats, 'bill_subject_areas': bills_by_subject_counts, 'vote_explanations': vote_explanations, }
def build_info(): global pronunciation_guide if re.match(r"\d", pk): person = get_object_or_404(Person, pk=pk) else: # support bioguide IDs for me person = get_object_or_404(Person, bioguideid=pk) # current role role = person.get_current_role() if role: active_role = True else: active_role = False try: role = person.roles.order_by('-enddate')[0] except IndexError: role = None # photo photo_url, photo_credit = person.get_photo() # analysis analysis_data = analysis.load_data(person) try: # Get session stats for the previous year. has_session_stats = person.get_session_stats(str(datetime.now().year-1)) except: # Not everyone has current stats, obviously. They may have stats # corresponding to their most recent role. Since stats are a # session behind, even-year stats might not correspond to # a legislator's most recent role, which is why I hard-coded # the current session stats above. has_session_stats = False if role: try: has_session_stats = role.get_most_recent_session_stats() except: pass links = [] if role.current: if role.website: links.append(("%s's Official Website" % person.lastname, role.website, "fa fa-external-link")) if person.twitterid: links.append(("@" + person.twitterid, "http://twitter.com/" + person.twitterid, "fa fa-twitter")) if person.osid: links.append(("OpenSecrets", "http://www.opensecrets.org/politicians/summary.php?cid=" + person.osid, "fa fa-money")) if person.pvsid: links.append(("VoteSmart", "http://votesmart.org/candidate/" + person.pvsid, "fa fa-th-list")) if person.bioguideid: links.append(("Bioguide", "http://bioguide.congress.gov/scripts/biodisplay.pl?index=" + person.bioguideid, "fa fa-user")) if person.cspanid: links.append(("C-SPAN", "http://www.c-spanvideo.org/person/" + str(person.cspanid), "fa fa-youtube-play")) # Get a break down of the top terms this person's sponsored bills fall into, # looking only at the most recent five years of bills. from bill.models import BillTerm most_recent_bill = person.sponsored_bills.order_by("-introduced_date").first() bills_by_subject_counts = list(person.sponsored_bills.filter( terms__id__in=BillTerm.get_top_term_ids(), introduced_date__gt=(most_recent_bill.introduced_date if most_recent_bill else datetime.now())-timedelta(days=5*365.25))\ .values("terms")\ .annotate(count=Count('id')).order_by('-count')\ .filter(count__gt=1)\ [0:8]) terms = BillTerm.objects.in_bulk(item["terms"] for item in bills_by_subject_counts) total_count = sum(item["count"] for item in bills_by_subject_counts) while len(bills_by_subject_counts) > 2 and bills_by_subject_counts[-1]["count"] < bills_by_subject_counts[0]["count"]/8: bills_by_subject_counts.pop(-1) for item in bills_by_subject_counts: item["term"] = terms[item["terms"]] item["pct"] = int(round(float(item["count"]) / total_count * 100)) del item["terms"] # Missed vote explanations from ProPublica try: vote_explanations = http_rest_json("https://projects.propublica.org/explanations/api/members/%s.json" % person.bioguideid) except: # squash all errors vote_explanations = { } # Misconduct - load and filter this person's entries, keeping original order. # Choose 'Alleged misconduct', 'Misconduct', 'Misconduct/alleged misconduct' as appropriate. from website.views import load_misconduct_data misconduct = [m for m in load_misconduct_data() if m["person"] == person ] misconduct_any_alleged = (len([ m for m in misconduct if m["alleged"] ]) > 0) misconduct_any_not_alleged = (len([ m for m in misconduct if not m["alleged"] ]) > 0) # Load pronunciation from guide. Turn into a mapping from GovTrack IDs to data. if pronunciation_guide is None: import rtyaml if not hasattr(settings, 'PRONUNCIATION_DATABASE_PATH'): # debugging pronunciation_guide = { } else: pronunciation_guide = { p["id"]["govtrack"]: p for p in rtyaml.load(open(settings.PRONUNCIATION_DATABASE_PATH)) } # Get this person's entry. pronunciation = pronunciation_guide.get(person.id) # TODO: Validate that the 'name' in the guide matches the name we're actually displaying. if pronunciation and not pronunciation.get("key"): # Show a key to the letters used in the pronunciation guide. Break apart the name # into words which we'll show in columns. pronunciation["key"] = [] for namepart in pronunciation["respell"].split(" // "): for nameword in namepart.split(" "): # Parse out the symbols actually used in the guide. Sweep from left to right chopping # off valid respelling letter combinations, chopping off the longest one where possible. pronunciation["key"].append([]) i = 0 while i < len(nameword): for s in sorted(pronunciation_guide_key, key = lambda s : -len(s)): if nameword[i:i+len(s)] in (s, s.upper()): pronunciation["key"][-1].append( (nameword[i:i+len(s)], pronunciation_guide_key[s]) ) i += len(s) break else: # respelling did not match any valid symbol, should be an error but we don't # want to issue an Oops! for this break # Get their enacted bills. enacted_bills_src_qs = person.sponsored_bills.exclude(original_intent_replaced=True).order_by('-current_status_date') return {'person': person, 'role': role, 'active_role': active_role, 'active_congressional_role': active_role and role.role_type in (RoleType.senator, RoleType.representative), 'pronunciation': pronunciation, 'photo': photo_url, 'photo_credit': photo_credit, 'links': links, 'analysis_data': analysis_data, 'enacted_bills': [b for b in enacted_bills_src_qs if b.was_enacted_ex(cache_related_bills_qs=enacted_bills_src_qs)], 'recent_bills': person.sponsored_bills.all().order_by('-introduced_date')[0:7], 'committeeassignments': get_committee_assignments(person), 'feed': person.get_feed(), 'has_session_stats': has_session_stats, 'bill_subject_areas': bills_by_subject_counts, 'vote_explanations': vote_explanations, 'key_votes': load_key_votes(person), 'misconduct': misconduct, 'misconduct_any_alleged': misconduct_any_alleged, 'misconduct_any_not_alleged': misconduct_any_not_alleged, }