def dereference(ref): m = re.match(r"([a-z]+)(\d+)-(\d+)", ref) bill_type = BillType.by_xml_code(m.group(1)) bill = Bill.objects.get(congress=m.group(2), bill_type=bill_type, number=m.group(3)) return bill
def process_bill(self, obj, node): amends_type = BillType.by_xml_code(node.xpath('string(amends/@type)')) amends_number = int(node.xpath('string(amends/@number)')) try: amends_seq = int(node.xpath('string(amends/@sequence)')) except ValueError: amends_seq = None obj.bill = Bill.objects.get(congress=obj.congress, bill_type=amends_type, number=amends_number) obj.sequence = amends_seq
def bill(self): if not hasattr(self, "_ref"): if ":" in self.feedname and self.feedname.split(":")[0] in ("bill",): from bill.models import Bill, BillType m = re.match(r"([a-z]+)(\d+)-(\d+)", self.feedname.split(":")[1]) bill_type = BillType.by_xml_code(m.group(1)) return Bill.objects.get(congress=m.group(2), bill_type=bill_type, number=m.group(3)) else: self._ref = None return self._ref
def process_relatedbills(self, obj, node): RelatedBill.objects.filter(bill=obj).delete() for subnode in node.xpath("./relatedbills/bill"): try: related_bill = Bill.objects.get( congress=subnode.get("session"), bill_type=BillType.by_xml_code(subnode.get("type")), number=int(subnode.get("number")), ) except Bill.DoesNotExist: continue RelatedBill.objects.create(bill=obj, related_bill=related_bill, relation=subnode.get("relation"))
def process_relatedbills(self, obj, node): RelatedBill.objects.filter(bill=obj).delete() for subnode in node.xpath('./relatedbills/bill'): try: related_bill = Bill.objects.get( congress=subnode.get("session"), bill_type=BillType.by_xml_code(subnode.get("type")), number=int(subnode.get("number"))) except Bill.DoesNotExist: continue RelatedBill.objects.create(bill=obj, related_bill=related_bill, relation=subnode.get("relation")[0:16])
def bill(self): if not hasattr(self, "_ref"): if ":" in self.feedname and self.feedname.split(":")[0] in ( "bill", ): from bill.models import Bill, BillType m = re.match(r"([a-z]+)(\d+)-(\d+)", self.feedname.split(":")[1]) bill_type = BillType.by_xml_code(m.group(1)) return Bill.objects.get(congress=m.group(2), bill_type=bill_type, number=m.group(3)) else: self._ref = None return self._ref
def process_bill(self, obj, node): if node.xpath('string(amends/@type)') == "treaty": # Cannot handle this. return False amends_type = BillType.by_xml_code(node.xpath('string(amends/@type)')) amends_number = int(node.xpath('string(amends/@number)')) try: amends_seq = int(node.xpath('string(amends/@sequence)')) except ValueError: amends_seq = None obj.bill = Bill.objects.get(congress=obj.congress, bill_type=amends_type, number=amends_number) obj.sequence = amends_seq return True
def process_bill(self, obj, node): if node.xpath('string(amends/@type)') == "treaty": # Cannot handle this. return False amends_type = BillType.by_xml_code(node.xpath('string(amends/@type)')) amends_number = int(node.xpath('string(amends/@number)')) try: amends_seq = int(node.xpath('string(amends/@sequence)')) except ValueError: amends_seq = None obj.bill = Bill.objects.get(congress=obj.congress, bill_type=amends_type, number=amends_number) obj.sequence = amends_seq return True
def bill_redirect(request, istext=None): """ Redirect requests to obsolete bill urls which look like: /congress/bill.xpd?bill=[type_code][congress_number]-[bill_num] """ token = request.GET.get('bill', '') match = BILL_TOKEN_REGEXP.search(token) if not match: raise Http404() type_code, congress, number = match.groups() try: bill_type = BillType.by_xml_code(type_code) except BillType.NotFound: raise Http404() bill = get_object_or_404(Bill, bill_type=bill_type, congress=congress, number=number) return redirect(bill.get_absolute_url() + ("" if not istext else "/text"), permanent=True)
def main(options): """ Parse rolls. """ # Setup XML processors vote_processor = VoteProcessor() option_processor = VoteOptionProcessor() voter_processor = VoterProcessor() voter_processor.PERSON_CACHE = dict((x.pk, x) for x in Person.objects.all()) # The pattern which the roll file matches # Filename contains info which should be placed to DB # along with info extracted from the XML file re_path = re.compile('data/us/(\d+)/rolls/([hs])(\w+)-(\d+)\.xml') chamber_mapping = {'s': CongressChamber.senate, 'h': CongressChamber.house} if options.filter: files = glob.glob(options.filter) log.info('Parsing rolls matching %s' % options.filter) elif options.congress: files = glob.glob('data/us/%s/rolls/*.xml' % options.congress) log.info('Parsing rolls of only congress#%s' % options.congress) else: files = glob.glob('data/us/*/rolls/*.xml') log.info('Processing votes: %d files' % len(files)) total = len(files) progress = Progress(total=total, name='files', step=10) def log_delete_qs(qs): if qs.count() > 0: try: print "Deleting: ", qs except Exception as e: print "Deleting [%s]..." % str(e) if qs.count() > 3: print "Delete skipped..." return qs.delete() seen_obj_ids = set() had_error = False for fname in files: progress.tick() match = re_path.search(fname) try: existing_vote = Vote.objects.get(congress=match.group(1), chamber=chamber_mapping[match.group(2)], session=match.group(3), number=match.group(4)) except Vote.DoesNotExist: existing_vote = None if not File.objects.is_changed(fname) and not options.force and existing_vote != None and not existing_vote.missing_data: seen_obj_ids.add(existing_vote.id) continue try: tree = etree.parse(fname) ## Look for votes with VP tie breakers. #if len(tree.xpath("/roll/voter[@VP='1']")) == 0: # had_error = True # prevent delete at the end # continue # Process role object for roll_node in tree.xpath('/roll'): vote = vote_processor.process(Vote(), roll_node) if existing_vote: vote.id = existing_vote.id match = re_path.search(fname) vote.congress = int(match.group(1)) vote.chamber = chamber_mapping[match.group(2)] vote.session = match.group(3) vote.number = int(match.group(4)) for bill_node in roll_node.xpath("bill"): try: vote.related_bill = Bill.objects.get(congress=bill_node.get("session"), bill_type=BillType.by_xml_code(bill_node.get("type")), number=bill_node.get("number")) # for votes on passage, reverse the order of the title so that the # name of the bill comes first, but keep the vote_type at the end # to distinguish suspension votes etc. also, the title that comes # from the upstream source is not formatted in our style. if vote.category in (VoteCategory.passage, VoteCategory.passage_suspension, VoteCategory.veto_override): vote.question = truncatewords(vote.related_bill.title, 12) + " (" + vote.vote_type + ")" except Bill.DoesNotExist: vote.missing_data = True vote.save() seen_obj_ids.add(vote.id) # don't delete me later # Process roll options, overwrite existing options where possible. seen_option_ids = set() roll_options = {} for option_node in roll_node.xpath('./option'): option = option_processor.process(VoteOption(), option_node) option.vote = vote if existing_vote: try: option.id = VoteOption.objects.filter(vote=vote, key=option.key)[0].id # get is better, but I had the database corruption problem except IndexError: pass option.save() roll_options[option.key] = option seen_option_ids.add(option.id) log_delete_qs(VoteOption.objects.filter(vote=vote).exclude(id__in=seen_option_ids)) # may cascade and delete the Voters too? # Process roll voters, overwriting existing voters where possible. if existing_vote: existing_voters = dict(Voter.objects.filter(vote=vote).values_list("person", "id")) seen_voter_ids = set() for voter_node in roll_node.xpath('./voter'): voter = voter_processor.process(roll_options, Voter(), voter_node) voter.vote = vote voter.created = vote.created # for VP votes, load the actual person... if voter.voter_type == VoterType.vice_president: try: r = PersonRole.objects.get(role_type=RoleType.vicepresident, startdate__lte=vote.created, enddate__gte=vote.created) voter.person = r.person except: # overlapping roles? missing data? log.error('Could not resolve vice president in %s' % fname, exc_info=ex) if existing_vote and voter.person: try: voter.id = existing_voters[voter.person.id] except KeyError: pass voter.save() if voter.voter_type == VoterType.unknown and not vote.missing_data: vote.missing_data = True vote.save() seen_voter_ids.add(voter.id) log_delete_qs(Voter.objects.filter(vote=vote).exclude(id__in=seen_voter_ids)) # possibly already deleted by cascade above vote.calculate_totals() if not options.disable_events: vote.create_event() File.objects.save_file(fname) except Exception, ex: log.error('Error in processing %s' % fname, exc_info=ex) had_error = True
def main(options): """ Process bill terms and bills """ # Terms term_processor = TermProcessor() terms_parsed = set() # Cache existing terms. There aren't so many. existing_terms = { } for term in BillTerm.objects.all(): existing_terms[(int(term.term_type), term.name)] = term log.info('Processing old bill terms') TERMS_FILE = 'data/us/liv.xml' tree = etree.parse(TERMS_FILE) for node in tree.xpath('/liv/top-term'): term = term_processor.process(BillTerm(), node) term.term_type = TermType.old try: # No need to update an existing term because there are no other attributes. term = existing_terms[(int(term.term_type), term.name)] terms_parsed.add(term.id) except: log.debug("Created %s" % term) term.save() term.subterms.clear() for subnode in node.xpath('./term'): subterm = term_processor.process(BillTerm(), subnode) subterm.term_type = TermType.old try: # No need to update an existing term because there are no other attributes. subterm = existing_terms[(int(subterm.term_type), subterm.name)] term.subterms.add(subterm) terms_parsed.add(subterm.id) except: try: log.debug("Created %s" % subterm) subterm.save() term.subterms.add(subterm) existing_terms[(int(subterm.term_type), subterm.name)] = subterm terms_parsed.add(subterm.id) except IntegrityError: log.error('Duplicated term %s' % term_processor.display_node(subnode)) log.info('Processing new bill terms') for FILE in ('data/us/liv111.xml', 'data/us/crsnet.xml'): tree = etree.parse(FILE) for node in tree.xpath('/liv/top-term'): term = term_processor.process(BillTerm(), node) term.term_type = TermType.new try: # No need to update an existing term because there are no other attributes. term = existing_terms[(int(term.term_type), term.name)] terms_parsed.add(term.id) except: log.debug("Created %s" % term) term.save() term.subterms.clear() for subnode in node.xpath('./term'): subterm = term_processor.process(BillTerm(), subnode) subterm.term_type = TermType.new try: # No need to update an existing term because there are no other attributes. subterm = existing_terms[(int(subterm.term_type), subterm.name)] terms_parsed.add(subterm.id) term.subterms.add(subterm) except: try: log.debug("Created %s" % term) subterm.save() term.subterms.add(subterm) existing_terms[(int(subterm.term_type), subterm.name)] = subterm terms_parsed.add(subterm.id) except IntegrityError: log.error('Duplicated term %s' % term_processor.display_node(subnode)) for term in existing_terms.values(): if not term.id in terms_parsed: log.debug("Deleted %s" % term) term.delete() # Bills bill_index = None if not options.disable_indexing: from bill.search_indexes import BillIndex bill_index = BillIndex() if options.congress and int(options.congress) <= 42: files = glob.glob('data/congress/%s/bills/*/*/*.xml' % options.congress) log.info('Parsing unitedstates/congress bills of only congress#%s' % options.congress) elif options.congress: files = glob.glob('data/us/%s/bills/*.xml' % options.congress) log.info('Parsing bills of only congress#%s' % options.congress) else: files = glob.glob('data/us/*/bills/*.xml') if options.filter: files = [f for f in files if re.match(options.filter, f)] log.info('Processing bills: %d files' % len(files)) total = len(files) progress = Progress(total=total, name='files', step=100) bill_processor = BillProcessor() seen_bill_ids = [] for fname in files: progress.tick() # With indexing or events enabled, if the bill metadata file hasn't changed check # the bill's latest text file for changes so we can create a text-is-available # event and so we can index the bill's text. if (not options.congress or options.congress>42) and (bill_index and not options.disable_events) and not File.objects.is_changed(fname) and not options.force: m = re.search(r"/(\d+)/bills/([a-z]+)(\d+)\.xml$", fname) try: b = Bill.objects.get(congress=m.group(1), bill_type=BillType.by_xml_code(m.group(2)), number=m.group(3)) seen_bill_ids.append(b.id) # Update the index/events for any bill with recently changed text textfile = get_bill_text_metadata(b, None) if not textfile: if b.congress >= 103 and b.introduced_date < (datetime.now()-timedelta(days=14)).date(): print "No bill text?", fname, b.introduced_date continue textfile = textfile["text_file"] if os.path.exists(textfile) and File.objects.is_changed(textfile): b.update_index(bill_index) # index the full text b.create_events() # events for new bill text documents File.objects.save_file(textfile) continue except Bill.DoesNotExist: print "Unchanged metadata file but bill doesn't exist:", fname pass # just parse as normal if options.slow: time.sleep(1) tree = etree.parse(fname) for node in tree.xpath('/bill'): try: bill = bill_processor.process(Bill(), node) except: print fname raise seen_bill_ids.append(bill.id) # don't delete me later # So far this is just for American Memory bills. if node.xpath("string(source/@url)"): bill.source_link = unicode(node.xpath("string(source/@url)")) else: bill.source_link = None actions = [] for axn in tree.xpath("actions/*[@state]"): actions.append( ( repr(bill_processor.parse_datetime(axn.xpath("string(@datetime)"))), BillStatus.by_xml_code(axn.xpath("string(@state)")), axn.xpath("string(text)"), etree.tostring(axn), ) ) bill.sliplawpubpriv = None bill.sliplawnum = None for axn in tree.xpath("actions/enacted"): bill.sliplawpubpriv = "PUB" if axn.get("type") == "public" else "PRI" bill.sliplawnum = int(axn.get("number").split("-")[1]) bill.major_actions = actions try: bill.save() except: print bill raise if bill_index: bill.update_index(bill_index) if not options.disable_events: bill.create_events() File.objects.save_file(fname) # delete bill objects that are no longer represented on disk.... this is too dangerous. if options.congress and not options.filter: # this doesn't work because seen_bill_ids is too big for sqlite! for b in Bill.objects.filter(congress=options.congress).exclude(id__in = seen_bill_ids): print "Bill is no longer on disk: ", b.id, b # The rest is for current only... if options.congress and int(options.congress) != CURRENT_CONGRESS: return # Find what might be coming up this week. load_docs_house_gov(options, bill_index) load_senate_floor_schedule(options, bill_index)
def main(options): """ Process bill terms and bills """ # Terms term_processor = TermProcessor() terms_parsed = set() # Cache existing terms. There aren't so many. existing_terms = {} for term in BillTerm.objects.all(): existing_terms[(int(term.term_type), term.name)] = term log.info('Processing old bill terms') TERMS_FILE = 'data/us/liv.xml' tree = etree.parse(TERMS_FILE) for node in tree.xpath('/liv/top-term'): term = term_processor.process(BillTerm(), node) term.term_type = TermType.old try: # No need to update an existing term because there are no other attributes. term = existing_terms[(int(term.term_type), term.name)] terms_parsed.add(term.id) except: log.debug("Created %s" % term) term.save() term.subterms.clear() for subnode in node.xpath('./term'): subterm = term_processor.process(BillTerm(), subnode) subterm.term_type = TermType.old try: # No need to update an existing term because there are no other attributes. subterm = existing_terms[(int(subterm.term_type), subterm.name)] term.subterms.add(subterm) terms_parsed.add(subterm.id) except: try: log.debug("Created %s" % subterm) subterm.save() term.subterms.add(subterm) existing_terms[(int(subterm.term_type), subterm.name)] = subterm terms_parsed.add(subterm.id) except IntegrityError: log.error('Duplicated term %s' % term_processor.display_node(subnode)) log.info('Processing new bill terms') for FILE in ('data/us/liv111.xml', 'data/us/crsnet.xml'): tree = etree.parse(FILE) for node in tree.xpath('/liv/top-term'): term = term_processor.process(BillTerm(), node) term.term_type = TermType.new try: # No need to update an existing term because there are no other attributes. term = existing_terms[(int(term.term_type), term.name)] terms_parsed.add(term.id) except: log.debug("Created %s" % term) term.save() term.subterms.clear() for subnode in node.xpath('./term'): subterm = term_processor.process(BillTerm(), subnode) subterm.term_type = TermType.new try: # No need to update an existing term because there are no other attributes. subterm = existing_terms[(int(subterm.term_type), subterm.name)] terms_parsed.add(subterm.id) term.subterms.add(subterm) except: try: log.debug("Created %s" % term) subterm.save() term.subterms.add(subterm) existing_terms[(int(subterm.term_type), subterm.name)] = subterm terms_parsed.add(subterm.id) except IntegrityError: log.error('Duplicated term %s' % term_processor.display_node(subnode)) for term in existing_terms.values(): if not term.id in terms_parsed: log.debug("Deleted %s" % term) term.delete() # Bills bill_index = None if not options.disable_indexing: from bill.search_indexes import BillIndex bill_index = BillIndex() if options.congress and int(options.congress) <= 42: files = glob.glob('data/congress/%s/bills/*/*/*.xml' % options.congress) log.info('Parsing unitedstates/congress bills of only congress#%s' % options.congress) elif options.congress: files = glob.glob('data/us/%s/bills/*.xml' % options.congress) log.info('Parsing bills of only congress#%s' % options.congress) else: files = glob.glob('data/us/*/bills/*.xml') if options.filter: files = [f for f in files if re.match(options.filter, f)] log.info('Processing bills: %d files' % len(files)) total = len(files) progress = Progress(total=total, name='files', step=100) bill_processor = BillProcessor() seen_bill_ids = [] for fname in files: progress.tick() # With indexing or events enabled, if the bill metadata file hasn't changed check # the bill's latest text file for changes so we can create a text-is-available # event and so we can index the bill's text. if (not options.congress or options.congress > 42) and ( bill_index and not options.disable_events ) and not File.objects.is_changed(fname) and not options.force: m = re.search(r"/(\d+)/bills/([a-z]+)(\d+)\.xml$", fname) try: b = Bill.objects.get(congress=m.group(1), bill_type=BillType.by_xml_code( m.group(2)), number=m.group(3)) seen_bill_ids.append(b.id) # Update the index/events for any bill with recently changed text textfile = get_bill_text_metadata(b, None) if not textfile: if b.congress >= 103 and b.introduced_date < ( datetime.now() - timedelta(days=14)).date(): print "No bill text?", fname, b.introduced_date continue textfile = textfile["text_file"] if os.path.exists(textfile) and File.objects.is_changed( textfile): b.update_index(bill_index) # index the full text b.create_events() # events for new bill text documents File.objects.save_file(textfile) continue except Bill.DoesNotExist: print "Unchanged metadata file but bill doesn't exist:", fname pass # just parse as normal if options.slow: time.sleep(1) tree = etree.parse(fname) for node in tree.xpath('/bill'): try: bill = bill_processor.process(Bill(), node) except: print fname raise seen_bill_ids.append(bill.id) # don't delete me later # So far this is just for American Memory bills. if node.xpath("string(source/@url)"): bill.source_link = unicode(node.xpath("string(source/@url)")) else: bill.source_link = None actions = [] for axn in tree.xpath("actions/*[@state]"): if axn.xpath("string(@state)") == "REFERRED": continue # we don't track this state actions.append(( repr( bill_processor.parse_datetime( axn.xpath("string(@datetime)"))), BillStatus.by_xml_code(axn.xpath("string(@state)")), axn.xpath("string(text)"), etree.tostring(axn), )) bill.sliplawpubpriv = None bill.sliplawnum = None for axn in tree.xpath("actions/enacted"): bill.sliplawpubpriv = "PUB" if axn.get( "type") == "public" else "PRI" bill.sliplawnum = int(axn.get("number").split("-")[1]) bill.major_actions = actions try: bill.save() except: print bill raise if bill_index: bill.update_index(bill_index) if not options.disable_events: bill.create_events() File.objects.save_file(fname) # delete bill objects that are no longer represented on disk.... this is too dangerous. if options.congress and not options.filter: # this doesn't work because seen_bill_ids is too big for sqlite! for b in Bill.objects.filter(congress=options.congress).exclude( id__in=seen_bill_ids): print "Bill is no longer on disk: ", b.id, b # The rest is for current only... if options.congress and int(options.congress) != CURRENT_CONGRESS: return # Find what might be coming up this week. load_docs_house_gov(options, bill_index) load_senate_floor_schedule(options, bill_index)
def main(options): """ Process bill terms and bills """ # Terms term_processor = TermProcessor() terms_parsed = set() # Cache existing terms. There aren't so many. existing_terms = {} for term in BillTerm.objects.all(): existing_terms[(int(term.term_type), term.name)] = term log.info('Processing old bill terms') TERMS_FILE = 'data/us/liv.xml' tree = etree.parse(TERMS_FILE) for node in tree.xpath('/liv/top-term'): term = term_processor.process(BillTerm(), node) term.term_type = TermType.old try: # No need to update an existing term because there are no other attributes. term = existing_terms[(int(term.term_type), term.name)] terms_parsed.add(term.id) except: log.debug("Created %s" % term) term.save() term.subterms.clear() for subnode in node.xpath('./term'): subterm = term_processor.process(BillTerm(), subnode) subterm.term_type = TermType.old try: # No need to update an existing term because there are no other attributes. subterm = existing_terms[(int(subterm.term_type), subterm.name)] term.subterms.add(subterm) terms_parsed.add(subterm.id) except: try: log.debug("Created %s" % subterm) subterm.save() term.subterms.add(subterm) existing_terms[(int(subterm.term_type), subterm.name)] = subterm terms_parsed.add(subterm.id) except IntegrityError: log.error('Duplicated term %s' % term_processor.display_node(subnode)) log.info('Processing new bill terms') for FILE in ('data/us/liv111.xml', 'data/us/crsnet.xml'): tree = etree.parse(FILE) for node in tree.xpath('/liv/top-term'): term = term_processor.process(BillTerm(), node) term.term_type = TermType.new try: # No need to update an existing term because there are no other attributes. term = existing_terms[(int(term.term_type), term.name)] terms_parsed.add(term.id) except: log.debug("Created %s" % term) term.save() term.subterms.clear() for subnode in node.xpath('./term'): subterm = term_processor.process(BillTerm(), subnode) subterm.term_type = TermType.new try: # No need to update an existing term because there are no other attributes. subterm = existing_terms[(int(subterm.term_type), subterm.name)] terms_parsed.add(subterm.id) term.subterms.add(subterm) except: try: log.debug("Created %s" % term) subterm.save() term.subterms.add(subterm) existing_terms[(int(subterm.term_type), subterm.name)] = subterm terms_parsed.add(subterm.id) except IntegrityError: log.error('Duplicated term %s' % term_processor.display_node(subnode)) for term in existing_terms.values(): if not term.id in terms_parsed: log.debug("Deleted %s" % term) term.delete() # Bills bill_index = None if not options.disable_indexing: from bill.search_indexes import BillIndex bill_index = BillIndex() if options.congress and int(options.congress) <= 42: files = glob.glob('data/congress/%s/bills/*/*/*.xml' % options.congress) log.info('Parsing unitedstates/congress bills of only congress#%s' % options.congress) elif options.congress: files = glob.glob('data/us/%s/bills/*.xml' % options.congress) log.info('Parsing bills of only congress#%s' % options.congress) else: files = glob.glob('data/us/*/bills/*.xml') if options.filter: files = [f for f in files if re.match(options.filter, f)] log.info('Processing bills: %d files' % len(files)) total = len(files) progress = Progress(total=total, name='files', step=100) bill_processor = BillProcessor() seen_bill_ids = [] for fname in files: progress.tick() # With indexing or events enabled, if the bill metadata file hasn't changed check # the bill's latest text file for changes so we can create a text-is-available # event and so we can index the bill's text. if (not options.congress or options.congress > 42) and ( bill_index and not options.disable_events ) and not File.objects.is_changed(fname) and not options.force: m = re.search(r"/(\d+)/bills/([a-z]+)(\d+)\.xml$", fname) try: b = Bill.objects.get(congress=m.group(1), bill_type=BillType.by_xml_code( m.group(2)), number=m.group(3)) seen_bill_ids.append(b.id) # Update the index/events for any bill with recently changed text textfile = get_bill_text_metadata(b, None) if not textfile: if b.congress >= 103 and b.introduced_date < ( datetime.now() - timedelta(days=14)).date(): print "No bill text?", fname, b.introduced_date continue textfile = textfile["text_file"] if os.path.exists(textfile) and File.objects.is_changed( textfile): bill_index.update_object( b, using="bill") # index the full text b.create_events() # events for new bill text documents File.objects.save_file(textfile) continue except Bill.DoesNotExist: print "Unchanged metadata file but bill doesn't exist:", fname pass # just parse as normal if options.slow: time.sleep(1) tree = etree.parse(fname) for node in tree.xpath('/bill'): try: bill = bill_processor.process(Bill(), node) except: print fname raise seen_bill_ids.append(bill.id) # don't delete me later if bill.congress >= 93: bill.source = "thomas-congproj" elif bill.congress >= 82: bill.source = "statutesatlarge" if bill.current_status == BillStatus.enacted_signed: bill.current_status = BillStatus.enacted_unknown elif bill.congress <= 42: bill.source = "americanmemory" else: raise ValueError() # So far this is just for American Memory bills. if node.xpath("string(source/@url)"): bill.source_link = unicode(node.xpath("string(source/@url)")) else: bill.source_link = None actions = [] for axn in tree.xpath("actions/*[@state]"): actions.append(( repr( bill_processor.parse_datetime( axn.xpath("string(@datetime)"))), BillStatus.by_xml_code(axn.xpath("string(@state)")), axn.xpath("string(text)"), etree.tostring(axn), )) bill.sliplawpubpriv = None bill.sliplawnum = None for axn in tree.xpath("actions/enacted"): bill.sliplawpubpriv = "PUB" if axn.get( "type") == "public" else "PRI" bill.sliplawnum = int(axn.get("number").split("-")[1]) bill.major_actions = actions try: bill.save() except: print bill raise if bill_index: bill_index.update_object(bill, using="bill") if not options.disable_events: bill.create_events() File.objects.save_file(fname) # delete bill objects that are no longer represented on disk.... this is too dangerous. if options.congress and not options.filter: # this doesn't work because seen_bill_ids is too big for sqlite! for b in Bill.objects.filter(congress=options.congress).exclude( id__in=seen_bill_ids): print "Bill is no longer on disk: ", b.id, b # The rest is for current only... if options.congress and int(options.congress) != CURRENT_CONGRESS: return # Parse docs.house.gov for what might be coming up this week. import iso8601 dhg_html = urllib.urlopen("http://docs.house.gov/floor/").read() m = re.search(r"class=\"downloadXML\" href=\"(Download.aspx\?file=.*?)\"", dhg_html) if not m: log.error( 'No docs.house.gov download link found at http://docs.house.gov.') else: def bt_re(bt): return re.escape(bt[1]).replace(r"\.", r"\.?\s*") try: dhg = etree.parse( urllib.urlopen("http://docs.house.gov/floor/" + m.group(1))).getroot() except: print "http://docs.house.gov/floor/" + m.group(1) raise # iso8601.parse_date(dhg.get("week-date")+"T00:00:00").date() for item in dhg.xpath("category/floor-items/floor-item"): billname = item.xpath("legis-num")[0].text if billname is None: continue # weird but OK m = re.match( r"\s*(?:Concur in the Senate Amendment to |Senate Amendment to )?(" + "|".join(bt_re(bt) for bt in BillType) + r")(\d+)\s*(\[Conference Report\]\s*)?$", billname, re.I) if not m: if not billname.strip().endswith(" __"): log.error( 'Could not parse legis-num "%s" in docs.house.gov.' % billname) else: for bt in BillType: if re.match(bt_re(bt) + "$", m.group(1), re.I): try: bill = Bill.objects.get(congress=CURRENT_CONGRESS, bill_type=bt[0], number=m.group(2)) bill.docs_house_gov_postdate = iso8601.parse_date( item.get("add-date")).replace(tzinfo=None) bill.save() if bill_index: bill_index.update_object(bill, using="bill") if not options.disable_events: bill.create_events() except Bill.DoesNotExist: log.error( 'Could not find bill "%s" in docs.house.gov.' % billname) break else: log.error( 'Could not parse legis-num bill type "%s" in docs.house.gov.' % m.group(1)) # Parse Senate.gov's "Floor Schedule" blurb for coming up tomorrow. now = datetime.now() sfs = urllib.urlopen( "http://www.senate.gov/pagelayout/legislative/d_three_sections_with_teasers/calendars.htm" ).read() try: sfs = re.search(r"Floor Schedule([\w\W]*)Previous Meeting", sfs).group(1) for congress, bill_type, number in re.findall( r"http://hdl.loc.gov/loc.uscongress/legislation.(\d+)([a-z]+)(\d+)", sfs): bill_type = BillType.by_slug(bill_type) bill = Bill.objects.get(congress=congress, bill_type=bill_type, number=number) if bill.senate_floor_schedule_postdate == None or now - bill.senate_floor_schedule_postdate > timedelta( days=7): bill.senate_floor_schedule_postdate = now bill.save() if bill_index: bill_index.update_object(bill, using="bill") if not options.disable_events: bill.create_events() except Exception as e: log.error('Could not parse Senate Floor Schedule: ' + repr(e))
#!script from bill.models import BillType import glob, re, os.path for congress in range(103, 112+1): for fn in glob.glob("data/us/bills.text/%d/*/*.xml" % congress): bill_type, bill_number, print_code, stuff = re.search(r"/([a-z]+)(\d+)([a-z][a-z0-9]*)?(\.gen|\.mods)?\.xml$", fn).groups() if not print_code: continue # my symbolic links to latest version if stuff: continue # ".gen".html bill_type = BillType.by_xml_code(bill_type).slug fn2 = "data/congress/%d/bills/%s/%s%s/text-versions/%s/mods.xml" % ( congress, bill_type, bill_type, bill_number, print_code) if not os.path.exists(fn2): print fn2
def main(options): """ Parse rolls. """ # Setup XML processors vote_processor = VoteProcessor() option_processor = VoteOptionProcessor() voter_processor = VoterProcessor() voter_processor.PERSON_CACHE = dict( (x.pk, x) for x in Person.objects.all()) # The pattern which the roll file matches # Filename contains info which should be placed to DB # along with info extracted from the XML file re_path = re.compile('data/us/(\d+)/rolls/([hs])(\w+)-(\d+)\.xml') chamber_mapping = {'s': CongressChamber.senate, 'h': CongressChamber.house} if options.filter: files = glob.glob(options.filter) log.info('Parsing rolls matching %s' % options.filter) elif options.congress: files = glob.glob('data/us/%s/rolls/*.xml' % options.congress) log.info('Parsing rolls of only congress#%s' % options.congress) else: files = glob.glob('data/us/*/rolls/*.xml') log.info('Processing votes: %d files' % len(files)) total = len(files) progress = Progress(total=total, name='files', step=10) def log_delete_qs(qs): if qs.count() == 0: return print "Deleting obsoleted records: ", qs #if qs.count() > 3: # print "Delete skipped..." # return qs.delete() seen_obj_ids = set() had_error = False for fname in files: progress.tick() match = re_path.search(fname) try: existing_vote = Vote.objects.get( congress=match.group(1), chamber=chamber_mapping[match.group(2)], session=match.group(3), number=match.group(4)) except Vote.DoesNotExist: existing_vote = None if not File.objects.is_changed( fname ) and not options.force and existing_vote != None and not existing_vote.missing_data: seen_obj_ids.add(existing_vote.id) continue try: tree = etree.parse(fname) ## Look for votes with VP tie breakers. #if len(tree.xpath("/roll/voter[@VP='1']")) == 0: # had_error = True # prevent delete at the end # continue # Process role object roll_node = tree.xpath('/roll')[0] # Sqlite is much faster when lots of saves are wrapped in a transaction, # and we do a lot of saves because it's a lot of voters. from django.db import transaction with transaction.atomic(): vote = vote_processor.process(Vote(), roll_node) if existing_vote: vote.id = existing_vote.id match = re_path.search(fname) vote.congress = int(match.group(1)) vote.chamber = chamber_mapping[match.group(2)] vote.session = match.group(3) vote.number = int(match.group(4)) # Get related bill & amendment. for bill_node in roll_node.xpath("bill"): related_bill_num = bill_node.get("number") if 9 <= vote.congress <= 42 and vote.session in ('1', '2'): # Bill numbering from the American Memory colletion is different. The number combines # the session, bill number, and a 0 or 5 for regular or 'half' numbering. Prior to # the 9th congress numbering seems to be wholly assigned by us and not related to # actual numbering, so we skip matching those bills. related_bill_num = "%d%04d%d" % (int( vote.session), int(bill_node.get("number")), 0) try: vote.related_bill = Bill.objects.get( congress=bill_node.get("session"), bill_type=BillType.by_xml_code( bill_node.get("type")), number=related_bill_num) except Bill.DoesNotExist: if vote.congress >= 93: vote.missing_data = True for amdt_node in roll_node.xpath("amendment"): if amdt_node.get( "ref" ) == "regular" and vote.related_bill is not None: try: vote.related_amendment = Amendment.objects.get( congress=vote.related_bill.congress, amendment_type=AmendmentType.by_slug( amdt_node.get("number")[0]), number=amdt_node.get("number")[1:]) except Amendment.DoesNotExist: if vote.congress >= 93: print "Missing amendment", fname vote.missing_data = True elif amdt_node.get("ref") == "bill-serial": # It is impossible to associate House votes with amendments just from the House # vote XML because the amendment-num might correspond either with the A___ number # or with the "An amendment, numbered ____" number from the amendment purpose, # and there's really no way to figure out which. Maybe we can use the amendment # sponsor instead? #vote.related_amendment = Amendment.objects.get(bill=vote.related_bill, sequence=amdt_node.get("number")) # Instead, we set related_amendment from the amendment parser. Here, we have to # preserve the related_amendment if it is set. if existing_vote: vote.related_amendment = existing_vote.related_amendment # clean up some question text and use the question_details field if vote.category in ( VoteCategory.passage, VoteCategory.passage_suspension, VoteCategory.veto_override) and vote.related_bill: # For passage votes, set the question to the bill title and put the question # details in the details field. vote.question = vote.related_bill.title vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display( ) elif vote.category == VoteCategory.amendment and vote.related_amendment: # For votes on amendments, make a better title/explanation. vote.question = vote.related_amendment.title vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display( ) elif vote.related_bill and vote.question.startswith( "On the Cloture Motion " + vote.related_bill.display_number): vote.question = "Cloture on " + vote.related_bill.title elif vote.related_bill and vote.question.startswith( "On Cloture on the Motion to Proceed " + vote.related_bill.display_number): vote.question = "Cloture on " + vote.related_bill.title vote.question_details = "On Cloture on the Motion to Proceed in the " + vote.get_chamber_display( ) elif vote.related_bill and vote.question.startswith( "On the Motion to Proceed " + vote.related_bill.display_number): vote.question = "Motion to Proceed on " + vote.related_bill.title elif vote.related_amendment and vote.question.startswith( "On the Cloture Motion " + vote.related_amendment.get_amendment_type_display() + " " + str(vote.related_amendment.number)): vote.question = "Cloture on " + vote.related_amendment.title vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display( ) # weird House foratting of bill numbers ("H RES 123 Blah blah") if vote.related_bill: vote.question = re.sub( "(On [^:]+): " + vote.related_bill.display_number.replace( ". ", " ").replace(".", " ").upper() + " .*", r"\1: " + truncatewords(vote.related_bill.title, 15), vote.question) vote.save() seen_obj_ids.add(vote.id) # don't delete me later # Process roll options, overwrite existing options where possible. seen_option_ids = set() roll_options = {} for option_node in roll_node.xpath('./option'): option = option_processor.process(VoteOption(), option_node) option.vote = vote if existing_vote: try: option.id = VoteOption.objects.filter( vote=vote, key=option.key )[0].id # get is better, but I had the database corruption problem except IndexError: pass option.save() roll_options[option.key] = option seen_option_ids.add(option.id) log_delete_qs( VoteOption.objects.filter(vote=vote).exclude( id__in=seen_option_ids) ) # may cascade and delete the Voters too? # Process roll voters, overwriting existing voters where possible. if existing_vote: existing_voters = dict( Voter.objects.filter(vote=vote).values_list( "person", "id")) seen_voter_ids = set() voters = list() for voter_node in roll_node.xpath('./voter'): voter = voter_processor.process(roll_options, Voter(), voter_node) voter.vote = vote voter.created = vote.created # for VP votes, load the actual person & role... if voter.voter_type == VoterType.vice_president: try: r = PersonRole.objects.get( role_type=RoleType.vicepresident, startdate__lte=vote.created, enddate__gte=vote.created) voter.person_role = r voter.person = r.person except PersonRole.DoesNotExist: # overlapping roles? missing data? log.error( 'Could not resolve vice president in %s' % fname) if existing_vote and voter.person: try: voter.id = existing_voters[voter.person.id] except KeyError: pass voters.append(voter) if voter.voter_type == VoterType.unknown and not vote.missing_data: vote.missing_data = True vote.save() # pre-fetch the role of each voter load_roles_at_date( [x.person for x in voters if x.person != None], vote.created, vote.congress) for voter in list(voters): if voter.voter_type != VoterType.vice_president: voter.person_role = voter.person.role # If we couldn't match a role for this person on the date of the vote, and if the voter was Not Voting, # and we're looking at historical data, then this is probably a data error --- the voter wasn't even in office. if voter.person_role is None: if vote.source == VoteSource.keithpoole and voter.option.key == "0": # Drop this record. voters.remove(voter) else: log.error("%s: Could not find role for %s on %s." % (fname, voter.person, vote.created)) vote.missing_data = True vote.save() # save all of the records (inserting/updating) for voter in voters: voter.save() seen_voter_ids.add(voter.id) # remove obsolete voter records log_delete_qs( Voter.objects.filter(vote=vote).exclude( id__in=seen_voter_ids) ) # possibly already deleted by cascade above # pre-calculate totals vote.calculate_totals() if not options.disable_events: vote.create_event() File.objects.save_file(fname) except Exception, ex: log.error('Error in processing %s' % fname, exc_info=ex) had_error = True
#!script from bill.models import BillType import glob, re, os.path for congress in range(103, 112 + 1): for fn in glob.glob("data/us/bills.text/%d/*/*.xml" % congress): bill_type, bill_number, print_code, stuff = re.search( r"/([a-z]+)(\d+)([a-z][a-z0-9]*)?(\.gen|\.mods)?\.xml$", fn).groups() if not print_code: continue # my symbolic links to latest version if stuff: continue # ".gen".html bill_type = BillType.by_xml_code(bill_type).slug fn2 = "data/congress/%d/bills/%s/%s%s/text-versions/%s/mods.xml" % ( congress, bill_type, bill_type, bill_number, print_code) if not os.path.exists(fn2): print(fn2)
def type_handler(self, value): return BillType.by_xml_code(value)
def type_handler(self, value): return BillType.by_xml_code(value)
def dereference(ref): m = re.match(r"([a-z]+)(\d+)-(\d+)", ref) bill_type = BillType.by_xml_code(m.group(1)) bill = Bill.objects.get(congress=m.group(2), bill_type=bill_type, number=m.group(3)) return bill
def main(options): """ Process bill terms and bills """ # Terms term_processor = TermProcessor() terms_parsed = set() # Cache existing terms. There aren't so many. existing_terms = {} for term in BillTerm.objects.all(): existing_terms[(int(term.term_type), term.name)] = term log.info("Processing old bill terms") TERMS_FILE = "data/us/liv.xml" tree = etree.parse(TERMS_FILE) for node in tree.xpath("/liv/top-term"): term = term_processor.process(BillTerm(), node) term.term_type = TermType.old try: # No need to update an existing term because there are no other attributes. term = existing_terms[(int(term.term_type), term.name)] terms_parsed.add(term.id) except: log.debug("Created %s" % term) term.save() term.subterms.clear() for subnode in node.xpath("./term"): subterm = term_processor.process(BillTerm(), subnode) subterm.term_type = TermType.old try: # No need to update an existing term because there are no other attributes. subterm = existing_terms[(int(subterm.term_type), subterm.name)] term.subterms.add(subterm) terms_parsed.add(subterm.id) except: try: log.debug("Created %s" % subterm) subterm.save() term.subterms.add(subterm) existing_terms[(int(subterm.term_type), subterm.name)] = subterm terms_parsed.add(subterm.id) except IntegrityError: log.error("Duplicated term %s" % term_processor.display_node(subnode)) log.info("Processing new bill terms") for FILE in ("data/us/liv111.xml", "data/us/crsnet.xml"): tree = etree.parse(FILE) for node in tree.xpath("/liv/top-term"): term = term_processor.process(BillTerm(), node) term.term_type = TermType.new try: # No need to update an existing term because there are no other attributes. term = existing_terms[(int(term.term_type), term.name)] terms_parsed.add(term.id) except: log.debug("Created %s" % term) term.save() term.subterms.clear() for subnode in node.xpath("./term"): subterm = term_processor.process(BillTerm(), subnode) subterm.term_type = TermType.new try: # No need to update an existing term because there are no other attributes. subterm = existing_terms[(int(subterm.term_type), subterm.name)] terms_parsed.add(subterm.id) term.subterms.add(subterm) except: try: log.debug("Created %s" % term) subterm.save() term.subterms.add(subterm) existing_terms[(int(subterm.term_type), subterm.name)] = subterm terms_parsed.add(subterm.id) except IntegrityError: log.error("Duplicated term %s" % term_processor.display_node(subnode)) for term in existing_terms.values(): if not term.id in terms_parsed: log.debug("Deleted %s" % term) term.delete() # Bills bill_index = None if not options.disable_indexing: from bill.search_indexes import BillIndex bill_index = BillIndex() if options.congress and int(options.congress) <= 42: files = glob.glob("data/congress/%s/bills/*/*/*.xml" % options.congress) log.info("Parsing unitedstates/congress bills of only congress#%s" % options.congress) elif options.congress: files = glob.glob("data/us/%s/bills/*.xml" % options.congress) log.info("Parsing bills of only congress#%s" % options.congress) else: files = glob.glob("data/us/*/bills/*.xml") if options.filter: files = [f for f in files if re.match(options.filter, f)] log.info("Processing bills: %d files" % len(files)) total = len(files) progress = Progress(total=total, name="files", step=100) bill_processor = BillProcessor() seen_bill_ids = [] for fname in files: progress.tick() # With indexing or events enabled, if the bill metadata file hasn't changed check # the bill's latest text file for changes so we can create a text-is-available # event and so we can index the bill's text. if ( (not options.congress or options.congress > 42) and (bill_index and not options.disable_events) and not File.objects.is_changed(fname) and not options.force ): m = re.search(r"/(\d+)/bills/([a-z]+)(\d+)\.xml$", fname) try: b = Bill.objects.get(congress=m.group(1), bill_type=BillType.by_xml_code(m.group(2)), number=m.group(3)) seen_bill_ids.append(b.id) # Update the index/events for any bill with recently changed text textfile = get_bill_text_metadata(b, None) if not textfile: if b.congress >= 103 and b.introduced_date < (datetime.now() - timedelta(days=14)).date(): print "No bill text?", fname, b.introduced_date continue textfile = textfile["text_file"] if os.path.exists(textfile) and File.objects.is_changed(textfile): bill_index.update_object(b, using="bill") # index the full text b.create_events() # events for new bill text documents File.objects.save_file(textfile) continue except Bill.DoesNotExist: print "Unchanged metadata file but bill doesn't exist:", fname pass # just parse as normal if options.slow: time.sleep(1) tree = etree.parse(fname) for node in tree.xpath("/bill"): try: bill = bill_processor.process(Bill(), node) except: print fname raise seen_bill_ids.append(bill.id) # don't delete me later if bill.congress >= 93: bill.source = "thomas-congproj" elif bill.congress >= 82: bill.source = "statutesatlarge" if bill.current_status == BillStatus.enacted_signed: bill.current_status = BillStatus.enacted_unknown elif bill.congress <= 42: bill.source = "americanmemory" else: raise ValueError() # So far this is just for American Memory bills. if node.xpath("string(source/@url)"): bill.source_link = unicode(node.xpath("string(source/@url)")) else: bill.source_link = None actions = [] for axn in tree.xpath("actions/*[@state]"): actions.append( ( repr(bill_processor.parse_datetime(axn.xpath("string(@datetime)"))), BillStatus.by_xml_code(axn.xpath("string(@state)")), axn.xpath("string(text)"), etree.tostring(axn), ) ) bill.sliplawpubpriv = None bill.sliplawnum = None for axn in tree.xpath("actions/enacted"): bill.sliplawpubpriv = "PUB" if axn.get("type") == "public" else "PRI" bill.sliplawnum = int(axn.get("number").split("-")[1]) bill.major_actions = actions try: bill.save() except: print bill raise if bill_index: bill_index.update_object(bill, using="bill") if not options.disable_events: bill.create_events() File.objects.save_file(fname) # delete bill objects that are no longer represented on disk.... this is too dangerous. if options.congress and not options.filter: # this doesn't work because seen_bill_ids is too big for sqlite! for b in Bill.objects.filter(congress=options.congress).exclude(id__in=seen_bill_ids): print "Bill is no longer on disk: ", b.id, b # The rest is for current only... if options.congress and int(options.congress) != CURRENT_CONGRESS: return # Parse docs.house.gov for what might be coming up this week. import iso8601 dhg_html = urllib.urlopen("http://docs.house.gov/floor/").read() m = re.search(r"class=\"downloadXML\" href=\"(Download.aspx\?file=.*?)\"", dhg_html) if not m: log.error("No docs.house.gov download link found at http://docs.house.gov.") else: def bt_re(bt): return re.escape(bt[1]).replace(r"\.", r"\.?\s*") try: dhg = etree.parse(urllib.urlopen("http://docs.house.gov/floor/" + m.group(1))).getroot() except: print "http://docs.house.gov/floor/" + m.group(1) raise # iso8601.parse_date(dhg.get("week-date")+"T00:00:00").date() for item in dhg.xpath("category/floor-items/floor-item"): billname = item.xpath("legis-num")[0].text if billname is None: continue # weird but OK m = re.match( r"\s*(?:Concur in the Senate Amendment to |Senate Amendment to )?(" + "|".join(bt_re(bt) for bt in BillType) + r")(\d+)\s*(\[Conference Report\]\s*)?$", billname, re.I, ) if not m: if not billname.strip().endswith(" __"): log.error('Could not parse legis-num "%s" in docs.house.gov.' % billname) else: for bt in BillType: if re.match(bt_re(bt) + "$", m.group(1), re.I): try: bill = Bill.objects.get(congress=CURRENT_CONGRESS, bill_type=bt[0], number=m.group(2)) bill.docs_house_gov_postdate = iso8601.parse_date(item.get("add-date")).replace(tzinfo=None) bill.save() if bill_index: bill_index.update_object(bill, using="bill") if not options.disable_events: bill.create_events() except Bill.DoesNotExist: log.error('Could not find bill "%s" in docs.house.gov.' % billname) break else: log.error('Could not parse legis-num bill type "%s" in docs.house.gov.' % m.group(1)) # Parse Senate.gov's "Floor Schedule" blurb for coming up tomorrow. now = datetime.now() sfs = urllib.urlopen( "http://www.senate.gov/pagelayout/legislative/d_three_sections_with_teasers/calendars.htm" ).read() try: sfs = re.search(r"Floor Schedule([\w\W]*)Previous Meeting", sfs).group(1) for congress, bill_type, number in re.findall( r"http://hdl.loc.gov/loc.uscongress/legislation.(\d+)([a-z]+)(\d+)", sfs ): bill_type = BillType.by_slug(bill_type) bill = Bill.objects.get(congress=congress, bill_type=bill_type, number=number) if bill.senate_floor_schedule_postdate == None or now - bill.senate_floor_schedule_postdate > timedelta( days=7 ): bill.senate_floor_schedule_postdate = now bill.save() if bill_index: bill_index.update_object(bill, using="bill") if not options.disable_events: bill.create_events() except Exception as e: log.error("Could not parse Senate Floor Schedule: " + repr(e))
def main(options): """ Parse rolls. """ # Setup XML processors vote_processor = VoteProcessor() option_processor = VoteOptionProcessor() voter_processor = VoterProcessor() voter_processor.PERSON_CACHE = dict((x.pk, x) for x in Person.objects.all()) chamber_mapping = {'s': CongressChamber.senate, 'h': CongressChamber.house} if options.filter: files = glob.glob(options.filter) log.info('Parsing rolls matching %s' % options.filter) elif options.congress: files = glob.glob(settings.CONGRESS_DATA_PATH + '/%s/votes/*/*/data.xml' % options.congress) log.info('Parsing rolls of only congress#%s' % options.congress) else: files = glob.glob('data/congress/*/votes/*/*/data.xml') log.info('Processing votes: %d files' % len(files)) total = len(files) progress = Progress(total=total, name='files', step=10) def log_delete_qs(qs): if qs.count() == 0: return print("Deleting obsoleted records: ", qs) #if qs.count() > 3: # print "Delete skipped..." # return qs.delete() seen_obj_ids = set() had_error = False for fname in files: progress.tick() match = re.match(r"data/congress/(?P<congress>\d+)/votes/(?P<session>[ABC0-9]+)/(?P<chamber>[hs])(?P<number>\d+)/data.xml$", fname) try: existing_vote = Vote.objects.get(congress=int(match.group("congress")), chamber=chamber_mapping[match.group("chamber")], session=match.group("session"), number=int(match.group("number"))) except Vote.DoesNotExist: existing_vote = None if not File.objects.is_changed(fname) and not options.force and existing_vote != None and not existing_vote.missing_data: seen_obj_ids.add(existing_vote.id) continue try: tree = etree.parse(fname) ## Look for votes with VP tie breakers. #if len(tree.xpath("/roll/voter[@VP='1']")) == 0: # had_error = True # prevent delete at the end # continue # Process role object roll_node = tree.xpath('/roll')[0] # Sqlite is much faster when lots of saves are wrapped in a transaction, # and we do a lot of saves because it's a lot of voters. from django.db import transaction with transaction.atomic(): vote = vote_processor.process(Vote(), roll_node) if existing_vote: vote.id = existing_vote.id vote.congress = int(match.group("congress")) vote.chamber = chamber_mapping[match.group("chamber")] vote.session = match.group("session") vote.number = int(match.group("number")) # Get related bill & amendment. for bill_node in roll_node.xpath("bill"): related_bill_num = bill_node.get("number") if 9 <= vote.congress <= 42 and vote.session in ('1', '2'): # Bill numbering from the American Memory colletion is different. The number combines # the session, bill number, and a 0 or 5 for regular or 'half' numbering. Prior to # the 9th congress numbering seems to be wholly assigned by us and not related to # actual numbering, so we skip matching those bills. related_bill_num = "%d%04d%d" % (int(vote.session), int(bill_node.get("number")), 0) try: vote.related_bill = Bill.objects.get(congress=bill_node.get("session"), bill_type=BillType.by_xml_code(bill_node.get("type")), number=related_bill_num) except Bill.DoesNotExist: if vote.congress >= 93: vote.missing_data = True for amdt_node in roll_node.xpath("amendment"): if amdt_node.get("ref") == "regular" and vote.related_bill is not None: try: vote.related_amendment = Amendment.objects.get(congress=vote.related_bill.congress, amendment_type=AmendmentType.by_slug(amdt_node.get("number")[0]+"amdt"), number=amdt_node.get("number")[1:]) except Amendment.DoesNotExist: if vote.congress >= 93: print("Missing amendment", fname) vote.missing_data = True elif amdt_node.get("ref") == "bill-serial": # It is impossible to associate House votes with amendments just from the House # vote XML because the amendment-num might correspond either with the A___ number # or with the "An amendment, numbered ____" number from the amendment purpose, # and there's really no way to figure out which. Maybe we can use the amendment # sponsor instead? #vote.related_amendment = Amendment.objects.get(bill=vote.related_bill, sequence=amdt_node.get("number")) # Instead, we set related_amendment from the amendment parser. Here, we have to # preserve the related_amendment if it is set. if existing_vote: vote.related_amendment = existing_vote.related_amendment # clean up some question text and use the question_details field if vote.category in (VoteCategory.passage, VoteCategory.passage_suspension, VoteCategory.veto_override) and vote.related_bill: # For passage votes, set the question to the bill title and put the question # details in the details field. vote.question = vote.related_bill.title vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display() elif vote.category == VoteCategory.amendment and vote.related_amendment: # For votes on amendments, make a better title/explanation. vote.question = vote.related_amendment.title vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display() elif vote.related_bill and vote.question.startswith("On the Cloture Motion " + vote.related_bill.display_number): vote.question = "Cloture on " + vote.related_bill.title elif vote.related_bill and vote.question.startswith("On Cloture on the Motion to Proceed " + vote.related_bill.display_number): vote.question = "Cloture on " + vote.related_bill.title vote.question_details = "On Cloture on the Motion to Proceed in the " + vote.get_chamber_display() elif vote.related_bill and vote.question.startswith("On the Motion to Proceed " + vote.related_bill.display_number): vote.question = "Motion to Proceed on " + vote.related_bill.title elif vote.related_amendment and vote.question.startswith("On the Cloture Motion " + vote.related_amendment.get_amendment_type_display() + " " + str(vote.related_amendment.number)): vote.question = "Cloture on " + vote.related_amendment.title vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display() # weird House foratting of bill numbers ("H RES 123 Blah blah") if vote.related_bill: vote.question = re.sub( "(On [^:]+): " + vote.related_bill.display_number.replace(". ", " ").replace(".", " ").upper() + " .*", r"\1: " + truncatewords(vote.related_bill.title, 15), vote.question) vote.save() seen_obj_ids.add(vote.id) # don't delete me later # Process roll options, overwrite existing options where possible. seen_option_ids = set() roll_options = {} for option_node in roll_node.xpath('./option'): option = option_processor.process(VoteOption(), option_node) option.vote = vote if existing_vote: try: option.id = VoteOption.objects.filter(vote=vote, key=option.key)[0].id # get is better, but I had the database corruption problem except IndexError: pass option.save() roll_options[option.key] = option seen_option_ids.add(option.id) log_delete_qs(VoteOption.objects.filter(vote=vote).exclude(id__in=seen_option_ids)) # may cascade and delete the Voters too? # Process roll voters, overwriting existing voters where possible. if existing_vote: existing_voters = dict(Voter.objects.filter(vote=vote).values_list("person", "id")) seen_voter_ids = set() voters = list() for voter_node in roll_node.xpath('./voter'): voter = voter_processor.process(roll_options, Voter(), voter_node) voter.vote = vote voter.created = vote.created # for VP votes, load the actual person & role... if voter.voter_type == VoterType.vice_president: try: r = PersonRole.objects.get(role_type=RoleType.vicepresident, startdate__lte=vote.created, enddate__gte=vote.created) voter.person_role = r voter.person = r.person except PersonRole.DoesNotExist: # overlapping roles? missing data? log.error('Could not resolve vice president in %s' % fname) if existing_vote and voter.person: try: voter.id = existing_voters[voter.person.id] except KeyError: pass voters.append(voter) if voter.voter_type == VoterType.unknown and not vote.missing_data: vote.missing_data = True vote.save() # pre-fetch the role of each voter load_roles_at_date([x.person for x in voters if x.person != None], vote.created, vote.congress) for voter in list(voters): if voter.voter_type != VoterType.vice_president: voter.person_role = voter.person.role # If we couldn't match a role for this person on the date of the vote, and if the voter was Not Voting, # and we're looking at historical data, then this is probably a data error --- the voter wasn't even in office. # At the start of each Congress, the House does a Call by States and Election of the Speaker, before swearing # in. In the 116th Congress, these votes had a Not Voting for Walter Jones who had not yet made it to DC, and # then omitted Jones in the votes after swearing in. In those cases, look for a role coming up. if voter.person_role is None and voter.option.key == "0" and vote.question in ("Call by States", "Election of the Speaker"): voter.person_role = voter.person.roles.filter(startdate__gt=vote.created, startdate__lt=vote.created+timedelta(days=30)).first() if voter.person_role is None: if vote.source == VoteSource.keithpoole and voter.option.key == "0": # Drop this record. voters.remove(voter) else: log.error("%s: Could not find role for %s on %s." % (fname, voter.person, vote.created)) vote.missing_data = True vote.save() # save all of the records (inserting/updating) for voter in voters: voter.save() seen_voter_ids.add(voter.id) # remove obsolete voter records log_delete_qs(Voter.objects.filter(vote=vote).exclude(id__in=seen_voter_ids)) # possibly already deleted by cascade above # pre-calculate totals vote.calculate_totals() if not options.disable_events: vote.create_event() File.objects.save_file(fname) except Exception as ex: log.error('Error in processing %s' % fname, exc_info=ex) had_error = True # delete vote objects that are no longer represented on disk if options.congress and not options.filter and not had_error: log_delete_qs(Vote.objects.filter(congress=options.congress).exclude(id__in = seen_obj_ids))
def main(options): """ Process bill terms and bills """ # Terms term_processor = TermProcessor() terms_parsed = set() # Cache existing terms. There aren't so many. existing_terms = { } for term in BillTerm.objects.all(): existing_terms[(int(term.term_type), term.name)] = term log.info('Processing old bill terms') TERMS_FILE = 'data/us/liv.xml' tree = etree.parse(TERMS_FILE) for node in tree.xpath('/liv/top-term'): term = term_processor.process(BillTerm(), node) term.term_type = TermType.old try: # No need to update an existing term because there are no other attributes. term = existing_terms[(int(term.term_type), term.name)] terms_parsed.add(term.id) except: log.debug("Created %s" % term) term.save() term.subterms.clear() for subnode in node.xpath('./term'): subterm = term_processor.process(BillTerm(), subnode) subterm.term_type = TermType.old try: # No need to update an existing term because there are no other attributes. subterm = existing_terms[(int(subterm.term_type), subterm.name)] term.subterms.add(subterm) terms_parsed.add(subterm.id) except: try: log.debug("Created %s" % subterm) subterm.save() term.subterms.add(subterm) existing_terms[(int(subterm.term_type), subterm.name)] = subterm terms_parsed.add(subterm.id) except IntegrityError: log.error('Duplicated term %s' % term_processor.display_node(subnode)) log.info('Processing new bill terms') for FILE in ('data/us/liv111.xml', 'data/us/crsnet.xml'): tree = etree.parse(FILE) for node in tree.xpath('/liv/top-term'): term = term_processor.process(BillTerm(), node) term.term_type = TermType.new try: # No need to update an existing term because there are no other attributes. term = existing_terms[(int(term.term_type), term.name)] terms_parsed.add(term.id) except: log.debug("Created %s" % term) term.save() term.subterms.clear() for subnode in node.xpath('./term'): subterm = term_processor.process(BillTerm(), subnode) subterm.term_type = TermType.new try: # No need to update an existing term because there are no other attributes. subterm = existing_terms[(int(subterm.term_type), subterm.name)] terms_parsed.add(subterm.id) term.subterms.add(subterm) except: try: log.debug("Created %s" % term) subterm.save() term.subterms.add(subterm) existing_terms[(int(subterm.term_type), subterm.name)] = subterm terms_parsed.add(subterm.id) except IntegrityError: log.error('Duplicated term %s' % term_processor.display_node(subnode)) for term in existing_terms.values(): if not term.id in terms_parsed: log.debug("Deleted %s" % term) term.delete() # Bills bill_index = None if not options.disable_indexing: from bill.search_indexes import BillIndex bill_index = BillIndex() if options.congress: files = glob.glob('data/us/%s/bills/*.xml' % options.congress) log.info('Parsing bills of only congress#%s' % options.congress) else: files = glob.glob('data/us/*/bills/*.xml') if options.filter: files = [f for f in files if re.match(options.filter, f)] log.info('Processing bills: %d files' % len(files)) total = len(files) progress = Progress(total=total, name='files', step=100) bill_processor = BillProcessor() seen_bill_ids = [] for fname in files: progress.tick() if not File.objects.is_changed(fname) and not options.force: m = re.search(r"/(\d+)/bills/([a-z]+)(\d+)\.xml$", fname) try: b = Bill.objects.get(congress=m.group(1), bill_type=BillType.by_xml_code(m.group(2)), number=m.group(3)) seen_bill_ids.append(b.id) # Update the index/events for any bill with recently changed text textfile = "data/us/bills.text/%s/%s/%s%s.txt" % (m.group(1), m.group(2), m.group(2), m.group(3)) if (bill_index and not options.disable_events) and os.path.exists(textfile) and File.objects.is_changed(textfile): bill_index.update_object(b, using="bill") # index the full text b.create_events() # events for new bill text documents File.objects.save_file(textfile) continue except Bill.DoesNotExist: pass # just parse as normal if options.slow: time.sleep(1) skip_stuff = False tree = etree.parse(fname) for node in tree.xpath('/bill'): if not skip_stuff: try: bill = bill_processor.process(Bill(), node) except: print fname raise else: m = re.search(r"/(\d+)/bills/([a-z]+)(\d+)\.xml$", fname) bill = Bill.objects.get(congress=m.group(1), bill_type=BillType.by_xml_code(m.group(2)), number=m.group(3)) seen_bill_ids.append(bill.id) # don't delete me later actions = [] bill.sliplawpubpriv = None bill.sliplawnum = None for axn in tree.xpath("actions/*[@state]"): actions.append( (repr(bill_processor.parse_datetime(axn.xpath("string(@datetime)"))), BillStatus.by_xml_code(axn.xpath("string(@state)")), axn.xpath("string(text)")) ) if actions[-1][1] in (BillStatus.enacted_signed, BillStatus.enacted_veto_override): bill.sliplawpubpriv = "PUB" if axn.get("type") == "public" else "PRI" bill.sliplawnum = int(axn.get("number").split("-")[1]) bill.major_actions = actions try: bill.save() except: print bill raise if bill_index: bill_index.update_object(bill, using="bill") if not options.disable_events: bill.create_events() if not skip_stuff: File.objects.save_file(fname) # delete bill objects that are no longer represented on disk.... this is too dangerous. if options.congress and not options.filter and False: # this doesn't work because seen_bill_ids is too big for sqlite! Bill.objects.filter(congress=options.congress).exclude(id__in = seen_bill_ids).delete() # Parse docs.house.gov for what might be coming up this week. import iso8601 dhg_html = urllib.urlopen("http://docs.house.gov/floor/").read() m = re.search(r"class=\"downloadXML\" href=\"(Download.aspx\?file=.*?)\"", dhg_html) if not m: log.error('No docs.house.gov download link found at http://docs.house.gov.') else: def bt_re(bt): return re.escape(bt[1]).replace(r"\.", "\.?\s*") try: dhg = etree.parse(urllib.urlopen("http://docs.house.gov/floor/" + m.group(1))).getroot() except: print "http://docs.house.gov/" + m.group(1) raise # iso8601.parse_date(dhg.get("week-date")+"T00:00:00").date() for item in dhg.xpath("category/floor-items/floor-item"): billname = item.xpath("legis-num")[0].text m = re.match("\s*(?:Concur in the Senate Amendment to |Senate Amendment to )?(" + "|".join(bt_re(bt) for bt in BillType) + ")(\d+)\s*(\[Conference Report\]\s*)?$", billname, re.I) if not m: if billname.strip() != "H.R. __": log.error('Could not parse legis-num "%s" in docs.house.gov.' % billname) else: for bt in BillType: if re.match(bt_re(bt) + "$", m.group(1)): try: bill = Bill.objects.get(congress=CURRENT_CONGRESS, bill_type=bt[0], number=m.group(2)) bill.docs_house_gov_postdate = iso8601.parse_date(item.get("add-date")).replace(tzinfo=None) bill.save() if bill_index: bill_index.update_object(bill, using="bill") if not options.disable_events: bill.create_events() except Bill.DoesNotExist: log.error('Could not find bill "%s" in docs.house.gov.' % billname) break else: log.error('Could not parse legis-num bill type "%s" in docs.house.gov.' % billname) # Parse Senate.gov's "Floor Schedule" blurb for coming up tomorrow. now = datetime.now() sfs = urllib.urlopen("http://www.senate.gov/pagelayout/legislative/d_three_sections_with_teasers/calendars.htm").read() try: sfs = re.search(r"Floor Schedule([\w\W]*)Previous Meeting", sfs).group(1) for congress, bill_type, number in re.findall(r"http://hdl.loc.gov/loc.uscongress/legislation.(\d+)([a-z]+)(\d+)", sfs): bill_type = BillType.by_slug(bill_type) bill = Bill.objects.get(congress=congress, bill_type=bill_type, number=number) if bill.senate_floor_schedule_postdate == None or now - bill.senate_floor_schedule_postdate > timedelta(days=7): bill.senate_floor_schedule_postdate = now bill.save() if bill_index: bill_index.update_object(bill, using="bill") if not options.disable_events: bill.create_events() except Exception as e: log.error('Could not parse Senate Floor Schedule: ' + repr(e))
def main(options): """ Parse rolls. """ # Setup XML processors vote_processor = VoteProcessor() option_processor = VoteOptionProcessor() voter_processor = VoterProcessor() voter_processor.PERSON_CACHE = dict((x.pk, x) for x in Person.objects.all()) # The pattern which the roll file matches # Filename contains info which should be placed to DB # along with info extracted from the XML file re_path = re.compile("data/us/(\d+)/rolls/([hs])(\w+)-(\d+)\.xml") chamber_mapping = {"s": CongressChamber.senate, "h": CongressChamber.house} if options.filter: files = glob.glob(options.filter) log.info("Parsing rolls matching %s" % options.filter) elif options.congress: files = glob.glob("data/us/%s/rolls/*.xml" % options.congress) log.info("Parsing rolls of only congress#%s" % options.congress) else: files = glob.glob("data/us/*/rolls/*.xml") log.info("Processing votes: %d files" % len(files)) total = len(files) progress = Progress(total=total, name="files", step=10) def log_delete_qs(qs): if qs.count() == 0: return print "Deleting obsoleted records: ", qs # if qs.count() > 3: # print "Delete skipped..." # return qs.delete() seen_obj_ids = set() had_error = False for fname in files: progress.tick() match = re_path.search(fname) try: existing_vote = Vote.objects.get( congress=match.group(1), chamber=chamber_mapping[match.group(2)], session=match.group(3), number=match.group(4), ) except Vote.DoesNotExist: existing_vote = None if ( not File.objects.is_changed(fname) and not options.force and existing_vote != None and not existing_vote.missing_data ): seen_obj_ids.add(existing_vote.id) continue try: tree = etree.parse(fname) ## Look for votes with VP tie breakers. # if len(tree.xpath("/roll/voter[@VP='1']")) == 0: # had_error = True # prevent delete at the end # continue # Process role object for roll_node in tree.xpath("/roll"): vote = vote_processor.process(Vote(), roll_node) if existing_vote: vote.id = existing_vote.id match = re_path.search(fname) vote.congress = int(match.group(1)) vote.chamber = chamber_mapping[match.group(2)] vote.session = match.group(3) vote.number = int(match.group(4)) # Get related bill & amendment. for bill_node in roll_node.xpath("bill"): related_bill_num = bill_node.get("number") if 9 <= vote.congress <= 42 and vote.session in ("1", "2"): # Bill numbering from the American Memory colletion is different. The number combines # the session, bill number, and a 0 or 5 for regular or 'half' numbering. Prior to # the 9th congress numbering seems to be wholly assigned by us and not related to # actual numbering, so we skip matching those bills. related_bill_num = "%d%04d%d" % (int(vote.session), int(bill_node.get("number")), 0) try: vote.related_bill = Bill.objects.get( congress=bill_node.get("session"), bill_type=BillType.by_xml_code(bill_node.get("type")), number=related_bill_num, ) except Bill.DoesNotExist: if vote.congress >= 93: vote.missing_data = True for amdt_node in roll_node.xpath("amendment"): if amdt_node.get("ref") == "regular" and vote.related_bill is not None: try: vote.related_amendment = Amendment.objects.get( congress=vote.related_bill.congress, amendment_type=AmendmentType.by_slug(amdt_node.get("number")[0]), number=amdt_node.get("number")[1:], ) except Amendment.DoesNotExist: if vote.congress >= 93: print "Missing amendment", fname vote.missing_data = True elif amdt_node.get("ref") == "bill-serial": # It is impossible to associate House votes with amendments just from the House # vote XML because the amendment-num might correspond either with the A___ number # or with the "An amendment, numbered ____" number from the amendment purpose, # and there's really no way to figure out which. Maybe we can use the amendment # sponsor instead? # vote.related_amendment = Amendment.objects.get(bill=vote.related_bill, sequence=amdt_node.get("number")) # Instead, we set related_amendment from the amendment parser. Here, we have to # preserve the related_amendment if it is set. if existing_vote: vote.related_amendment = existing_vote.related_amendment # clean up some question text and use the question_details field if ( vote.category in (VoteCategory.passage, VoteCategory.passage_suspension, VoteCategory.veto_override) and vote.related_bill ): # For passage votes, set the question to the bill title and put the question # details in the details field. vote.question = truncatewords(vote.related_bill.title, 20) vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display() elif vote.category == VoteCategory.amendment and vote.related_amendment: # For votes on amendments, make a better title/explanation. vote.question = truncatewords(vote.related_amendment.title, 20) vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display() elif vote.related_bill and vote.question.startswith( "On the Cloture Motion " + vote.related_bill.display_number ): vote.question = "Cloture on " + truncatewords(vote.related_bill.title, 20) elif vote.related_bill and vote.question.startswith( "On Cloture on the Motion to Proceed " + vote.related_bill.display_number ): vote.question = "Cloture on " + truncatewords(vote.related_bill.title, 20) vote.question_details = "On Cloture on the Motion to Proceed in the " + vote.get_chamber_display() elif vote.related_bill and vote.question.startswith( "On the Motion to Proceed " + vote.related_bill.display_number ): vote.question = "Motion to Proceed on " + truncatewords(vote.related_bill.title, 20) elif vote.related_amendment and vote.question.startswith( "On the Cloture Motion " + vote.related_amendment.get_amendment_type_display() + " " + str(vote.related_amendment.number) ): vote.question = "Cloture on " + truncatewords(vote.related_amendment.title, 20) vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display() # weird House foratting of bill numbers ("H RES 123 Blah blah") if vote.related_bill: vote.question = re.sub( "(On [^:]+): " + vote.related_bill.display_number.replace(". ", " ").replace(".", " ").upper() + " .*", r"\1: " + truncatewords(vote.related_bill.title, 15), vote.question, ) vote.save() seen_obj_ids.add(vote.id) # don't delete me later # Process roll options, overwrite existing options where possible. seen_option_ids = set() roll_options = {} for option_node in roll_node.xpath("./option"): option = option_processor.process(VoteOption(), option_node) option.vote = vote if existing_vote: try: option.id = VoteOption.objects.filter(vote=vote, key=option.key)[ 0 ].id # get is better, but I had the database corruption problem except IndexError: pass option.save() roll_options[option.key] = option seen_option_ids.add(option.id) log_delete_qs( VoteOption.objects.filter(vote=vote).exclude(id__in=seen_option_ids) ) # may cascade and delete the Voters too? # Process roll voters, overwriting existing voters where possible. if existing_vote: existing_voters = dict(Voter.objects.filter(vote=vote).values_list("person", "id")) seen_voter_ids = set() voters = list() for voter_node in roll_node.xpath("./voter"): voter = voter_processor.process(roll_options, Voter(), voter_node) voter.vote = vote voter.created = vote.created # for VP votes, load the actual person... if voter.voter_type == VoterType.vice_president: try: r = PersonRole.objects.get( role_type=RoleType.vicepresident, startdate__lte=vote.created, enddate__gte=vote.created ) voter.person_role = r voter.person = r.person except: # overlapping roles? missing data? log.error("Could not resolve vice president in %s" % fname, exc_info=ex) if existing_vote and voter.person: try: voter.id = existing_voters[voter.person.id] except KeyError: pass voters.append(voter) if voter.voter_type == VoterType.unknown and not vote.missing_data: vote.missing_data = True vote.save() # pre-fetch the role of each voter load_roles_at_date([x.person for x in voters if x.person != None], vote.created) for voter in voters: voter.person_role = voter.person.role if voter.person_role is None: log.error("%s: Could not find role for %s on %s." % (fname, voter.person, vote.created)) vote.missing_data = True vote.save() # save all of the records (inserting/updating) for voter in voters: voter.save() seen_voter_ids.add(voter.id) # remove obsolete voter records log_delete_qs( Voter.objects.filter(vote=vote).exclude(id__in=seen_voter_ids) ) # possibly already deleted by cascade above # pre-calculate totals vote.calculate_totals() if not options.disable_events: vote.create_event() File.objects.save_file(fname) except Exception, ex: log.error("Error in processing %s" % fname, exc_info=ex) had_error = True
def main(options): """ Process committees, subcommittees and members of current congress committees. """ BASE_PATH = settings.CONGRESS_LEGISLATORS_PATH meeting_processor = CommitteeMeetingProcessor() log.info('Processing committees') COMMITTEES_FILE = BASE_PATH + 'committees-current.yaml' if not File.objects.is_changed(COMMITTEES_FILE) and not options.force: log.info('File %s was not changed' % COMMITTEES_FILE) else: tree = yaml_load(COMMITTEES_FILE) total = len(tree) progress = Progress(total=total) seen_committees = set() for committee in tree: try: cobj = Committee.objects.get(code=committee["thomas_id"]) except Committee.DoesNotExist: print "New committee:", committee["thomas_id"] cobj = Committee(code=committee["thomas_id"]) cobj.committee_type = TYPE_MAPPING[committee["type"]] cobj.name = committee["name"] cobj.url = committee.get("url", None) cobj.obsolete = False cobj.committee = None cobj.save() seen_committees.add(cobj.id) for subcom in committee.get('subcommittees', []): code = committee["thomas_id"] + subcom["thomas_id"] try: sobj = Committee.objects.get(code=code) except Committee.DoesNotExist: print "New subcommittee:", code sobj = Committee(code=code) sobj.name = subcom["name"] sobj.url = subcom.get("url", None) sobj.type = None sobj.committee = cobj sobj.obsolete = False sobj.save() seen_committees.add(sobj.id) progress.tick() # Check for non-obsolete committees in the database that aren't in our # file. other_committees = Committee.objects.filter(obsolete=False).exclude(id__in=seen_committees) if len(other_committees) > 0: print "Marking obsolete:", ", ".join(c.code for c in other_committees) other_committees.update(obsolete=True) File.objects.save_file(COMMITTEES_FILE) log.info('Processing committee members') MEMBERS_FILE = BASE_PATH + 'committee-membership-current.yaml' file_changed = File.objects.is_changed(MEMBERS_FILE) if not file_changed and not options.force: log.info('File %s was not changed' % MEMBERS_FILE) else: # map THOMAS IDs to GovTrack IDs y = yaml_load(BASE_PATH + "legislators-current.yaml") person_id_map = { } for m in y: if "id" in m and "govtrack" in m["id"] and "thomas" in m["id"]: person_id_map[m["id"]["thomas"]] = m["id"]["govtrack"] # load committee members tree = yaml_load(MEMBERS_FILE) total = len(tree) progress = Progress(total=total, name='committees') # We can delete CommitteeMember objects because we don't have # any foreign keys to them. CommitteeMember.objects.all().delete() # Process committee nodes for committee, members in tree.items(): if committee[0] == "H": continue # House data is out of date try: cobj = Committee.objects.get(code=committee) except Committee.DoesNotExist: print "Committee not found:", committee continue # Process members of current committee node for member in members: mobj = CommitteeMember() mobj.person = Person.objects.get(id=person_id_map[member["thomas"]]) mobj.committee = cobj if "title" in member: mobj.role = ROLE_MAPPING[member["title"]] mobj.save() progress.tick() File.objects.save_file(MEMBERS_FILE) return log.info('Processing committee schedule') SCHEDULE_FILE = 'data/us/112/committeeschedule.xml' file_changed = File.objects.is_changed(SCHEDULE_FILE) if not file_changed and not options.force: log.info('File %s was not changed' % SCHEDULE_FILE) else: tree = etree.parse(SCHEDULE_FILE) # We have to clear out all CommitteeMeeting objects when we refresh because # we have no unique identifier in the upstream data for a meeting. We might use # the meeting's committee & date as an identifier, but since meeting times can # change this might have awkward consequences for the end user if we even # attempted to track that. CommitteeMeeting.objects.all().delete() # Process committee event nodes for meeting in tree.xpath('/committee-schedule/meeting'): try: mobj = meeting_processor.process(CommitteeMeeting(), meeting) mobj.save() mobj.bills.clear() for bill in meeting.xpath('bill'): bill = Bill.objects.get(congress=bill.get("session"), bill_type=BillType.by_xml_code(bill.get("type")), number=int(bill.get("number"))) mobj.bills.add(bill) except Committee.DoesNotExist: log.error('Could not load Committee object for meeting %s' % meeting_processor.display_node(meeting)) for committee in Committee.objects.all(): if not options.disable_events: committee.create_events() File.objects.save_file(SCHEDULE_FILE)
def main(options): """ Parse rolls. """ # Setup XML processors vote_processor = VoteProcessor() option_processor = VoteOptionProcessor() voter_processor = VoterProcessor() voter_processor.PERSON_CACHE = dict( (x.pk, x) for x in Person.objects.all()) # The pattern which the roll file matches # Filename contains info which should be placed to DB # along with info extracted from the XML file re_path = re.compile('data/us/(\d+)/rolls/([hs])(\w+)-(\d+)\.xml') chamber_mapping = {'s': CongressChamber.senate, 'h': CongressChamber.house} if options.filter: files = glob.glob(options.filter) log.info('Parsing rolls matching %s' % options.filter) elif options.congress: files = glob.glob('data/us/%s/rolls/*.xml' % options.congress) log.info('Parsing rolls of only congress#%s' % options.congress) else: files = glob.glob('data/us/*/rolls/*.xml') log.info('Processing votes: %d files' % len(files)) total = len(files) progress = Progress(total=total, name='files', step=10) def log_delete_qs(qs): if qs.count() > 0: try: print "Deleting: ", qs except Exception as e: print "Deleting [%s]..." % str(e) if qs.count() > 3: print "Delete skipped..." return qs.delete() seen_obj_ids = set() had_error = False for fname in files: progress.tick() match = re_path.search(fname) try: existing_vote = Vote.objects.get( congress=match.group(1), chamber=chamber_mapping[match.group(2)], session=match.group(3), number=match.group(4)) except Vote.DoesNotExist: existing_vote = None if not File.objects.is_changed( fname ) and not options.force and existing_vote != None and not existing_vote.missing_data: seen_obj_ids.add(existing_vote.id) continue try: tree = etree.parse(fname) ## Look for votes with VP tie breakers. #if len(tree.xpath("/roll/voter[@VP='1']")) == 0: # had_error = True # prevent delete at the end # continue # Process role object for roll_node in tree.xpath('/roll'): vote = vote_processor.process(Vote(), roll_node) if existing_vote: vote.id = existing_vote.id match = re_path.search(fname) vote.congress = int(match.group(1)) vote.chamber = chamber_mapping[match.group(2)] vote.session = match.group(3) vote.number = int(match.group(4)) # Get related bill & amendment. for bill_node in roll_node.xpath("bill"): try: vote.related_bill = Bill.objects.get( congress=bill_node.get("session"), bill_type=BillType.by_xml_code( bill_node.get("type")), number=bill_node.get("number")) except Bill.DoesNotExist: vote.missing_data = True for amdt_node in roll_node.xpath("amendment"): if amdt_node.get("ref") == "regular": try: vote.related_amendment = Amendment.objects.get( congress=vote.related_bill.congress, amendment_type=AmendmentType.by_slug( amdt_node.get("number")[0]), number=amdt_node.get("number")[1:]) except Amendment.DoesNotExist: print "Missing amendment", fname vote.missing_data = True elif amdt_node.get("ref") == "bill-serial": # It is impossible to associate House votes with amendments just from the House # vote XML because the amendment-num might correspond either with the A___ number # or with the "An amendment, numbered ____" number from the amendment purpose, # and there's really no way to figure out which. Maybe we can use the amendment # sponsor instead? #vote.related_amendment = Amendment.objects.get(bill=vote.related_bill, sequence=amdt_node.get("number")) # Instead, we set related_amendment from the amendment parser. Here, we have to # preserve the related_amendment if it is set. if existing_vote: vote.related_amendment = existing_vote.related_amendment # clean up some question text and use the question_details field if vote.category in ( VoteCategory.passage, VoteCategory.passage_suspension, VoteCategory.veto_override) and vote.related_bill: # For passage votes, set the question to the bill title and put the question # details in the details field. vote.question = truncatewords(vote.related_bill.title, 20) vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display( ) elif vote.category == VoteCategory.amendment and vote.related_amendment: # For votes on amendments, make a better title/explanation. vote.question = truncatewords(vote.related_amendment.title, 20) vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display( ) elif vote.related_bill and vote.question.startswith( "On the Cloture Motion " + vote.related_bill.display_number): vote.question = "Cloture on " + truncatewords( vote.related_bill.title, 20) elif vote.related_bill and vote.question.startswith( "On Cloture on the Motion to Proceed " + vote.related_bill.display_number): vote.question = "Cloture on " + truncatewords( vote.related_bill.title, 20) vote.question_details = "On Cloture on the Motion to Proceed in the " + vote.get_chamber_display( ) elif vote.related_bill and vote.question.startswith( "On the Motion to Proceed " + vote.related_bill.display_number): vote.question = "Motion to Proceed on " + truncatewords( vote.related_bill.title, 20) elif vote.related_amendment and vote.question.startswith( "On the Cloture Motion " + vote.related_amendment.get_amendment_type_display() + " " + str(vote.related_amendment.number)): vote.question = "Cloture on " + truncatewords( vote.related_amendment.title, 20) vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display( ) # weird House foratting of bill numbers ("H RES 123 Blah blah") if vote.related_bill: vote.question = re.sub( "(On [^:]+): " + vote.related_bill.display_number.replace( ". ", " ").replace(".", " ").upper() + " .*", r"\1: " + truncatewords(vote.related_bill.title, 15), vote.question) vote.save() seen_obj_ids.add(vote.id) # don't delete me later # Process roll options, overwrite existing options where possible. seen_option_ids = set() roll_options = {} for option_node in roll_node.xpath('./option'): option = option_processor.process(VoteOption(), option_node) option.vote = vote if existing_vote: try: option.id = VoteOption.objects.filter( vote=vote, key=option.key )[0].id # get is better, but I had the database corruption problem except IndexError: pass option.save() roll_options[option.key] = option seen_option_ids.add(option.id) log_delete_qs( VoteOption.objects.filter(vote=vote).exclude( id__in=seen_option_ids) ) # may cascade and delete the Voters too? # Process roll voters, overwriting existing voters where possible. if existing_vote: existing_voters = dict( Voter.objects.filter(vote=vote).values_list( "person", "id")) seen_voter_ids = set() for voter_node in roll_node.xpath('./voter'): voter = voter_processor.process(roll_options, Voter(), voter_node) voter.vote = vote voter.created = vote.created # for VP votes, load the actual person... if voter.voter_type == VoterType.vice_president: try: r = PersonRole.objects.get( role_type=RoleType.vicepresident, startdate__lte=vote.created, enddate__gte=vote.created) voter.person = r.person except: # overlapping roles? missing data? log.error( 'Could not resolve vice president in %s' % fname, exc_info=ex) if existing_vote and voter.person: try: voter.id = existing_voters[voter.person.id] except KeyError: pass voter.save() if voter.voter_type == VoterType.unknown and not vote.missing_data: vote.missing_data = True vote.save() seen_voter_ids.add(voter.id) log_delete_qs( Voter.objects.filter(vote=vote).exclude( id__in=seen_voter_ids) ) # possibly already deleted by cascade above vote.calculate_totals() if not options.disable_events: vote.create_event() File.objects.save_file(fname) except Exception, ex: log.error('Error in processing %s' % fname, exc_info=ex) had_error = True