Esempio n. 1
0
def run(options):
    amendment_id = options.get('amendment_id', None)
    bill_id = options.get('bill_id', None)

    search_state = {}

    if amendment_id:
        amendment_type, number, congress = utils.split_bill_id(amendment_id)
        to_fetch = [amendment_id]

    elif bill_id:
        # first, crawl the bill
        bill_type, number, congress = utils.split_bill_id(bill_id)
        bill_status = fetch_bill(bill_id, options)
        if bill_status['ok']:
            bill = json.loads(utils.read(output_for_bill(bill_id, "json")))
            to_fetch = [x["amendment_id"] for x in bill["amendments"]]
        else:
            logging.error("Couldn't download information for that bill.")
            return None

    else:
        congress = options.get('congress', utils.current_congress())

        to_fetch = bill_ids_for(congress,
                                utils.merge(options, {'amendments': True}),
                                bill_states=search_state)
        if not to_fetch:
            if options.get("fast", False):
                logging.warn("No amendments changed.")
            else:
                logging.error(
                    "Error figuring out which amendments to download, aborting."
                )

            return None

        limit = options.get('limit', None)
        if limit:
            to_fetch = to_fetch[:int(limit)]

    if options.get('pages_only', False):
        return None

    logging.warn("Going to fetch %i amendments from congress #%s" %
                 (len(to_fetch), congress))
    saved_amendments = utils.process_set(to_fetch, fetch_amendment, options)

    # keep record of the last state of all these amendments, for later fast-searching
    save_bill_search_state(saved_amendments, search_state)
Esempio n. 2
0
def run(options):
    bill_id = options.get('bill_id', None)
    bill_version_id = options.get('bill_version_id', None)

    # using a specific bill or version overrides the congress flag/default
    if bill_id:
        bill_type, number, congress = utils.split_bill_id(bill_id)
    elif bill_version_id:
        bill_type, number, congress, version_code = utils.split_bill_version_id(bill_version_id)
    else:
        congress = options.get('congress', utils.current_congress())

    if bill_version_id:
        to_fetch = [bill_version_id]
    else:
        to_fetch = bill_version_ids_for(congress, options)
        if not to_fetch:
            logging.error("Error figuring out which bills to download, aborting.")
            return None

    limit = options.get('limit', None)
    if limit:
        to_fetch = to_fetch[:int(limit)]

    logging.warn("Going to fetch %i bill versions for congress #%s" % (len(to_fetch), congress))

    saved_versions = utils.process_set(to_fetch, fetch_version, options)
Esempio n. 3
0
def amendment_url_for(amendment_id):
    amendment_type, number, congress = utils.split_bill_id(amendment_id)
    thomas_type = utils.thomas_types[amendment_type][0]
    congress = int(congress)
    number = int(number)
    return "http://thomas.loc.gov/cgi-bin/bdquery/z?d%03d:%s%s:" % (
        congress, thomas_type, number)
Esempio n. 4
0
def output_for_bill(bill_id, format, is_data_dot=True):
    bill_type, number, congress = utils.split_bill_id(bill_id)
    if is_data_dot:
        fn = "data.%s" % format
    else:
        fn = format
    return "%s/%s/bills/%s/%s%s/%s" % (utils.data_dir(), congress, bill_type, bill_type, number, fn)
Esempio n. 5
0
def run(options):
    bill_id = options.get('bill_id', None)

    search_state = {}

    if bill_id:
        bill_type, number, congress = utils.split_bill_id(bill_id)
        to_fetch = [bill_id]
    else:
        congress = options.get('congress', utils.current_congress())
        to_fetch = bill_ids_for(congress, options, bill_states=search_state)

        if not to_fetch:
            if options.get("fast", False):
                logging.warn("No bills changed.")
            else:
                logging.error(
                    "Error figuring out which bills to download, aborting.")
            return None

        limit = options.get('limit', None)
        if limit:
            to_fetch = to_fetch[:int(limit)]

    logging.warn("Going to fetch %i bills from congress #%s" %
                 (len(to_fetch), congress))

    saved_bills = utils.process_set(to_fetch, bill_info.fetch_bill, options)

    save_bill_search_state(saved_bills, search_state)
Esempio n. 6
0
def run(options):
    bill_version_id = options.get("bill_version_id", None)

    if bill_version_id:
        bill_type, bill_number, congress, version_code = utils.split_bill_version_id(
            bill_version_id)
        bill_id = utils.build_bill_id(bill_type, bill_number, congress)
    else:
        version_code = None
        bill_id = options.get("bill_id", None)

        if bill_id:
            bill_type, bill_number, congress = utils.split_bill_id(bill_id)
        else:
            bill_type = bill_number = None
            congress = options.get("congress", utils.current_congress())

    force = options.get("force", False)

    to_fetch = bill_version_ids_for(congress, bill_type, bill_number,
                                    version_code, force)

    if not to_fetch:
        return None

    saved_versions = utils.process_set(to_fetch, write_bill_catoxml, options)
Esempio n. 7
0
def output_for_bill(bill_id, format, is_data_dot=True):
    bill_type, number, congress = utils.split_bill_id(bill_id)
    if is_data_dot:
        fn = "data.%s" % format
    else:
        fn = format
    return "%s/%s/bills/%s/%s%s/%s" % (utils.data_dir(), congress, bill_type, bill_type, number, fn)
Esempio n. 8
0
def run(options):
  amdt_id = options.get('amendment_id', None)
  
  search_state = { }

  if amdt_id:
    amdt_type, number, congress = utils.split_bill_id(amdt_id)
    to_fetch = [amdt_id]
  else:
    congress = options.get('congress', utils.current_congress())
    to_fetch = bill_ids_for(congress, utils.merge(options, {'amendments': True}), bill_states=search_state)
    if not to_fetch:
      if options.get("fast", False):
        logging.warn("No amendments changed.")
      else:
        logging.error("Error figuring out which amendments to download, aborting.")
      return None

    limit = options.get('limit', None)
    if limit:
      to_fetch = to_fetch[:int(limit)]

  if options.get('pages_only', False):
    return None

  logging.warn("Going to fetch %i amendments from congress #%s" % (len(to_fetch), congress))
  
  saved_amendments = utils.process_set(to_fetch, fetch_amendment, options)

  save_bill_search_state(saved_amendments, search_state)
Esempio n. 9
0
def run(options):
    bill_id = options.get('bill_id', None)

    search_state = {}

    if bill_id:
        bill_type, number, congress = utils.split_bill_id(bill_id)
        to_fetch = [bill_id]
    else:
        congress = options.get('congress', utils.current_congress())
        to_fetch = bill_ids_for(congress, options, bill_states=search_state)

        if not to_fetch:
            if options.get("fast", False):
                logging.warn("No bills changed.")
            else:
                logging.error("Error figuring out which bills to download, aborting.")
            return None

        limit = options.get('limit', None)
        if limit:
            to_fetch = to_fetch[:int(limit)]

    logging.warn("Going to fetch %i bills from congress #%s" % (len(to_fetch), congress))

    saved_bills = utils.process_set(to_fetch, bill_info.fetch_bill, options)

    save_bill_search_state(saved_bills, search_state)
Esempio n. 10
0
def run(options):
  bill_id = options.get('bill_id', None)
  bill_version_id = options.get('bill_version_id', None)

  # using a specific bill or version overrides the congress flag/default
  if bill_id:
    bill_type, number, congress = utils.split_bill_id(bill_id)
  elif bill_version_id:
    bill_type, number, congress, version_code = utils.split_bill_version_id(bill_version_id)
  else:
    congress = options.get('congress', utils.current_congress())

  if bill_version_id:
    to_fetch = [bill_version_id]
  else:
    to_fetch = bill_version_ids_for(congress, options)
    if not to_fetch:
      logging.error("Error figuring out which bills to download, aborting.")
      return None

  limit = options.get('limit', None)
  if limit:
    to_fetch = to_fetch[:int(limit)]

  logging.warn("Going to fetch %i bill versions for congress #%s" % (len(to_fetch), congress))
  
  saved_versions = utils.process_set(to_fetch, fetch_version, options)
Esempio n. 11
0
def run(options):
  amendment_id = options.get('amendment_id', None)
  bill_id = options.get('bill_id', None)
  
  search_state = { }

  if amendment_id:
    amendment_type, number, congress = utils.split_bill_id(amendment_id)
    to_fetch = [amendment_id]

  elif bill_id:
    # first, crawl the bill
    bill_type, number, congress = utils.split_bill_id(bill_id)
    bill_status = fetch_bill(bill_id, options)
    if bill_status['ok']:
      bill = json.loads(utils.read(output_for_bill(bill_id, "json")))
      to_fetch = [x["amendment_id"] for x in bill["amendments"]]
    else:
      logging.error("Couldn't download information for that bill.")
      return None

  else:
    congress = options.get('congress', utils.current_congress())

    to_fetch = bill_ids_for(congress, utils.merge(options, {'amendments': True}), bill_states=search_state)
    if not to_fetch:
      if options.get("fast", False):
        logging.warn("No amendments changed.")
      else:
        logging.error("Error figuring out which amendments to download, aborting.")

      return None

    limit = options.get('limit', None)
    if limit:
      to_fetch = to_fetch[:int(limit)]

  if options.get('pages_only', False):
    return None

  logging.warn("Going to fetch %i amendments from congress #%s" % (len(to_fetch), congress))
  saved_amendments = utils.process_set(to_fetch, fetch_amendment, options)

  # keep record of the last state of all these amendments, for later fast-searching
  save_bill_search_state(saved_amendments, search_state)
Esempio n. 12
0
def fetch_amendment(amdt_id, options):
  logging.info("\n[%s] Fetching..." % amdt_id)
  
  # fetch bill details body
  body = utils.download(
    amdt_url_for(amdt_id), 
    amdt_cache_for(amdt_id, "information.html"),
    options)

  if not body:
    return {'saved': False, 'ok': False, 'reason': "failed to download"}

  if options.get("download_only", False):
    return {'saved': False, 'ok': True, 'reason': "requested download only"}
    
  amdt_type, number, congress = utils.split_bill_id(amdt_id)
  
  actions = actions_for(body, amdt_id, is_amendment=True)
  if actions is None: actions = []
  parse_amendment_actions(actions)

  amdt = {
    'amendment_id': amdt_id,
    'amendment_type': amdt_type,
    'chamber': amdt_type[0],
    'number': number,
    'congress': congress,
    
    'amends': amends_for(body, grab_bill=False),
    'amends_bill': amends_for(body, grab_bill=True),
    'house_number': house_number_for(body),

    'offered_at': offered_at_for(body, 'offered'),
    'submitted_at': offered_at_for(body, 'submitted'),
    'proposed_at': offered_at_for(body, 'proposed'),
    'sponsor': sponsor_for(body),

    'title': amendment_simple_text_for(body, "title"),
    'description': amendment_simple_text_for(body, "description"),
    'purpose': amendment_simple_text_for(body, "purpose"),
    
    'actions': actions,

    'updated_at': datetime.datetime.fromtimestamp(time.time()),
  }
  
  set_amendment_status(amdt)
  
  output_amendment(amdt, options)

  return {'ok': True, 'saved': True}
Esempio n. 13
0
def run(options):
  bill_id = options.get('bill_id', None)

  if bill_id:
    bill_type, number, congress = utils.split_bill_id(bill_id)
    to_fetch = [bill_id]
  else:
    congress = options.get('congress', utils.current_congress())
    to_fetch = bill_ids_for(congress, options)
    if not to_fetch:
      logging.error("Error figuring out which bills to download, aborting.")
      return None

    limit = options.get('limit', None)
    if limit:
      to_fetch = to_fetch[:int(limit)]

  if options.get('pages_only', False):
    return None

  print "Going to fetch %i bills from congress #%s" % (len(to_fetch), congress)
  
  errors = []
  saved = []
  skips = []

  for bill_id in to_fetch:
    try:
      results = bill_info.fetch_bill(bill_id, options)
    except Exception, e:
      if options.get('raise', False):
        raise
      else:
        errors.append((bill_id, e))
        continue

    if results.get('ok', False):
      if results.get('saved', False):
        saved.append(bill_id)
        logging.info("[%s] Updated bill" % bill_id)
      else:
        skips.append(bill_id)
        logging.error("[%s] Skipping bill: %s" % (bill_id, results['reason']))
    else:
      errors.append((bill_id, results))
      logging.error("[%s] Error: %s" % (bill_id, results['reason']))
Esempio n. 14
0
def run(options):
    bill_id = options.get('bill_id', None)

    if bill_id:
        bill_type, number, congress = utils.split_bill_id(bill_id)
        to_fetch = [bill_id]
    else:
        to_fetch = get_bills_to_process(options)

        if not to_fetch:
            logging.warn("No bills changed.")
            return None

        limit = options.get('limit', None)
        if limit:
            to_fetch = to_fetch[:int(limit)]

    utils.process_set(to_fetch, process_bill, options)
Esempio n. 15
0
def run(options):
    bill_id = options.get('bill_id', None)

    if bill_id:
        bill_type, number, congress = utils.split_bill_id(bill_id)
        to_fetch = [bill_id]
    else:
        to_fetch = get_bills_to_process(options)

        if not to_fetch:
            logging.warn("No bills changed.")
            return None

        limit = options.get('limit', None)
        if limit:
            to_fetch = to_fetch[:int(limit)]

    utils.process_set(to_fetch, process_bill, options)
Esempio n. 16
0
def run(options):
  bill_id = options.get('bill_id', None)
  bill_version_id = options.get('bill_version_id', None)

  # using a specific bill or version overrides the congress flag/default
  if bill_id:
    bill_type, number, congress = utils.split_bill_id(bill_id)
  elif bill_version_id:
    bill_type, number, congress, version_code = utils.split_bill_version_id(bill_version_id)
  else:
    congress = options.get('congress', utils.current_congress())

  if bill_version_id:
    to_fetch = [bill_version_id]
  else:
    to_fetch = bill_version_ids_for(congress, options)
    if not to_fetch:
      logging.error("Error figuring out which bills to download, aborting.")
      return None

  saved_versions = utils.process_set(to_fetch, write_bill_catoxml, options)
Esempio n. 17
0
def run(options):
    bill_id = options.get('bill_id', None)
    bill_version_id = options.get('bill_version_id', None)

    # using a specific bill or version overrides the congress flag/default
    if bill_id:
        bill_type, number, congress = utils.split_bill_id(bill_id)
    elif bill_version_id:
        bill_type, number, congress, version_code = utils.split_bill_version_id(
            bill_version_id)
    else:
        congress = options.get('congress', utils.current_congress())

    if bill_version_id:
        to_fetch = [bill_version_id]
    else:
        to_fetch = bill_version_ids_for(congress, options)
        if not to_fetch:
            logging.error(
                "Error figuring out which bills to download, aborting.")
            return None

    saved_versions = utils.process_set(to_fetch, write_bill_catoxml, options)
Esempio n. 18
0
def run(options):
    bill_version_id = options.get("bill_version_id", None)

    if bill_version_id:
        bill_type, bill_number, congress, version_code = utils.split_bill_version_id(bill_version_id)
        bill_id = utils.build_bill_id(bill_type, bill_number, congress)
    else:
        version_code = None
        bill_id = options.get("bill_id", None)

        if bill_id:
            bill_type, bill_number, congress = utils.split_bill_id(bill_id)
        else:
            bill_type = bill_number = None
            congress = options.get("congress", utils.current_congress())

    force = options.get("force", False)

    to_fetch = bill_version_ids_for(congress, bill_type, bill_number, version_code, force)

    if not to_fetch:
        return None

    saved_versions = utils.process_set(to_fetch, write_bill_catoxml, options)
Esempio n. 19
0
def parse_bill_action(action_dict, prev_status, bill_id, title):
    """Parse a THOMAS bill action line. Returns attributes to be set in the XML file on the action line."""

    bill_type, number, congress = utils.split_bill_id(bill_id)

    line = action_dict['text']

    status = None
    action = {
        "type": "action"
    }

    # If a line starts with an amendment number, this action is on the amendment and cannot
    # be parsed yet.
    m = re.search(r"^(H|S)\.Amdt\.(\d+)", line, re.I)
    if m != None:
        # Process actions specific to amendments separately.
        return None, None

    # Otherwise, parse the action line for key actions.

    # VOTES

    # A House Vote.
    line = re.sub(", the Passed", ", Passed", line) # 106 h4733 and others
    m = re.search("("
        + "|".join([
            "On passage",
            "Passed House",
            "Two-thirds of the Members present having voted in the affirmative the bill is passed,?",
            "On motion to suspend the rules and pass the (?:bill|resolution)",
            "On agreeing to the (?:resolution|conference report)",
            "On motion to suspend the rules and agree to the (?:resolution|conference report)",
            "House Agreed to Senate Amendments.*?",
            "On motion (?:that )?the House (?:suspend the rules and )?(?:agree(?: with an amendment)? to|concur in) the Senate amendments?(?: to the House amendments?| to the Senate amendments?)*",
        ])
        + ")"
        + "(, the objections of the President to the contrary notwithstanding.?)?"
        + "(, as amended| \(Amended\))?"
        + "\.? (Passed|Failed|Agreed to|Rejected)?" # hr1625-115 has a stray period here
        + " ?(by voice vote|without objection|by (the Yeas and Nays?|Yea-Nay Vote|recorded vote)"
        + "(:? \(2/3 required\))?: (\d+ ?- ?\d+(, \d+ Present)? [ \)]*)?\((Roll no\.|Record Vote No:) \d+\))",
        line, re.I)
    if m != None:
        motion, is_override, as_amended, pass_fail, how = m.group(1), m.group(2), m.group(3), m.group(4), m.group(5)

        # print line
        # print m.groups()

        if re.search(r"Passed House|House Agreed to", motion, re.I):
            pass_fail = 'pass'
        elif re.search("(ayes|yeas) had prevailed", line, re.I):
            pass_fail = 'pass'
        elif re.search(r"Pass|Agreed", pass_fail, re.I):
            pass_fail = 'pass'
        else:
            pass_fail = 'fail'

        if "Two-thirds of the Members present" in motion:
            is_override = True

        if is_override:
            vote_type = "override"
        elif re.search(r"(agree (with an amendment )?to|concur in) the Senate amendment", line, re.I):
            vote_type = "pingpong"
        elif re.search("conference report", line, re.I):
            vote_type = "conference"
        elif bill_type[0] == "h":
            vote_type = "vote"
        else:
            vote_type = "vote2"

        roll = None
        m = re.search(r"\((Roll no\.|Record Vote No:) (\d+)\)", how, re.I)
        if m != None:
            how = "roll"  # normalize the ugly how
            roll = m.group(2)

        suspension = None
        if roll and "On motion to suspend the rules" in motion:
            suspension = True

        # alternate form of as amended, e.g. hr3979-113
        if "the House agree with an amendment" in motion:
            as_amended = True

        action["type"] = "vote"
        action["vote_type"] = vote_type
        action["how"] = how
        action['where'] = "h"
        action['result'] = pass_fail
        if roll:
            action["roll"] = roll
        action["suspension"] = suspension

        # correct upstream data error
        if bill_id == "s2012-114" and "Roll no. 250" in line: as_amended = True
        if bill_id == "s2943-114" and "On passage Passed without objection" in line: as_amended = True

        # get the new status of the bill after this vote
        new_status = new_status_after_vote(vote_type, pass_fail == "pass", "h", bill_type, suspension, as_amended, title, prev_status)
        if new_status:
            status = new_status

    # Passed House, not necessarily by an actual vote (think "deem")
    m = re.search(r"Passed House pursuant to|House agreed to Senate amendment (with amendment )?pursuant to", line, re.I)
    if m != None:
        vote_type = "vote" if (bill_type[0] == "h") else "vote2"
        if "agreed to Senate amendment" in line: vote_type = "pingpong"
        pass_fail = "pass"
        as_amended = bool(m.group(1))

        action["type"] = "vote"
        action["vote_type"] = vote_type
        action["how"] = "by special rule"
        action["where"] = "h"
        action["result"] = pass_fail

        # It's always pursuant to another bill, and a bill number is given in the action line, which we parse out
        # into the bill_ids field of the action. It's also represented
        # structurally in the links->link elements of the original XML which we just put in "links".

        # get the new status of the bill after this vote
        new_status = new_status_after_vote(vote_type, pass_fail == "pass", "h", bill_type, False, as_amended, title, prev_status)

        if new_status:
            status = new_status

    # House motions to table adversely dispose of a pending matter, if agreed to. An agreed-to "motion to table the measure",
    # which is very infrequent, kills the legislation. If not agreed to, nothing changes. So this regex only captures
    # agreed-to motions to table.
    m = re.search("On motion to table the measure Agreed to"
        + " ?(by voice vote|without objection|by (the Yeas and Nays|Yea-Nay Vote|recorded vote)"
        + ": (\d+ - \d+(, \d+ Present)? [ \)]*)?\((Roll no\.|Record Vote No:) \d+\))",
        line, re.I)
    if m != None:
        how = m.group(1)
        pass_fail = 'fail'

        # In order to classify this as resulting in the same thing as regular failed vote on passage, new_status_after_vote
        # needs to know if this was a vote in the originating chamber or not.
        if prev_status == "INTRODUCED" or bill_id.startswith("hres"):
            vote_type = "vote"
        elif False:
            vote_type = "vote2"
        else:
            raise Exception("Need to classify %s as being in the originating chamber or not." % prev_status)

        roll = None
        m = re.search(r"\((Roll no\.|Record Vote No:) (\d+)\)", how, re.I)
        if m != None:
            how = "roll"  # normalize the ugly how
            roll = m.group(2)

        action["type"] = "vote"
        action["vote_type"] = vote_type
        action["how"] = how
        action['where'] = "h"
        action['result'] = pass_fail
        if roll:
            action["roll"] = roll

        # get the new status of the bill after this vote
        new_status = new_status_after_vote(vote_type, pass_fail == "pass", "h", bill_type, False, False, title, prev_status)
        if new_status:
            status = new_status

    # A Senate Vote
    # (There are some annoying weird cases of double spaces which are taken care of
    # at the end.)
    m = re.search("("
        + "|".join([
        "Passed Senate",
        "Failed of passage in Senate",
        "Disagreed to in Senate",
        "Resolution agreed to in Senate",
        "Senate (?:agreed to|concurred in) (?:the )?(?:conference report|House amendment(?: to the Senate amendments?| to the House amendments?)*)",
        "Senate receded from its amendment and concurred", # hr1-115
        r"Cloture \S*\s?on the motion to proceed .*?not invoked in Senate",
        r"Cloture(?: motion)? on the motion to proceed to the (?:bill|measure) invoked in Senate",
        "Cloture invoked in Senate",
        "Cloture on (?:the motion to (?:proceed to |concur in )(?:the House amendment (?:to the Senate amendment )?to )?)(?:the bill|H.R. .*) (?:not )?invoked in Senate",
        "(?:Introduced|Received|Submitted) in the Senate, (?:read twice, |considered, |read the third time, )+and (?:passed|agreed to)",
        ])
        + ")"
        + "(,?.*,?) "
        + "(without objection|by Unanimous Consent|by Voice Vote|(?:by )?Yea-Nay( Vote)?\. \d+\s*-\s*\d+\. Record Vote (No|Number): \d+)",
        line.replace("  ", " "), re.I)
    if m != None:
        motion, extra, how = m.group(1), m.group(2), m.group(3)
        roll = None

        # put disagreed check first, cause "agreed" is contained inside it
        if re.search("disagreed|not invoked", motion, re.I):
            pass_fail = "fail"
        elif re.search("passed|agreed|concurred|invoked", motion, re.I):
            pass_fail = "pass"
        else:
            pass_fail = "fail"

        voteaction_type = "vote"
        if re.search("over veto", extra, re.I):
            vote_type = "override"
        elif re.search("conference report", motion, re.I):
            vote_type = "conference"
        elif re.search("cloture", motion, re.I):
            vote_type = "cloture"
            voteaction_type = "vote-aux"  # because it is not a vote on passage
        elif re.search("Senate agreed to (the )?House amendment|Senate concurred in (the )?House amendment", motion, re.I):
            vote_type = "pingpong"
        elif bill_type[0] == "s":
            vote_type = "vote"
        else:
            vote_type = "vote2"

        m = re.search(r"Record Vote (No|Number): (\d+)", how, re.I)
        if m != None:
            roll = m.group(2)
            how = "roll"

        as_amended = False
        if re.search(r"with amendments|with an amendment", extra, re.I):
            as_amended = True

        action["type"] = voteaction_type
        action["vote_type"] = vote_type
        action["how"] = how
        action["result"] = pass_fail
        action["where"] = "s"
        if roll:
            action["roll"] = roll

        # get the new status of the bill after this vote
        new_status = new_status_after_vote(vote_type, pass_fail == "pass", "s", bill_type, False, as_amended, title, prev_status)

        if new_status:
            status = new_status

    # OLD-STYLE VOTES (93rd Congress-ish)

    m = re.search(r"Measure passed (House|Senate)(, amended(?: \(.*?\)|, with an amendment to the title)?)?(?:,? in lieu[^,]*)?(?:, roll call #(\d+) \(\d+-\d+\))?", line, re.I)
    if m != None:
        chamber = m.group(1)[0].lower()  # 'h' or 's'
        as_amended = m.group(2)
        roll_num = m.group(3)
        # GovTrack legacy scraper missed these: if chamber == 's' and (as_amended or roll_num or "lieu" in line): return action, status
        pass_fail = "pass"
        vote_type = "vote" if bill_type[0] == chamber else "vote2"
        action["type"] = "vote"
        action["vote_type"] = vote_type
        action["how"] = "(method not recorded)" if not roll_num else "roll"
        if roll_num:
            action["roll"] = roll_num
        action["result"] = pass_fail
        action["where"] = chamber
        new_status = new_status_after_vote(vote_type, pass_fail == "pass", chamber, bill_type, False, as_amended, title, prev_status)
        if new_status:
            status = new_status

    m = re.search(r"(House|Senate) agreed to (?:House|Senate) amendments?( with an amendment)?( under Suspension of the Rules)?(?:, roll call #(\d+) \(\d+-\d+\))?\.", line, re.I)
    if m != None:
        chamber = m.group(1)[0].lower()  # 'h' or 's'
        as_amended = m.group(2)
        suspension = m.group(3)
        roll_num = m.group(4)
        # GovTrack legacy scraper missed these: if (chamber == 'h' and not roll_num) or (chamber == 's' and rull_num): return action, status # REMOVE ME
        pass_fail = "pass"
        vote_type = "pingpong"
        action["type"] = "vote"
        action["vote_type"] = vote_type
        action["how"] = "(method not recorded)" if not roll_num else "roll"
        if roll_num:
            action["roll"] = roll_num
        action["result"] = pass_fail
        action["where"] = chamber
        action["suspension"] = (suspension != None)
        new_status = new_status_after_vote(vote_type, pass_fail == "pass", chamber, bill_type, False, as_amended, title, prev_status)
        if new_status:
            status = new_status

    # PSUDO-REPORTING (because GovTrack did this, but should be changed)

    # TODO: Make a new status for this as pre-reported.
    m = re.search(r"Placed on (the )?([\w ]+) Calendar( under ([\w ]+))?[,\.] Calendar No\. (\d+)\.|Committee Agreed to Seek Consideration Under Suspension of the Rules|Ordered to be Reported", line, re.I)
    if m != None:
        # TODO: This makes no sense.
        if prev_status in ("INTRODUCED", "REFERRED"):
            status = "REPORTED"

        action["type"] = "calendar"

        # TODO: Useless. But good for GovTrack compatibility.
        if m.group(2):  # not 'Ordered to be Reported'
            action["calendar"] = m.group(2)
            action["under"] = m.group(4)
            action["number"] = m.group(5)

    # COMMITTEE ACTIONS

    # reported
    m = re.search(r"Committee on (.*)\. Reported by", line, re.I)
    if m != None:
        action["type"] = "reported"
        action["committee"] = m.group(1)
        if prev_status in ("INTRODUCED", "REFERRED"):
            status = "REPORTED"
    m = re.search(r"Reported to Senate from the (.*?)( \(without written report\))?\.", line, re.I)
    if m != None:  # 93rd Congress
        action["type"] = "reported"
        action["committee"] = m.group(1)
        if prev_status in ("INTRODUCED", "REFERRED"):
            status = "REPORTED"

    # hearings held by a committee
    m = re.search(r"(Committee on .*?)\. Hearings held", line, re.I)
    if m != None:
        action["committee"] = m.group(1)
        action["type"] = "hearings"

    m = re.search(r"Committee on (.*)\. Discharged (by Unanimous Consent)?", line, re.I)
    if m != None:
        action["committee"] = m.group(1)
        action["type"] = "discharged"
        if prev_status in ("INTRODUCED", "REFERRED"):
            status = "REPORTED"

    m = re.search("Cleared for White House|Presented to President", line, re.I)
    if m != None:
        action["type"] = "topresident"

    m = re.search("Signed by President", line, re.I)
    if m != None:
        action["type"] = "signed"
        status = "ENACTED:SIGNED"

    m = re.search("Pocket Vetoed by President", line, re.I)
    if m != None:
        action["type"] = "vetoed"
        action["pocket"] = "1"
        status = "VETOED:POCKET"

    # need to put this in an else, or this regex will match the pocket veto and override it
    else:
        m = re.search("Vetoed by President", line, re.I)
        if m != None:
            action["type"] = "vetoed"
            status = "PROV_KILL:VETO"

    m = re.search("Sent to Archivist of the United States unsigned", line, re.I)
    if m != None:
        status = "ENACTED:TENDAYRULE"

    m = re.search("^(?:Became )?(Public|Private) Law(?: No:)? ([\d\-]+)\.", line, re.I)
    if m != None:
        action["law"] = m.group(1).lower()
        pieces = m.group(2).split("-")
        action["congress"] = pieces[0]
        action["number"] = pieces[1]
        action["type"] = "enacted"
        if prev_status in ("ENACTED:SIGNED", "ENACTED:VETO_OVERRIDE", "ENACTED:TENDAYRULE"):
            pass  # this is a final administrative step
        elif prev_status == "PROV_KILL:VETO" or prev_status.startswith("VETOED:"):
            # somehow missed the override steps
            status = "ENACTED:VETO_OVERRIDE"
        elif bill_id in ("s2641-93", "hr1589-94", "s2527-100", "hr1677-101", "hr2978-101", "hr2126-104", "s1322-104"):
            status = "ENACTED:TENDAYRULE"
        else:
            raise Exception("Missing Signed by President action? If this is a case of the 10-day rule, hard code the bill id %s here." % bill_id)

    # Check for referral type
    m = re.search(r"Referred to (?:the )?(House|Senate)?\s?(?:Committee|Subcommittee)?", line, re.I)
    if m != None:
        action["type"] = "referral"
        if prev_status == "INTRODUCED":
            status = "REFERRED"

    # sweep the action line for bill IDs of related bills
    bill_ids = utils.extract_bills(line, congress)
    bill_ids = filter(lambda b: b != bill_id, bill_ids)
    if bill_ids and (len(bill_ids) > 0):
        action['bill_ids'] = bill_ids

    return action, status
Esempio n. 20
0
def fetch_amendment(amendment_id, options):
    logging.info("\n[%s] Fetching..." % amendment_id)

    body = utils.download(
        amendment_url_for(amendment_id),
        amendment_cache_for(amendment_id, "information.html"),
        options)

    if not body:
        return {'saved': False, 'ok': False, 'reason': "failed to download"}

    if options.get("download_only", False):
        return {'saved': False, 'ok': True, 'reason': "requested download only"}

    if "Amends:" not in body:
        return {'saved': False, 'ok': True, 'reason': "orphaned amendment"}

    amendment_type, number, congress = utils.split_bill_id(amendment_id)

    actions = actions_for(body, amendment_id, is_amendment=True)
    if actions is None:
        actions = []
    parse_amendment_actions(actions)

    chamber = amendment_type[0]

    # good set of tests for each situation:
    # samdt712-113 - amendment to bill
    # samdt112-113 - amendment to amendment on bill
    # samdt4904-111 - amendment to treaty
    # samdt4922-111 - amendment to amendment to treaty

    amends_bill = amends_bill_for(body)  # almost always present
    amends_treaty = amends_treaty_for(body)  # present if bill is missing
    amends_amendment = amends_amendment_for(body)  # sometimes present
    if not amends_bill and not amends_treaty:
        raise Exception("Choked finding out what bill or treaty the amendment amends.")

    amdt = {
        'amendment_id': amendment_id,
        'amendment_type': amendment_type,
        'chamber': chamber,
        'number': int(number),
        'congress': congress,

        'amends_bill': amends_bill,
        'amends_treaty': amends_treaty,
        'amends_amendment': amends_amendment,

        'sponsor': sponsor_for(body),

        'description': amendment_simple_text_for(body, "description"),
        'purpose': amendment_simple_text_for(body, "purpose"),

        'actions': actions,

        'updated_at': datetime.datetime.fromtimestamp(time.time()),
    }

    if chamber == 'h':
        amdt['introduced_at'] = offered_at_for(body, 'offered')
    elif chamber == 's':
        amdt['introduced_at'] = offered_at_for(body, 'submitted')
        amdt['proposed_at'] = offered_at_for(body, 'proposed')

    if not amdt.get('introduced_at', None):
        raise Exception("Couldn't find a reliable introduction date for amendment.")

    # needs to come *after* the setting of introduced_at
    amdt['status'], amdt['status_at'] = amendment_status_for(amdt)

    # only set a house_number if it's a House bill -
    # this lets us choke if it's not found.
    if amdt['chamber'] == 'h':
        # numbers found in vote XML
        # summary = amdt['purpose'] if amdt['purpose'] else amdt['description']
        # amdt['house_number'] = house_simple_number_for(amdt['amendment_id'], summary)

        if int(amdt['congress']) > 100:
            # A___-style numbers, present only starting with the 101st Congress
            amdt['house_number'] = house_number_for(body)

    output_amendment(amdt, options)

    return {'ok': True, 'saved': True}
Esempio n. 21
0
def billstatus_url_for(bill_id):
    bill_type, bill_number, congress = utils.split_bill_id(bill_id)
    return govinfo.BULKDATA_BASE_URL + 'BILLSTATUS/{0}/{1}/BILLSTATUS-{0}{1}{2}.xml'.format(congress, bill_type, bill_number)
Esempio n. 22
0
def amendment_url_for(amendment_id):
    amendment_type, number, congress = utils.split_bill_id(amendment_id)
    thomas_type = utils.thomas_types[amendment_type][0]
    congress = int(congress)
    number = int(number)
    return "http://thomas.loc.gov/cgi-bin/bdquery/z?d%03d:%s%s:" % (congress, thomas_type, number)
Esempio n. 23
0
def amdt_url_for(amdt_id):
  amdt_type, number, congress = utils.split_bill_id(amdt_id)
  thomas_type = utils.thomas_types[amdt_type][0]
  congress = int(congress)
  number = int(number)
  return "http://thomas.loc.gov/cgi-bin/bdquery/D?d%03d:%d:./list/bss/d%03d%s.lst::" % (congress, number, congress, thomas_type)
Esempio n. 24
0
def amendment_cache_for(amendment_id, file):
    amendment_type, number, congress = utils.split_bill_id(amendment_id)
    return "%s/amendments/%s/%s%s/%s" % (congress, amendment_type,
                                         amendment_type, number, file)
Esempio n. 25
0
def billstatus_url_for(bill_id):
    bill_type, bill_number, congress = utils.split_bill_id(bill_id)
    return fdsys.BULKDATA_BASE_URL + 'BILLSTATUS/{0}/{1}/BILLSTATUS-{0}{1}{2}.xml'.format(congress, bill_type, bill_number)
Esempio n. 26
0
def fetch_amendment(amendment_id, options):
    logging.info("\n[%s] Fetching..." % amendment_id)

    body = utils.download(
        amendment_url_for(amendment_id),
        amendment_cache_for(amendment_id, "information.html"), options)

    if not body:
        return {'saved': False, 'ok': False, 'reason': "failed to download"}

    if options.get("download_only", False):
        return {
            'saved': False,
            'ok': True,
            'reason': "requested download only"
        }

    if "Amends:" not in body:
        return {'saved': False, 'ok': True, 'reason': "orphaned amendment"}

    amendment_type, number, congress = utils.split_bill_id(amendment_id)

    actions = actions_for(body, amendment_id, is_amendment=True)
    if actions is None: actions = []
    parse_amendment_actions(actions)

    chamber = amendment_type[0]

    # good set of tests for each situation:
    # samdt712-113 - amendment to bill
    # samdt112-113 - amendment to amendment on bill
    # samdt4904-111 - amendment to treaty
    # samdt4922-111 - amendment to amendment to treaty

    amends_bill = amends_bill_for(body)  # almost always present
    amends_treaty = amends_treaty_for(body)  # present if bill is missing
    amends_amendment = amends_amendment_for(body)  # sometimes present
    if not amends_bill and not amends_treaty:
        raise Exception(
            "Choked finding out what bill or treaty the amendment amends.")

    amdt = {
        'amendment_id': amendment_id,
        'amendment_type': amendment_type,
        'chamber': chamber,
        'number': int(number),
        'congress': congress,
        'amends_bill': amends_bill,
        'amends_treaty': amends_treaty,
        'amends_amendment': amends_amendment,
        'sponsor': sponsor_for(body),
        'description': amendment_simple_text_for(body, "description"),
        'purpose': amendment_simple_text_for(body, "purpose"),
        'actions': actions,
        'updated_at': datetime.datetime.fromtimestamp(time.time()),
    }

    if chamber == 'h':
        amdt['introduced_at'] = offered_at_for(body, 'offered')
    elif chamber == 's':
        amdt['introduced_at'] = offered_at_for(body, 'submitted')
        amdt['proposed_at'] = offered_at_for(body, 'proposed')

    if not amdt.get('introduced_at', None):
        raise Exception(
            "Couldn't find a reliable introduction date for amendment.")

    # needs to come *after* the setting of introduced_at
    amdt['status'], amdt['status_at'] = amendment_status_for(amdt)

    # only set a house_number if it's a House bill -
    # this lets us choke if it's not found.
    if amdt['chamber'] == 'h':
        # numbers found in vote XML
        # summary = amdt['purpose'] if amdt['purpose'] else amdt['description']
        # amdt['house_number'] = house_simple_number_for(amdt['amendment_id'], summary)

        if int(amdt['congress']) > 100:
            # A___-style numbers, present only starting with the 101st Congress
            amdt['house_number'] = house_number_for(body)

    output_amendment(amdt, options)

    return {'ok': True, 'saved': True}
Esempio n. 27
0
def output_for_amdt(amendment_id, format):
    amendment_type, number, congress = utils.split_bill_id(amendment_id)
    return "%s/%s/amendments/%s/%s%s/%s" % (utils.data_dir(), congress, amendment_type, amendment_type, number, "data.%s" % format)
Esempio n. 28
0
def create_govtrack_xml(bill, options):
    govtrack_type_codes = {'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc'}
    root = etree.Element("bill")
    root.set("session", bill['congress'])
    root.set("type", govtrack_type_codes[bill['bill_type']])
    root.set("number", bill['number'])
    root.set("updated", utils.format_datetime(bill['updated_at']))

    def make_node(parent, tag, text, **attrs):
        if options.get("govtrack", False):
            # Rewrite bioguide_id attributes as just id with GovTrack person IDs.
            attrs2 = {}
            for k, v in attrs.items():
                if v:
                    if k == "bioguide_id":
                        # remap "bioguide_id" attributes to govtrack "id"
                        k = "id"
                        v = str(utils.translate_legislator_id('bioguide', v, 'govtrack'))
                    attrs2[k] = v
            attrs = attrs2

        return utils.make_node(parent, tag, text, **attrs)

    # for American Memory Century of Lawmaking bills...
    for source in bill.get("sources", []):
        n = make_node(root, "source", "")
        for k, v in sorted(source.items()):
            if k == "source":
                n.text = v
            elif k == "source_url":
                n.set("url", v)
            else:
                n.set(k, unicode(v))
    if "original_bill_number" in bill:
        make_node(root, "bill-number", bill["original_bill_number"])

    make_node(root, "state", bill['status'], datetime=bill['status_at'])
    old_status = make_node(root, "status", None)
    make_node(old_status, "introduced" if bill['status'] in ("INTRODUCED", "REFERRED") else "unknown", None, datetime=bill['status_at'])  # dummy for the sake of comparison

    make_node(root, "introduced", None, datetime=bill['introduced_at'])
    titles = make_node(root, "titles", None)
    for title in bill['titles']:
        n = make_node(titles, "title", title['title'])
        n.set("type", title['type'])
        if title['as']:
            n.set("as", title['as'])
        if title['is_for_portion']:
            n.set("partial", "1")

    if bill['sponsor']:
        # TODO: Sponsored by committee?
        make_node(root, "sponsor", None, bioguide_id=bill['sponsor']['bioguide_id'])
    else:
        make_node(root, "sponsor", None)

    cosponsors = make_node(root, "cosponsors", None)
    for cosp in bill['cosponsors']:
        n = make_node(cosponsors, "cosponsor", None, bioguide_id=cosp["bioguide_id"])
        if cosp["sponsored_at"]:
            n.set("joined", cosp["sponsored_at"])
        if cosp["withdrawn_at"]:
            n.set("withdrawn", cosp["withdrawn_at"])

    actions = make_node(root, "actions", None)
    for action in bill['actions']:
        a = make_node(actions,
                      action['type'] if action['type'] in ("vote", "vote-aux", "calendar", "topresident", "signed", "enacted", "vetoed") else "action",
                      None,
                      datetime=action['acted_at'])
        if action.get("status"):
            a.set("state", action["status"])
        if action['type'] in ('vote', 'vote-aux'):
            a.clear()  # re-insert date between some of these attributes
            a.set("how", action["how"])
            a.set("type", action["vote_type"])
            if action.get("roll") != None:
                a.set("roll", action["roll"])
            a.set("datetime", utils.format_datetime(action['acted_at']))
            a.set("where", action["where"])
            a.set("result", action["result"])
            if action.get("suspension"):
                a.set("suspension", "1")
            if action.get("status"):
                a.set("state", action["status"])
        if action['type'] == 'calendar' and "calendar" in action:
            a.set("calendar", action["calendar"])
            if action["under"]:
                a.set("under", action["under"])
            if action["number"]:
                a.set("number", action["number"])
        if action['type'] == 'enacted':
            a.clear()  # re-insert date between some of these attributes
            a.set("number", "%s-%s" % (bill['congress'], action["number"]))
            a.set("type", action["law"])
            a.set("datetime", utils.format_datetime(action['acted_at']))
            if action.get("status"):
                a.set("state", action["status"])
        if action['type'] == 'vetoed':
            if action.get("pocket"):
                a.set("pocket", "1")
        if action.get('text'):
            make_node(a, "text", action['text'])
        if action.get('in_committee'):
            make_node(a, "committee", None, name=action['in_committee'])
        for cr in action['references']:
            make_node(a, "reference", None, ref=cr['reference'], label=cr['type'])

    committees = make_node(root, "committees", None)
    for cmt in bill['committees']:
        make_node(committees, "committee", None, code=(cmt["committee_id"] + cmt["subcommittee_id"]) if cmt.get("subcommittee_id", None) else cmt["committee_id"], name=cmt["committee"], subcommittee=cmt.get("subcommittee").replace("Subcommittee on ", "") if cmt.get("subcommittee") else "", activity=", ".join(c.title() for c in cmt["activity"]))

    relatedbills = make_node(root, "relatedbills", None)
    for rb in bill['related_bills']:
        if rb['type'] == "bill":
            rb_bill_type, rb_number, rb_congress = utils.split_bill_id(rb['bill_id'])
            make_node(relatedbills, "bill", None, session=rb_congress, type=govtrack_type_codes[rb_bill_type], number=rb_number, relation="unknown" if rb['reason'] == "related" else rb['reason'])

    subjects = make_node(root, "subjects", None)
    if bill['subjects_top_term']:
        make_node(subjects, "term", None, name=bill['subjects_top_term'])
    for s in bill['subjects']:
        if s != bill['subjects_top_term']:
            make_node(subjects, "term", None, name=s)

    amendments = make_node(root, "amendments", None)
    for amd in bill['amendments']:
        make_node(amendments, "amendment", None, number=amd["chamber"] + str(amd["number"]))

    if bill.get('summary'):
        make_node(root, "summary", bill['summary']['text'], date=bill['summary']['date'], status=bill['summary']['as'])

    committee_reports = make_node(root, "committee-reports", None)
    for report in bill['committee_reports']:
        make_node(committee_reports, "report", report)

    return etree.tostring(root, pretty_print=True)
Esempio n. 29
0
def amdt_cache_for(amdt_id, file):
  amdt_type, number, congress = utils.split_bill_id(amdt_id)
  return "%s/amendments/%s/%s%s/%s" % (congress, amdt_type, amdt_type, number, file)
Esempio n. 30
0
def output_for_amdt(amendment_id, format):
    amendment_type, number, congress = utils.split_bill_id(amendment_id)
    return "%s/%s/amendments/%s/%s%s/%s" % (utils.data_dir(), congress,
                                            amendment_type, amendment_type,
                                            number, "data.%s" % format)