def test_committee_detail_resp(self):
committee = Committee(name="Committee Name",
main_contact="Main Contact", slug="commitee-name")
committee.save()
response = self.client.get(committee.get_absolute_url())
self.assertEqual(response.status_code, 200)
def committee_list(request):
from events.models import Feed
import re
def getlist(type_):
items = list(
Committee.objects.filter(committee_type=type_, obsolete=False))
for c in items:
if c.name.startswith("Joint "):
c.display_name = c.name
else:
c.display_name = c.sortname()
return sorted(items, key=lambda c: c.display_name)
return {
'senate_committees': getlist(CommitteeType.senate),
'house_committees': getlist(CommitteeType.house),
'joint_committees': getlist(CommitteeType.joint),
'feed': Committee.AllCommitteesFeed(),
}
def committee_list(request):
from events.models import Feed
import re
def getlist(type_):
items = list(Committee.objects.filter(committee_type=type_, obsolete=False))
for c in items:
if c.name.startswith("Joint "):
c.display_name = c.name
else:
c.display_name = c.sortname()
return sorted(items, key=lambda c : c.display_name)
# Get data on recent committee activity by date so we can display a visualization.
committee_activity_by_date = { }
today = datetime.now().date()
def first_monday_after(d): # d.weekday() is zero if d is a Monday
return d + timedelta((7-d.weekday()) % 7)
start_date = first_monday_after(today - timedelta(days=365))
def daterange(start_date, end_date): # https://stackoverflow.com/a/1060330
for n in range(int((end_date - start_date).days)):
yield start_date + timedelta(n)
for d in list(daterange(start_date, today)):
if d.weekday() <= 4: # exclude Saturday/Sunday because Congress rarely conducts business then
committee_activity_by_date[d] = {
"count": 0,
"date": d.strftime("%b %d").replace(" 0", " "),
}
for cm in CommitteeMeeting.objects.filter(when__gte=start_date).values_list("when", flat=True):
if cm.date() in committee_activity_by_date:
committee_activity_by_date[cm.date()]["count"] += 1
committee_activity_by_date = [v for k, v in sorted(committee_activity_by_date.items())]
return {
'senate_committees': getlist(CommitteeType.senate),
'house_committees': getlist(CommitteeType.house),
'joint_committees': getlist(CommitteeType.joint),
'feed': Committee.AllCommitteesFeed(),
'upcoming_meetings': CommitteeMeeting.objects.filter(when__gte=datetime.now().date()).count(),
'committee_activity_by_date': committee_activity_by_date,
}
def main(options):
"""
Process committees, subcommittees and
members of current congress committees.
"""
BASE_PATH = settings.CONGRESS_LEGISLATORS_PATH
meeting_processor = CommitteeMeetingProcessor()
log.info('Processing committees')
COMMITTEES_FILE = BASE_PATH + 'committees-current.yaml'
if not File.objects.is_changed(COMMITTEES_FILE) and not options.force:
log.info('File %s was not changed' % COMMITTEES_FILE)
else:
tree = yaml_load(COMMITTEES_FILE)
total = len(tree)
progress = Progress(total=total)
seen_committees = set()
for committee in tree:
try:
cobj = Committee.objects.get(code=committee["thomas_id"])
except Committee.DoesNotExist:
print "New committee:", committee["thomas_id"]
cobj = Committee(code=committee["thomas_id"])
cobj.committee_type = TYPE_MAPPING[committee["type"]]
cobj.name = committee["name"]
cobj.url = committee.get("url", None)
cobj.obsolete = False
cobj.committee = None
cobj.save()
seen_committees.add(cobj.id)
for subcom in committee.get('subcommittees', []):
code = committee["thomas_id"] + subcom["thomas_id"]
try:
sobj = Committee.objects.get(code=code)
except Committee.DoesNotExist:
print "New subcommittee:", code
sobj = Committee(code=code)
sobj.name = subcom["name"]
sobj.url = subcom.get("url", None)
sobj.type = None
sobj.committee = cobj
sobj.obsolete = False
sobj.save()
seen_committees.add(sobj.id)
progress.tick()
# Check for non-obsolete committees in the database that aren't in our
# file.
other_committees = Committee.objects.filter(obsolete=False).exclude(id__in=seen_committees)
if len(other_committees) > 0:
print "Marking obsolete:", ", ".join(c.code for c in other_committees)
other_committees.update(obsolete=True)
File.objects.save_file(COMMITTEES_FILE)
log.info('Processing committee members')
MEMBERS_FILE = BASE_PATH + 'committee-membership-current.yaml'
file_changed = File.objects.is_changed(MEMBERS_FILE)
if not file_changed and not options.force:
log.info('File %s was not changed' % MEMBERS_FILE)
else:
# map THOMAS IDs to GovTrack IDs
y = yaml_load(BASE_PATH + "legislators-current.yaml")
person_id_map = { }
for m in y:
if "id" in m and "govtrack" in m["id"] and "thomas" in m["id"]:
person_id_map[m["id"]["thomas"]] = m["id"]["govtrack"]
# load committee members
tree = yaml_load(MEMBERS_FILE)
total = len(tree)
progress = Progress(total=total, name='committees')
# We can delete CommitteeMember objects because we don't have
# any foreign keys to them.
CommitteeMember.objects.all().delete()
# Process committee nodes
for committee, members in tree.items():
if committee[0] == "H": continue # House data is out of date
try:
cobj = Committee.objects.get(code=committee)
except Committee.DoesNotExist:
print "Committee not found:", committee
continue
# Process members of current committee node
for member in members:
mobj = CommitteeMember()
mobj.person = Person.objects.get(id=person_id_map[member["thomas"]])
mobj.committee = cobj
if "title" in member:
mobj.role = ROLE_MAPPING[member["title"]]
mobj.save()
progress.tick()
File.objects.save_file(MEMBERS_FILE)
return
log.info('Processing committee schedule')
SCHEDULE_FILE = 'data/us/112/committeeschedule.xml'
file_changed = File.objects.is_changed(SCHEDULE_FILE)
if not file_changed and not options.force:
log.info('File %s was not changed' % SCHEDULE_FILE)
else:
tree = etree.parse(SCHEDULE_FILE)
# We have to clear out all CommitteeMeeting objects when we refresh because
# we have no unique identifier in the upstream data for a meeting. We might use
# the meeting's committee & date as an identifier, but since meeting times can
# change this might have awkward consequences for the end user if we even
# attempted to track that.
CommitteeMeeting.objects.all().delete()
# Process committee event nodes
for meeting in tree.xpath('/committee-schedule/meeting'):
try:
mobj = meeting_processor.process(CommitteeMeeting(), meeting)
mobj.save()
mobj.bills.clear()
for bill in meeting.xpath('bill'):
bill = Bill.objects.get(congress=bill.get("session"), bill_type=BillType.by_xml_code(bill.get("type")), number=int(bill.get("number")))
mobj.bills.add(bill)
except Committee.DoesNotExist:
log.error('Could not load Committee object for meeting %s' % meeting_processor.display_node(meeting))
for committee in Committee.objects.all():
if not options.disable_events:
committee.create_events()
File.objects.save_file(SCHEDULE_FILE)
def main(options):
"""
Process committees, subcommittees and
members of current congress committees.
"""
BASE_PATH = settings.CONGRESS_LEGISLATORS_PATH
meeting_processor = CommitteeMeetingProcessor()
log.info('Processing committees')
COMMITTEES_FILE = BASE_PATH + 'committees-current.yaml'
if not File.objects.is_changed(COMMITTEES_FILE) and not options.force:
log.info('File %s was not changed' % COMMITTEES_FILE)
else:
tree = yaml_load(COMMITTEES_FILE)
total = len(tree)
progress = Progress(total=total)
seen_committees = set()
for committee in tree:
try:
cobj = Committee.objects.get(code=committee["thomas_id"])
except Committee.DoesNotExist:
print "New committee:", committee["thomas_id"]
cobj = Committee(code=committee["thomas_id"])
cobj.committee_type = TYPE_MAPPING[committee["type"]]
cobj.name = committee["name"]
cobj.url = committee.get("url", None)
cobj.obsolete = False
cobj.committee = None
cobj.jurisdiction = committee.get("jurisdiction")
cobj.jurisdiction_link = committee.get("jurisdiction_source")
cobj.save()
seen_committees.add(cobj.id)
for subcom in committee.get('subcommittees', []):
code = committee["thomas_id"] + subcom["thomas_id"]
try:
sobj = Committee.objects.get(code=code)
except Committee.DoesNotExist:
print "New subcommittee:", code
sobj = Committee(code=code)
sobj.name = subcom["name"]
sobj.url = subcom.get("url", None)
sobj.type = None
sobj.committee = cobj
sobj.obsolete = False
sobj.save()
seen_committees.add(sobj.id)
progress.tick()
# Check for non-obsolete committees in the database that aren't in our
# file.
other_committees = Committee.objects.filter(obsolete=False).exclude(
id__in=seen_committees)
if len(other_committees) > 0:
print "Marking obsolete:", ", ".join(c.code
for c in other_committees)
other_committees.update(obsolete=True)
File.objects.save_file(COMMITTEES_FILE)
log.info('Processing committee members')
MEMBERS_FILE = BASE_PATH + 'committee-membership-current.yaml'
file_changed = File.objects.is_changed(MEMBERS_FILE)
if not file_changed and not options.force:
log.info('File %s was not changed' % MEMBERS_FILE)
else:
# map THOMAS IDs to GovTrack IDs
y = yaml_load(BASE_PATH + "legislators-current.yaml")
person_id_map = {}
for m in y:
if "id" in m and "govtrack" in m["id"] and "thomas" in m["id"]:
person_id_map[m["id"]["thomas"]] = m["id"]["govtrack"]
# load committee members
tree = yaml_load(MEMBERS_FILE)
total = len(tree)
progress = Progress(total=total, name='committees')
# We can delete CommitteeMember objects because we don't have
# any foreign keys to them.
CommitteeMember.objects.all().delete()
# Process committee nodes
for committee, members in tree.items():
try:
cobj = Committee.objects.get(code=committee)
except Committee.DoesNotExist:
print "Committee not found:", committee
continue
# Process members of current committee node
for member in members:
mobj = CommitteeMember()
mobj.person = Person.objects.get(
id=person_id_map[member["thomas"]])
mobj.committee = cobj
if "title" in member:
mobj.role = ROLE_MAPPING[member["title"]]
mobj.save()
progress.tick()
File.objects.save_file(MEMBERS_FILE)
log.info('Processing committee schedule')
for chamber in ("house", "senate"):
meetings_file = 'data/congress/committee_meetings_%s.json' % chamber
file_changed = File.objects.is_changed(meetings_file)
if not file_changed and not options.force:
log.info('File %s was not changed' % meetings_file)
else:
meetings = json.load(open(meetings_file))
# Process committee event nodes
for meeting in meetings:
try:
# Associate it with an existing meeting object if GUID is already known.
# Must get it like this, vs just assigning the ID as we do in other parsers,
# because of the auto_now_add created field, which otherwise misbehaves.
try:
mobj = CommitteeMeeting.objects.get(
guid=meeting['guid'])
except CommitteeMeeting.DoesNotExist:
mobj = CommitteeMeeting()
# Parse.
mobj = meeting_processor.process(mobj, meeting)
# Attach the meeting to the subcommittee if set.
if mobj.subcommittee:
mobj.committee = Committee.objects.get(
code=mobj.committee.code + mobj.subcommittee)
mobj.save()
mobj.bills.clear()
for bill in meeting["bill_ids"]:
try:
bill_type, bill_num, bill_cong = re.match(
r"([a-z]+)(\d+)-(\d+)$", bill).groups()
bill = Bill.objects.get(
congress=bill_cong,
bill_type=BillType.by_slug(bill_type),
number=int(bill_num))
mobj.bills.add(bill)
except AttributeError:
pass # regex failed
except common.enum.NotFound:
pass # invalid bill type code in source data
except Bill.DoesNotExist:
pass # we don't know about bill yet
except Committee.DoesNotExist:
log.error(
'Could not load Committee object for meeting %s' %
meeting_processor.display_node(meeting))
for committee in Committee.objects.all():
if not options.disable_events:
committee.create_events()
File.objects.save_file(meetings_file)
def main(options):
"""
Process committees, subcommittees and
members of current congress committees.
"""
BASE_PATH = settings.CONGRESS_LEGISLATORS_PATH
meeting_processor = CommitteeMeetingProcessor()
log.info('Processing committees')
COMMITTEES_FILE = BASE_PATH + 'committees-current.yaml'
if not File.objects.is_changed(COMMITTEES_FILE) and not options.force:
log.info('File %s was not changed' % COMMITTEES_FILE)
else:
tree = yaml_load(COMMITTEES_FILE)
total = len(tree)
progress = Progress(total=total)
seen_committees = set()
for committee in tree:
try:
cobj = Committee.objects.get(code=committee["thomas_id"])
except Committee.DoesNotExist:
print "New committee:", committee["thomas_id"]
cobj = Committee(code=committee["thomas_id"])
cobj.committee_type = TYPE_MAPPING[committee["type"]]
cobj.name = committee["name"]
cobj.url = committee.get("url", None)
cobj.obsolete = False
cobj.committee = None
cobj.jurisdiction = committee.get("jurisdiction")
cobj.jurisdiction_link = committee.get("jurisdiction_source")
cobj.save()
seen_committees.add(cobj.id)
for subcom in committee.get('subcommittees', []):
code = committee["thomas_id"] + subcom["thomas_id"]
try:
sobj = Committee.objects.get(code=code)
except Committee.DoesNotExist:
print "New subcommittee:", code
sobj = Committee(code=code)
sobj.name = subcom["name"]
sobj.url = subcom.get("url", None)
sobj.type = None
sobj.committee = cobj
sobj.obsolete = False
sobj.save()
seen_committees.add(sobj.id)
progress.tick()
# Check for non-obsolete committees in the database that aren't in our
# file.
other_committees = Committee.objects.filter(obsolete=False).exclude(id__in=seen_committees)
if len(other_committees) > 0:
print "Marking obsolete:", ", ".join(c.code for c in other_committees)
other_committees.update(obsolete=True)
File.objects.save_file(COMMITTEES_FILE)
log.info('Processing committee members')
MEMBERS_FILE = BASE_PATH + 'committee-membership-current.yaml'
file_changed = File.objects.is_changed(MEMBERS_FILE)
if not file_changed and not options.force:
log.info('File %s was not changed' % MEMBERS_FILE)
else:
# map THOMAS IDs to GovTrack IDs
y = yaml_load(BASE_PATH + "legislators-current.yaml")
person_id_map = { }
for m in y:
if "id" in m and "govtrack" in m["id"] and "thomas" in m["id"]:
person_id_map[m["id"]["thomas"]] = m["id"]["govtrack"]
# load committee members
tree = yaml_load(MEMBERS_FILE)
total = len(tree)
progress = Progress(total=total, name='committees')
# We can delete CommitteeMember objects because we don't have
# any foreign keys to them.
CommitteeMember.objects.all().delete()
# Process committee nodes
for committee, members in tree.items():
try:
cobj = Committee.objects.get(code=committee)
except Committee.DoesNotExist:
print "Committee not found:", committee
continue
# Process members of current committee node
for member in members:
mobj = CommitteeMember()
mobj.person = Person.objects.get(id=person_id_map[member["thomas"]])
mobj.committee = cobj
if "title" in member:
mobj.role = ROLE_MAPPING[member["title"]]
mobj.save()
progress.tick()
File.objects.save_file(MEMBERS_FILE)
log.info('Processing committee schedule')
for chamber in ("house", "senate"):
meetings_file = 'data/congress/committee_meetings_%s.json' % chamber
file_changed = File.objects.is_changed(meetings_file)
if not file_changed and not options.force:
log.info('File %s was not changed' % meetings_file)
else:
meetings = json.load(open(meetings_file))
# Process committee event nodes
for meeting in meetings:
try:
# Associate it with an existing meeting object if GUID is already known.
# Must get it like this, vs just assigning the ID as we do in other parsers,
# because of the auto_now_add created field, which otherwise misbehaves.
try:
mobj = CommitteeMeeting.objects.get(guid=meeting['guid'])
except CommitteeMeeting.DoesNotExist:
mobj = CommitteeMeeting()
# Parse.
mobj = meeting_processor.process(mobj, meeting)
# Attach the meeting to the subcommittee if set.
if mobj.subcommittee:
mobj.committee = Committee.objects.get(code=mobj.committee.code + mobj.subcommittee)
mobj.save()
mobj.bills.clear()
for bill in meeting["bill_ids"]:
try:
bill_type, bill_num, bill_cong = re.match(r"([a-z]+)(\d+)-(\d+)$", bill).groups()
bill = Bill.objects.get(congress=bill_cong, bill_type=BillType.by_slug(bill_type), number=int(bill_num))
mobj.bills.add(bill)
except AttributeError:
pass # regex failed
except common.enum.NotFound:
pass # invalid bill type code in source data
except Bill.DoesNotExist:
pass # we don't know about bill yet
except Committee.DoesNotExist:
log.error('Could not load Committee object for meeting %s' % meeting_processor.display_node(meeting))
for committee in Committee.objects.all():
if not options.disable_events:
committee.create_events()
File.objects.save_file(meetings_file)
def parse_committee_names(options):
log.info('Processing committees')
COMMITTEES_FILE = settings.CONGRESS_PROJECT_PATH + '/congress-legislators/committees-current.yaml'
if not File.objects.is_changed(COMMITTEES_FILE) and not options.force:
log.info('File %s was not changed' % COMMITTEES_FILE)
else:
tree = yaml_load(COMMITTEES_FILE)
total = len(tree)
progress = Progress(total=total)
seen_committees = set()
for committee in tree:
try:
cobj = Committee.objects.get(code=committee["thomas_id"])
except Committee.DoesNotExist:
print("New committee:", committee["thomas_id"])
cobj = Committee(code=committee["thomas_id"])
cobj.committee_type = TYPE_MAPPING[committee["type"]]
cobj.name = committee["name"]
cobj.url = committee.get("url", None)
cobj.obsolete = False
cobj.committee = None
cobj.jurisdiction = committee.get("jurisdiction")
cobj.jurisdiction_link = committee.get("jurisdiction_source")
cobj.save()
seen_committees.add(cobj.id)
for subcom in committee.get('subcommittees', []):
code = committee["thomas_id"] + subcom["thomas_id"]
try:
sobj = Committee.objects.get(code=code)
except Committee.DoesNotExist:
print("New subcommittee:", code)
sobj = Committee(code=code)
sobj.name = subcom["name"]
sobj.url = subcom.get("url", None)
sobj.type = None
sobj.committee = cobj
sobj.obsolete = False
sobj.save()
seen_committees.add(sobj.id)
progress.tick()
# Check for non-obsolete committees in the database that aren't in our
# file.
other_committees = Committee.objects.filter(obsolete=False).exclude(
id__in=seen_committees)
if len(other_committees) > 0:
print("Marking obsolete:",
", ".join(c.code for c in other_committees))
other_committees.update(obsolete=True)
File.objects.save_file(COMMITTEES_FILE)
def parse_committee_names(options):
log.info('Processing committees')
COMMITTEES_FILE = settings.CONGRESS_PROJECT_PATH + '/congress-legislators/committees-current.yaml'
if not File.objects.is_changed(COMMITTEES_FILE) and not options.force:
log.info('File %s was not changed' % COMMITTEES_FILE)
else:
tree = yaml_load(COMMITTEES_FILE)
total = len(tree)
progress = Progress(total=total)
seen_committees = set()
for committee in tree:
try:
cobj = Committee.objects.get(code=committee["thomas_id"])
except Committee.DoesNotExist:
print("New committee:", committee["thomas_id"])
cobj = Committee(code=committee["thomas_id"])
cobj.committee_type = TYPE_MAPPING[committee["type"]]
cobj.name = committee["name"]
cobj.url = committee.get("url", None)
cobj.obsolete = False
cobj.committee = None
cobj.jurisdiction = committee.get("jurisdiction")
cobj.jurisdiction_link = committee.get("jurisdiction_source")
cobj.save()
seen_committees.add(cobj.id)
for subcom in committee.get('subcommittees', []):
code = committee["thomas_id"] + subcom["thomas_id"]
try:
sobj = Committee.objects.get(code=code)
except Committee.DoesNotExist:
print("New subcommittee:", code)
sobj = Committee(code=code)
sobj.name = subcom["name"]
sobj.url = subcom.get("url", None)
sobj.type = None
sobj.committee = cobj
sobj.obsolete = False
sobj.save()
seen_committees.add(sobj.id)
progress.tick()
# Check for non-obsolete committees in the database that aren't in our
# file.
other_committees = Committee.objects.filter(obsolete=False).exclude(id__in=seen_committees)
if len(other_committees) > 0:
print("Marking obsolete:", ", ".join(c.code for c in other_committees))
other_committees.update(obsolete=True)
File.objects.save_file(COMMITTEES_FILE)