def get_organizations(self): city = Organization('City of Minneapolis', classification='executive') city.add_post('Mayor', 'Mayor', division_id='ocd-division/country:us/state:mn/place:minneapolis') city.add_post('City Clerk', 'City Clerk', division_id='ocd-division/country:us/state:mn/place:minneapolis') yield city council = Organization(name="Minneapolis City Council", classification="legislature", parent_id=city) for x in range(1, 14): council.add_post( "Ward {}".format(x), "Councilmember", division_id='ocd-division/country:us/state:mn/place:minneapolis/ward:{}'.format(x)) yield council frey = Person(name="Frey, Jacob") frey.add_term('Mayor', 'executive', start_date=datetime.date(2018, 1, 19), appointment=True) frey.add_source('http://www.google.com') yield frey parks = Organization('Minneapolis Parks and Recreation', classification='legislature') for x in range(1, 7): parks.add_post( "District {}".format(x), "Commissioner") parks.add_post("At Large", "Commissioner") yield parks school = Organization('Minneapolis School Board', classification='legislature') for x in range(1, 7): school.add_post( "District {}".format(x), "Director",) school.add_post("At Large", "Director") yield school cmt_link = 'https://lims.minneapolismn.gov/Calendar/GetCommittees' cmts_site = requests.get(cmt_link) cmts = cmts_site.json() for c in cmts: name = c['Name'] abbv = c['Abbreviation'] org_id = c['Id'] active = c['Active'] member_count = c['MembersCount'] purpose = c['Purpose'] start_date = c['StartDate'] chair = c['ChairMan'] members = c['Members'] location = c['Location'] address = c['Address'] mtg_time = c['MeetingTime'] quorum = c['QuorumCount'] if name != 'City Council': org = Organization(name, classification='committee', parent_id=council) org.add_source(cmt_link) if start_date != None: org.founding_date = start_date.split('T')[0] yield org
def scrape(self): current_path = Path(__file__) committee_path = current_path.parent / 'congress-legislators/committees-historical.yaml' with committee_path.open() as f: committees = yaml.load(f, Loader=yaml.CLoader) for committee in committees: if committee['type'] == 'house': chamber = 'lower' elif committee['type'] == 'senate': chamber = 'upper' else: print(committee) raise names_int = { int(key): name for key, name in committee['names'].items() } _, current_name = max(names_int.items()) if chamber == 'lower': current_name = 'House Committee on ' + current_name elif chamber == 'upper': current_name = 'Senate Committee on ' + current_name c = Organization(current_name, classification='committee', chamber=chamber) start, end = duration(committee) c.founding_date = start if end: c.dissolution_date = end c.add_identifier(committee['thomas_id'] + '00', scheme='thomas_id') if 'house_committee_id' in committee: c.add_identifier(committee['house_committee_id'], scheme='house_committee_id') if 'senate_committee_id' in committee: c.add_identifier(committee['senate_committee_id'], scheme='senate_committee_id') c.add_source( 'https://github.com/unitedstates/congress-legislators/blob/master/committees-historical.yaml' ) for name in set(committee['names'].values()): if chamber == 'lower': name = 'House Committee on ' + name elif chamber == 'upper': name = 'Senate Committee on ' + name if name != c.name: c.add_name(name) yield c for subcommittee in committee.get('subcommittees', []): names_int = { int(key): name for key, name in subcommittee['names'].items() } _, current_name = max(names_int.items()) sc = Organization('Subcommittee on ' + current_name, classification='committee', parent_id=c) start, end = duration(subcommittee) sc.founding_date = start if end: sc.dissolution_date = end thomas_id = (committee['thomas_id'] + subcommittee['thomas_id']) sc.add_identifier(thomas_id, scheme='thomas_id') sc.add_source( 'https://github.com/unitedstates/congress-legislators/blob/master/committees-historical.yaml' ) if thomas_id == 'SSJU12': sc.add_identifier('SSJU15', scheme='thomas_id') elif thomas_id == 'SSJU15': continue if 'Oversight and Investigations' in sc.name: print(thomas_id) #input() for name in set(subcommittee['names'].values()): name = 'Subcommittee on ' + name if name != sc.name: sc.add_name(name) yield sc