def can_get_history_detail_using_summary_row():
config = {"hostname": "phila.legistar.com", "fulltext": True}
scraper = LegistarScraper(config)
legislation_summary = {
"URL": "http://phila.legistar.com/LegislationDetail.aspx?ID=1236768&GUID=EB92A4C2-469A-4D73-97C0-A620BBDDD5BE&Options=ID|Text|&Search="
}
legislation_details = scraper.expandLegislationSummary(legislation_summary)
history_summary = legislation_details[1][2]
attrs, votes = scraper.expandHistorySummary(history_summary)
ayes = [vote for vote in votes if vote["Vote"] == "Ayes"]
assert_equal(len(ayes), 14)
assert_equal(attrs["Result"], "Pass")
def can_get_history_detail_using_summary_row():
config = Config(hostname="chicago.legistar.com", sponsor_links=False).defaults(DEFAULT_CONFIG)
scraper = LegistarScraper(config)
legislation_summary = {
"URL": "https://chicago.legistar.com/LegislationDetail.aspx?ID=1450228&GUID=97689689-D0EA-47A2-8474-09B3A71C221B&Options=Advanced&Search="
}
legislation_details = scraper.expandLegislationSummary(legislation_summary)
history_summary = legislation_details[1][0]["Action Details"]
attrs, votes = scraper.expandHistorySummary(history_summary)
ayes = [vote for vote in votes if vote["Vote"] == "Yea"]
assert_equal(len(ayes), 49)
assert_equal(attrs["Result"], "Pass")
def can_get_history_detail_using_summary_row():
config = Config(hostname='phila.legistar.com',
sponsor_links=False).defaults(DEFAULT_CONFIG)
scraper = LegistarScraper(config)
legislation_summary = {
'URL':
'http://phila.legistar.com/LegislationDetail.aspx?ID=1236768&GUID=EB92A4C2-469A-4D73-97C0-A620BBDDD5BE&Options=ID|Text|&Search='
}
legislation_details = scraper.expandLegislationSummary(legislation_summary)
history_summary = legislation_details[1][2]
attrs, votes = scraper.expandHistorySummary(history_summary)
ayes = [vote for vote in votes if vote['Vote'] == 'Ayes']
assert_equal(len(ayes), 14)
assert_equal(attrs['Result'], 'Pass')
def can_get_history_detail_using_summary_row():
config = Config(hostname='chicago.legistar.com',
sponsor_links=False).defaults(DEFAULT_CONFIG)
scraper = LegistarScraper(config)
legislation_summary = {
'URL':
'https://chicago.legistar.com/LegislationDetail.aspx?ID=1450228&GUID=97689689-D0EA-47A2-8474-09B3A71C221B&Options=Advanced&Search='
}
legislation_details = scraper.expandLegislationSummary(legislation_summary)
history_summary = legislation_details[1][0]['Action Details']
attrs, votes = scraper.expandHistorySummary(history_summary)
ayes = [vote for vote in votes if vote['Vote'] == 'Yea']
assert_equal(len(ayes), 49)
assert_equal(attrs['Result'], 'Pass')
class HostedLegistarSiteWrapper (object):
"""
A generic facade over hosted legistar site data scraper.
It is responsible for interpreting data scraped out of the site by LegistarScraper.
The main external point of interaction is scrape_legis_file.
NOTE that this is a superclass that will not run by itself and isn't
meant to be; you are expected to run a subclass that implements
some functions with names starting with "pluck".
requires: BeautifulSoup, mechanize
"""
def __init__(self, cmdline_options, **options):
self.scraper = LegistarScraper(options)
if cmdline_options['year']:
self.legislation_summaries = self.scraper.searchLegislation('',year=cmdline_options['year'])
else:
self.legislation_summaries = self.scraper.searchLegislation('')
def scrape_legis_file(self, key, summary):
'''Extract a record from the given document (soup). The key is for the
sake of record-keeping. It is the key passed to the site URL.'''
while True :
try:
legislation_attrs, legislation_history = self.scraper.expandLegislationSummary(summary)
break
except urllib2.URLError as e:
print e
print 'skipping to next leg record'
except AttributeError as e :
print e
print 'skipping to next leg record'
while True :
try:
summary = self.legislation_summaries.next()
break
except urllib2.URLError as e:
print e
print 'sleeping for five minutes'
time.sleep('360')
parsed_url = urlparse.urlparse(summary['URL'])
key = urlparse.parse_qs(parsed_url.query)['ID'][0]
record = self.pluck_record(key, summary, legislation_attrs)
attachments = self.pluck_attachments(key, legislation_attrs)
actions = []
for act in legislation_history :
act_details = act_votes = []
try:
act_details, act_votes = self.scraper.expandHistorySummary(act)
except (KeyError, AttributeError) as e:
log.debug('LegAction has no url')
else:
if act_votes:
print "act_votes", act_votes
try:
action = self.pluck_action(key, act, act_details, act_votes)
except TypeError as e:
print e
print summary
continue
except KeyError as e :
print act
print e
print summary
raise
actions.append(action)
# we should probably remove this from the model since the hosted
# legistar does not have minutes
minutes = []
log.info('Scraped legfile with key %r' % (key,))
log.debug("%r %r %r %r" % (record, attachments, actions, minutes))
return record, attachments, actions, minutes
def convert_date(self, orig_date):
if orig_date:
return datetime.datetime.strptime(orig_date, '%m/%d/%Y').date()
else:
return ''
def check_for_new_content(self, last_key):
'''Grab the next legislation summary row. Doesn't use the last_key
parameter; just starts at the beginning for each instance of the
scraper.
'''
try:
print 'next leg record'
next_summary = self.legislation_summaries.next()
return 0, next_summary
except StopIteration:
return None, None
def init_pdf_cache(self, pdf_mapping) :
pass
class HostedLegistarSiteWrapper(object):
"""
A facade over the Philadelphia city council legistar site data. It is
responsible for scraping data out of the site. The main external point
of interaction is scrape_legis_file.
requires: BeautifulSoup, mechanize
"""
def __init__(self, **options):
self.scraper = LegistarScraper(options)
self.legislation_summaries = self.scraper.searchLegislation(
'', created_before='2012-10-5')
def scrape_legis_file(self, key, summary):
'''Extract a record from the given document (soup). The key is for the
sake of record-keeping. It is the key passed to the site URL.'''
while True:
try:
legislation_attrs, legislation_history = self.scraper.expandLegislationSummary(
summary)
break
except urllib2.URLError as e:
print e
print 'skipping to next leg record'
except AttributeError as e:
print e
print 'skipping to next leg record'
while True:
try:
summary = self.legislation_summaries.next()
break
except urllib2.URLError as e:
print e
print 'sleeping for five minutes'
time.sleep('360')
parsed_url = urlparse.urlparse(summary['URL'])
key = urlparse.parse_qs(parsed_url.query)['ID'][0]
# re-order the sponsor name by '[First] [Last]' instead of '[Last], [First]'
sponsors = legislation_attrs['Sponsors']
first_name_first_sponsors = []
for sponsor in sponsors:
if ',' in sponsor:
name_list = sponsor.split(',')
name_list.reverse()
sponsor = ' '.join(name_list).strip()
first_name_first_sponsors.append(sponsor)
record = {
'key':
key,
'id':
summary['Record #'],
'url':
summary['URL'],
'type':
summary['Type'],
'status':
summary['Status'],
'title':
summary['Title'],
'controlling_body':
legislation_attrs['Current Controlling Legislative Body'],
'intro_date':
self.convert_date(summary['Intro Date']),
'final_date':
self.convert_date(summary.setdefault('Final Date', '')),
'version':
summary.setdefault('Version', ''),
#'contact' : None,
'sponsors':
first_name_first_sponsors,
# probably remove this from the model as well
'minutes_url':
None
}
try:
attachments = legislation_attrs['Attachments']
for attachment in attachments:
attachment['key'] = key
attachment['file'] = attachment['label']
attachment['description'] = attachment['label']
del attachment['label']
except KeyError:
attachments = []
actions = []
for act in legislation_history:
try:
act_details, act_votes = self.scraper.expandHistorySummary(act)
except (KeyError, AttributeError) as e:
print e
print summary
continue
try:
action = {
'key': key,
'date_taken': self.convert_date(act['Date']),
'acting_body': act['Action By']['label'],
'motion': act['Result'],
'description': act['Action'],
'notes': ''
}
except TypeError as e:
print e
print summary
continue
except KeyError as e:
print act
print e
print summary
raise
actions.append(action)
# we should probably remove this from the model since the hosted
# legistar does not have minutes
minutes = []
log.info('Scraped legfile with key %r' % (key, ))
log.debug("%r %r %r %r" % (record, attachments, actions, minutes))
return record, attachments, actions, minutes
def convert_date(self, orig_date):
if orig_date:
return datetime.datetime.strptime(orig_date, '%m/%d/%Y').date()
else:
return ''
def check_for_new_content(self, last_key):
'''Grab the next legislation summary row. Doesn't use the last_key
parameter; just starts at the beginning for each instance of the
scraper.
'''
try:
print 'next leg record'
next_summary = self.legislation_summaries.next()
return 0, next_summary
except StopIteration:
return None, None
def init_pdf_cache(self, pdf_mapping):
pass
class HostedLegistarSiteWrapper (object):
"""
A facade over the Philadelphia city council legistar site data. It is
responsible for scraping data out of the site. The main external point
of interaction is scrape_legis_file.
requires: BeautifulSoup, mechanize
"""
def __init__(self, **options):
self.scraper = LegistarScraper(options)
self.legislation_summaries = self.scraper.searchLegislation('', created_before='2012-10-5')
def scrape_legis_file(self, key, summary):
'''Extract a record from the given document (soup). The key is for the
sake of record-keeping. It is the key passed to the site URL.'''
while True :
try:
legislation_attrs, legislation_history = self.scraper.expandLegislationSummary(summary)
break
except urllib2.URLError as e:
print e
print 'skipping to next leg record'
except AttributeError as e :
print e
print 'skipping to next leg record'
while True :
try:
summary = self.legislation_summaries.next()
break
except urllib2.URLError as e:
print e
print 'sleeping for five minutes'
time.sleep('360')
parsed_url = urlparse.urlparse(summary['URL'])
key = urlparse.parse_qs(parsed_url.query)['ID'][0]
# re-order the sponsor name by '[First] [Last]' instead of '[Last], [First]'
sponsors = legislation_attrs['Sponsors']
first_name_first_sponsors = []
for sponsor in sponsors :
if ',' in sponsor :
name_list = sponsor.rsplit(',', 1)
name_list.reverse()
sponsor = ' '.join(name_list).strip().replace(',', '')
first_name_first_sponsors.append(sponsor)
record = {
'key' : key,
'id' : summary['Record #'],
'url' : summary['URL'],
'type' : summary['Type'],
'status' : summary['Status'],
'title' : summary['Title'],
'controlling_body' : legislation_attrs['Current Controlling Legislative Body'],
'intro_date' : self.convert_date(summary['Intro Date']),
'final_date' : self.convert_date(summary.setdefault('Final Date', '')),
'version' : summary.setdefault('Version', ''),
#'contact' : None,
'sponsors' : first_name_first_sponsors,
# probably remove this from the model as well
'minutes_url' : None
}
try:
attachments = legislation_attrs['Attachments']
for attachment in attachments:
attachment['key'] = key
attachment['file'] = attachment['label']
attachment['description'] = attachment['label']
del attachment['label']
except KeyError:
attachments = []
actions = []
for act in legislation_history :
try:
act_details, act_votes = self.scraper.expandHistorySummary(act)
except (KeyError, AttributeError) as e:
print e
print summary
continue
try:
action = {
'key' : key,
'date_taken' : self.convert_date(act['Date']),
'acting_body' : act['Action By']['label'],
'motion' : act['Result'],
'description' : act['Action'],
'notes' : ''
}
except TypeError as e:
print e
print summary
continue
except KeyError as e :
print act
print e
print summary
raise
actions.append(action)
# we should probably remove this from the model since the hosted
# legistar does not have minutes
minutes = []
log.info('Scraped legfile with key %r' % (key,))
log.debug("%r %r %r %r" % (record, attachments, actions, minutes))
return record, attachments, actions, minutes
def convert_date(self, orig_date):
if orig_date:
return datetime.datetime.strptime(orig_date, '%m/%d/%Y').date()
else:
return ''
def check_for_new_content(self, last_key):
'''Grab the next legislation summary row. Doesn't use the last_key
parameter; just starts at the beginning for each instance of the
scraper.
'''
try:
print 'next leg record'
next_summary = self.legislation_summaries.next()
return 0, next_summary
except StopIteration:
return None, None
def init_pdf_cache(self, pdf_mapping) :
pass