def get_top_bills(self):
"""
Parse updated XML list of ranked bills From OpenCongress Service
"""
import urllib
self.request_args = {'order': 'desc',
'page' : 1,
'sort' : 'vote_count_1',
'timeframe' : '1day' }
self.formatted_args = urllib.urlencode(self.request_args)
from google.appengine.api import urlfetch
fetch_page = urlfetch.fetch( url = OPENCONGRESS_BR_URL + self.formatted_args,
method = urlfetch.GET)
from utils.BeautifulSoup import BeautifulStoneSoup
document = BeautifulStoneSoup(fetch_page.content)
bills = []
ids = [i.contents[0] for i in document.findAll('ident')]
titles = [t.contents[0] for t in document.findAll('title-full-common')]
current_rank = 1
for i,t in zip(ids, titles)[:BILL_LIMIT + 1]:
this_title = " ".join(str(t).decode('utf-8').split(' ')[1:]).split('Act')[0] + "Act"
if len(this_title) > 35 : this_title = " ".join( this_title.split(' ')[:3] ) + "..."
if this_title in [bill['title'] for bill in bills]:
logging.warning('duplicate bill found with title %s and id %s' %(t, i))
continue # this bill has already been processed once
bills.append({'id': str(i),
'title': this_title,
'rank' : current_rank })
current_rank += 1
return bills
def get_similar_topics(self, tag):
# Freebase request to get similar topics for a tag.
tag = tag.replace(' ','%20') #urlencode tag
tilde_request = str(TILDE_BASE_URL) + "?format=xml&topic_limit=" + str(TILDE_TOPIC_LIMIT) + "&type_limit=" + str(TILDE_TYPE_LIMIT) + "&exclude=" + str(TILDE_EXCLUDE) + "&request=" + str(tag)
try:
logging.debug('Fetching tilde response')
tilde_response = urlfetch.fetch(tilde_request)
except:
logging.debug('Unable to fetch tilde response')
return False
tilde_soup = BeautifulStoneSoup(tilde_response.content)
similar_topics = []
for topic in tilde_soup.findAll('title'):
try: encoded_topic = unicode(topic.contents[0]).encode('utf-8')
except: continue
try: encoded_topic.decode('ascii')
except: continue
if len(encoded_topic) < len(tag) + 10: similar_topics.append(encoded_topic)
similar_topics = self.rank_tags(similar_topics)
if len(similar_topics) >= AC_MIN:
return similar_topics[1:AC_LIMIT]
else:
return False
def get_fixture(self):
fixture_file = open(ROOT_PATH + "/data/fixtures.xml")
fixtures = BeautifulStoneSoup(fixture_file.read())
this_name = fixtures.findAll('record')[int(self.fixture_offset.value)].contents[1].contents[0]
this_email = fixtures.findAll('record')[int(self.fixture_offset.value)].contents[3].contents[0]
return self.create_user(str(this_name), str(this_email))
