def get_top_bills(self): """ Parse updated XML list of ranked bills From OpenCongress Service """ import urllib self.request_args = {'order': 'desc', 'page' : 1, 'sort' : 'vote_count_1', 'timeframe' : '1day' } self.formatted_args = urllib.urlencode(self.request_args) from google.appengine.api import urlfetch fetch_page = urlfetch.fetch( url = OPENCONGRESS_BR_URL + self.formatted_args, method = urlfetch.GET) from utils.BeautifulSoup import BeautifulStoneSoup document = BeautifulStoneSoup(fetch_page.content) bills = [] ids = [i.contents[0] for i in document.findAll('ident')] titles = [t.contents[0] for t in document.findAll('title-full-common')] current_rank = 1 for i,t in zip(ids, titles)[:BILL_LIMIT + 1]: this_title = " ".join(str(t).decode('utf-8').split(' ')[1:]).split('Act')[0] + "Act" if len(this_title) > 35 : this_title = " ".join( this_title.split(' ')[:3] ) + "..." if this_title in [bill['title'] for bill in bills]: logging.warning('duplicate bill found with title %s and id %s' %(t, i)) continue # this bill has already been processed once bills.append({'id': str(i), 'title': this_title, 'rank' : current_rank }) current_rank += 1 return bills
def get_similar_topics(self, tag): # Freebase request to get similar topics for a tag. tag = tag.replace(' ','%20') #urlencode tag tilde_request = str(TILDE_BASE_URL) + "?format=xml&topic_limit=" + str(TILDE_TOPIC_LIMIT) + "&type_limit=" + str(TILDE_TYPE_LIMIT) + "&exclude=" + str(TILDE_EXCLUDE) + "&request=" + str(tag) try: logging.debug('Fetching tilde response') tilde_response = urlfetch.fetch(tilde_request) except: logging.debug('Unable to fetch tilde response') return False tilde_soup = BeautifulStoneSoup(tilde_response.content) similar_topics = [] for topic in tilde_soup.findAll('title'): try: encoded_topic = unicode(topic.contents[0]).encode('utf-8') except: continue try: encoded_topic.decode('ascii') except: continue if len(encoded_topic) < len(tag) + 10: similar_topics.append(encoded_topic) similar_topics = self.rank_tags(similar_topics) if len(similar_topics) >= AC_MIN: return similar_topics[1:AC_LIMIT] else: return False
def get_fixture(self): fixture_file = open(ROOT_PATH + "/data/fixtures.xml") fixtures = BeautifulStoneSoup(fixture_file.read()) this_name = fixtures.findAll('record')[int(self.fixture_offset.value)].contents[1].contents[0] this_email = fixtures.findAll('record')[int(self.fixture_offset.value)].contents[3].contents[0] return self.create_user(str(this_name), str(this_email))