def fetchHousehPVIfeed(): response = get_contents_of_url(house_hpvi_feed_url) data = json.loads(response[62:-2]) hpvi = {} for r in data['table']['rows']: hpvi[r['c'][0]['v']] = r['c'][1]['v'] return hpvi
def fetchHousehPVIfeed(): response = get_contents_of_url(house_hpvi_feed_url) data=json.loads(response[62:-2]) hpvi={} for r in data['table']['rows']: hpvi[r['c'][0]['v']]=r['c'][1]['v'] return hpvi
def getSenateCommittees(): response = get_contents_of_url(mn_senate_base + '/committees/') if response != None: soup = BeautifulSoup(response) info = soup.find_all('div', 'HRDFlyer') links = links = info[0].find_all('a') name = '' committees = [] count = 0 for l in links: if l.text.find('Members') >= 0: members = l['href'] elif l.text.find('Schedule') >= 0: schedule = l['href'] elif l.text.find('Audio/Video') >= 0: av = l['href'] committee = { 'committee': name, 'chamber': 'upper', 'id': getCommitteeIDFromURL(l['href']), 'members_url': members, 'meetings_url': schedule, 'media_url': av, } committees.append(committee) else: if l.text[1:].find(' ') > 0: name = l.text[1:l.text[1:].find(' ')] else: name = l.text[1:] return committees else: return None
def getSenateCommitteeMembers(url): response = get_contents_of_url(url) if response != None: soup = BeautifulSoup(response) title = soup.find('div', 'leg_PageContent').h2.text title = title[:title.find('Membership') - 1] info = soup.find_all('div', 'leg_Col3of4-First HRDFlyer') meetings = soup.find('div', 'leg_Col1of4-Last HRDFlyer') items = info[0].find_all('td') results = [] for i in items: t = [text for text in i.stripped_strings] if t: if t[0].find(':') > 0: m = { 'name': t[1][:t[1].find(' (')], 'role': t[0][:-1], 'leg_id': getLegislatorIDByName(t[1][:t[1].find(' (')]), } else: m = { 'name': t[0][:t[0].find(' (')], 'role': 'member', 'leg_id': getLegislatorIDByName(t[0][:t[0].find(' (')]), } results.append(m) return title, results, meetings
def getSenateCommittees(): response=get_contents_of_url(mn_senate_base+'/committees/') if response!=None: soup=BeautifulSoup(response) info = soup.find_all('div','HRDFlyer') links = links=info[0].find_all('a') name='' committees=[] count=0 for l in links: if l.text.find('Members')>=0: members=l['href'] elif l.text.find('Schedule')>=0: schedule=l['href'] elif l.text.find('Audio/Video')>=0: av=l['href'] committee={'committee':name, 'chamber':'upper', 'id': getCommitteeIDFromURL(l['href']), 'members_url':members, 'meetings_url':schedule, 'media_url':av,} committees.append(committee) else: if l.text[1:].find(' ')>0: name=l.text[1:l.text[1:].find(' ')] else: name=l.text[1:] return committees else: return None
def fetchRawHousehPVIfeed(): response = get_contents_of_url(house_raw_hpvi_feed_url) data=json.loads(response[62:-2]) hpvi=[] for r in data['table']['rows']: hpvi.append(r['c'][0]['v']) return sorted(hpvi)
def fetchRawHousehPVIfeed(): response = get_contents_of_url(house_raw_hpvi_feed_url) data = json.loads(response[62:-2]) hpvi = [] for r in data['table']['rows']: hpvi.append(r['c'][0]['v']) return sorted(hpvi)
def getSenateCommitteeMeetingsByID(com_id): url=mn_senate_base+'/committees/committee_members.php?cmte_id='+com_id response=get_contents_of_url(url) if response!=None: soup=BeautifulSoup(response) meetings = soup.find('div','leg_Col1of4-Last HRDFlyer') return meetings return None
def getHouseCommitteeMeetings(url): response=get_contents_of_url(url) if response!=None: soup=BeautifulSoup(response) meeting_text = soup.find_all('div','leg_col1of3-Last') if meeting_text[0]: return meeting_text[0] return None
def getSenateCommitteeMeetingsByID(com_id): url = mn_senate_base + '/committees/committee_members.php?cmte_id=' + com_id response = get_contents_of_url(url) if response != None: soup = BeautifulSoup(response) meetings = soup.find('div', 'leg_Col1of4-Last HRDFlyer') return meetings return None
def sendGetRequest(url): url = substitute_char(url, ' ', '%20') response = get_contents_of_url(url) if response: data = json.loads(response) return data else: return None
def getHouseCommitteeMeetings(url): response = get_contents_of_url(url) if response != None: soup = BeautifulSoup(response) meeting_text = soup.find_all('div', 'leg_col1of3-Last') if meeting_text[0]: return meeting_text[0] return None
def sendGetRequest(url): url=substitute_char(url,' ','%20') response = get_contents_of_url(url) if response: data=json.loads(response) return data else: return None
def get_source(self, environment, template): path = self.path+template page=get_contents_of_url(path) if not page: raise jinja2.TemplateNotFound(template) #mtime = os.path.getmtime(path) #with file(path) as f: source = page.decode('utf-8') return source, path, lambda: False #lambda: mtime == os.path.getmtime(path)
def get_source(self, environment, template): path = self.path + template page = get_contents_of_url(path) if not page: raise jinja2.TemplateNotFound(template) #mtime = os.path.getmtime(path) #with file(path) as f: source = page.decode('utf-8') return source, path, lambda: False #lambda: mtime == os.path.getmtime(path)
def fetchElectionResults(url): response = get_contents_of_url(url) if response and response!=None: response = parseCSVfromURL(response,',') results=[] for r in response: results.append([r[1],r[2],r[3],r[4],r[5],r[6],r[7],r[8],r[9],r[10],r[11],r[12],r[13],r[14],r[15]]) return results else: return None
def fetchPrecinctTable(url): response = get_contents_of_url(url) if response and response!=None: response = parseCSVfromURL(response,';') results=[] for r in response: results.append([r[0],r[1],r[2],r[3],r[4],r[5],r[6],r[7],r[8],r[9]]) return results else: return None
def parseHouseCommitteeMeetings(url): response=get_contents_of_url(url) if response!=None: soup=BeautifulSoup(response) meeting_text = soup.find_all('div','leg_col1of3-Last') if meeting_text: if meeting_text[0].p.p: m=meeting_text[0].p.p return [text for text in m.stripped_strings] return None
def parseHouseCommitteeMeetings(url): response = get_contents_of_url(url) if response != None: soup = BeautifulSoup(response) meeting_text = soup.find_all('div', 'leg_col1of3-Last') if meeting_text: if meeting_text[0].p.p: m = meeting_text[0].p.p return [text for text in m.stripped_strings] return None
def getSenateCommitteeMeetingInfo(url): results = [] response = get_contents_of_url(url) if response != None: soup = BeautifulSoup(response) meetings = soup.find('div', 'leg_Col3of4') meet = meetings.table m = meet.find_all('table') for i in m: results.append([text for text in i.stripped_strings]) return results
def getSenateCommitteeMeetingInfo(url): results=[] response=get_contents_of_url(url) if response!=None: soup=BeautifulSoup(response) meetings = soup.find('div','leg_Col3of4') meet=meetings.table m=meet.find_all('table') for i in m: results.append([text for text in i.stripped_strings]) return results
def fetchPrecinctTable(url): response = get_contents_of_url(url) if response and response != None: response = parseCSVfromURL(response, ';') results = [] for r in response: results.append( [r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9]]) return results else: return None
def get(self,chamber): params=self.check_login('districts/'+chamber) if 'loggedin_user' not in params: self.redirect('/signup') else: page=get_contents_of_url(aws_output+'districts/'+chamber) if page: self.write(page) else: params['district_map']=get_chamber_name(chamber) params['districts']=getAllDistrictsByID(params['district_map']) self.render(all_districts_page, **params)
def fetchSenateElectionResults(): response=getFromCache('senate2012elections') if not response: response = get_contents_of_url(senate_2012_election_results) if response and response!=None: putInCache('senate2012elections',response) else: return None response = parseCSVfromURL(response,';') results=[] for r in response: results.append([r[5],r[4],r[12],r[15]," ".join([x.capitalize() for x in r[7].split(" ")]),r[10],r[14],r[13]]) return results
def get(self, chamber): params = self.check_login('districts/' + chamber) if 'loggedin_user' not in params: self.redirect('/signup') else: page = get_contents_of_url(aws_output + 'districts/' + chamber) if page: self.write(page) else: params['district_map'] = get_chamber_name(chamber) params['districts'] = getAllDistrictsByID( params['district_map']) self.render(all_districts_page, **params)
def fetchElectionResults(url): response = get_contents_of_url(url) if response and response != None: response = parseCSVfromURL(response, ',') results = [] for r in response: results.append([ r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15] ]) return results else: return None
def get(self): params=self.check_login('committees') chamber=self.request.get("q") if chamber!="house" and chamber!="senate" and chamber: self.redirect('/') else: params['chamber']='upper' if self.request.get("q")=="house": params['chamber']='lower' page=get_contents_of_url(aws_output+'committees/'+params['chamber']) if page: self.write(page) else: params['committees']=getCommitteesByChamber(params['chamber']) self.render(all_committees_page, **params)
def getBillText(url): bill_page=get_contents_of_url(url) clean_bill=substitute_char(bill_page,var_re,'') clean_bill=substitute_char(clean_bill,markup_id_re,'') soup=BeautifulSoup(clean_bill) for e in soup.find_all('br'): e.extract() br = soup.new_tag('br') bill_text = soup.find_all('div','xtend') # for e in bill_text[0]('a'): # bill_text[0].a.insert_before(br) # first_link = bill_text[0].a # first_link.find_next("a") return bill_text[0]
def get(self): params = self.check_login('committees') chamber = self.request.get("q") if chamber != "house" and chamber != "senate" and chamber: self.redirect('/') else: params['chamber'] = 'upper' if self.request.get("q") == "house": params['chamber'] = 'lower' page = get_contents_of_url(aws_output + 'committees/' + params['chamber']) if page: self.write(page) else: params['committees'] = getCommitteesByChamber( params['chamber']) self.render(all_committees_page, **params)
def fetchHouseElectionResults(): response = getFromCache('house2012elections') if not response: response = get_contents_of_url(house_2012_election_results) if response and response != None: putInCache('house2012elections', response) else: return None response = parseCSVfromURL(response, ';') results = [] for r in response: results.append([ r[5], r[4], r[12], r[15], " ".join([x.capitalize() for x in r[7].split(" ")]), r[10], r[14], r[13] ]) return results
def getBillText(url): bill_page = get_contents_of_url(url) clean_bill = substitute_char(bill_page, var_re, '') clean_bill = substitute_char(clean_bill, markup_id_re, '') soup = BeautifulSoup(clean_bill) for e in soup.find_all('br'): e.extract() br = soup.new_tag('br') bill_text = soup.find_all('div', 'xtend') # for e in bill_text[0]('a'): # bill_text[0].a.insert_before(br) # first_link = bill_text[0].a # first_link.find_next("a") return bill_text[0]
def get(self): params=self.check_login('legislators') chamber=self.request.get("q") if chamber!="house" and chamber!="senate" and chamber: self.redirect('/') else: params['chamber']='upper' params['chamber_name']="Senate" if self.request.get("q")=="house": params['chamber']='lower' params['chamber_name']='House' page=get_contents_of_url(aws_output+'legislators/'+params['chamber']) if page: self.write(page) else: params['legislators']=getLegislatorsByChamber(params['chamber']) params['search_page']="True" self.render(current_legislators_page, **params)
def get(self): params = self.check_login('legislators') chamber = self.request.get("q") if chamber != "house" and chamber != "senate" and chamber: self.redirect('/') else: params['chamber'] = 'upper' params['chamber_name'] = "Senate" if self.request.get("q") == "house": params['chamber'] = 'lower' params['chamber_name'] = 'House' page = get_contents_of_url(aws_output + 'legislators/' + params['chamber']) if page: self.write(page) else: params['legislators'] = getLegislatorsByChamber( params['chamber']) params['search_page'] = "True" self.render(current_legislators_page, **params)
def fetchDistrictDemoData(district): if district.find('u')>0: # senate district district=district[-2:] else: # house district district=district[-3:].upper() url=mnleg_district_demo_info+district+'.txt' page = get_contents_of_url(url) demographics={} if page: loop=True while loop: w,page=page[:page.find('\n')],page[page.find('\n')+1:] v,page=page[:page.find('\n')],page[page.find('\n')+1:] # y,page=page[:page.find('\n')],page[page.find('\n')+1:] # z,page=page[:page.find('\n')],page[page.find('\n')+1:] # results.append((w+': '+v,y+': '+z)) demographics[w]=v if len(page)<=0: loop=False return demographics,url
def fetchDistrictDemoData(district): if district.find('u') > 0: # senate district district = district[-2:] else: # house district district = district[-3:].upper() url = mnleg_district_demo_info + district + '.txt' page = get_contents_of_url(url) demographics = {} if page: loop = True while loop: w, page = page[:page.find('\n')], page[page.find('\n') + 1:] v, page = page[:page.find('\n')], page[page.find('\n') + 1:] # y,page=page[:page.find('\n')],page[page.find('\n')+1:] # z,page=page[:page.find('\n')],page[page.find('\n')+1:] # results.append((w+': '+v,y+': '+z)) demographics[w] = v if len(page) <= 0: loop = False return demographics, url
def get(self): params = self.check_login('/bills') if 'loggedin_user' not in params: self.redirect('/signup') else: keyword = self.request.get("k") leg = self.request.get("l") if keyword: sort = self.request.get("s") s = getSortValue(sort) params['keyword'] = keyword params['bills'] = getBillsbyKeyword(keyword, s) self.render(bills_search_results_page, **params) elif leg: params['goodstring'] = 'no' legs = getAllLegislators() for l in legs: if leg == l.name: leg = l.leg_id sort = self.request.get("s") s = getSortValue(sort) params['goodstring'] = 'yes' params['author'] = leg params['bills'] = getBillsbyAuthor(params['author'], sort=s) params['author_data'] = getLegislatorByID(leg) break if params['goodstring'] == 'no': params['string'] = leg self.render(bills_search_results_page, **params) else: page = get_contents_of_url(aws_output + 'bills/front') if page: self.write(page) else: params["sessions"], params["details"] = getSessionNames() params['legislators'] = getAllLegislators() params['search_page'] = "True" self.render(bills_search_page, **params)
def get(self): params=self.check_login('/bills') if 'loggedin_user' not in params: self.redirect('/signup') else: keyword=self.request.get("k") leg=self.request.get("l") if keyword: sort=self.request.get("s") s=getSortValue(sort) params['keyword']=keyword params['bills']=getBillsbyKeyword(keyword,s) self.render(bills_search_results_page, **params) elif leg: params['goodstring']='no' legs=getAllLegislators() for l in legs: if leg==l.name: leg=l.leg_id sort=self.request.get("s") s=getSortValue(sort) params['goodstring']='yes' params['author']=leg params['bills']=getBillsbyAuthor(params['author'],sort=s) params['author_data']=getLegislatorByID(leg) break if params['goodstring']=='no': params['string']=leg self.render(bills_search_results_page, **params) else: page=get_contents_of_url(aws_output+'bills/front') if page: self.write(page) else: params["sessions"],params["details"]=getSessionNames() params['legislators']=getAllLegislators() params['search_page']="True" self.render(bills_search_page, **params)
def getSenateCommitteeMembers(url): response=get_contents_of_url(url) if response!=None: soup=BeautifulSoup(response) title = soup.find('div','leg_PageContent').h2.text title = title[:title.find('Membership')-1] info = soup.find_all('div','leg_Col3of4-First HRDFlyer') meetings = soup.find('div','leg_Col1of4-Last HRDFlyer') items=info[0].find_all('td') results=[] for i in items: t = [text for text in i.stripped_strings] if t: if t[0].find(':')>0: m={'name': t[1][:t[1].find(' (')], 'role': t[0][:-1], 'leg_id': getLegislatorIDByName(t[1][:t[1].find(' (')]),} else: m={'name': t[0][:t[0].find(' (')], 'role': 'member', 'leg_id': getLegislatorIDByName(t[0][:t[0].find(' (')]),} results.append(m) return title, results, meetings