def fetchHousehPVIfeed():
response = get_contents_of_url(house_hpvi_feed_url)
data = json.loads(response[62:-2])
hpvi = {}
for r in data['table']['rows']:
hpvi[r['c'][0]['v']] = r['c'][1]['v']
return hpvi
def fetchHousehPVIfeed():
response = get_contents_of_url(house_hpvi_feed_url)
data=json.loads(response[62:-2])
hpvi={}
for r in data['table']['rows']:
hpvi[r['c'][0]['v']]=r['c'][1]['v']
return hpvi
def getSenateCommittees():
response = get_contents_of_url(mn_senate_base + '/committees/')
if response != None:
soup = BeautifulSoup(response)
info = soup.find_all('div', 'HRDFlyer')
links = links = info[0].find_all('a')
name = ''
committees = []
count = 0
for l in links:
if l.text.find('Members') >= 0:
members = l['href']
elif l.text.find('Schedule') >= 0:
schedule = l['href']
elif l.text.find('Audio/Video') >= 0:
av = l['href']
committee = {
'committee': name,
'chamber': 'upper',
'id': getCommitteeIDFromURL(l['href']),
'members_url': members,
'meetings_url': schedule,
'media_url': av,
}
committees.append(committee)
else:
if l.text[1:].find(' ') > 0:
name = l.text[1:l.text[1:].find(' ')]
else:
name = l.text[1:]
return committees
else:
return None
def getSenateCommitteeMembers(url):
response = get_contents_of_url(url)
if response != None:
soup = BeautifulSoup(response)
title = soup.find('div', 'leg_PageContent').h2.text
title = title[:title.find('Membership') - 1]
info = soup.find_all('div', 'leg_Col3of4-First HRDFlyer')
meetings = soup.find('div', 'leg_Col1of4-Last HRDFlyer')
items = info[0].find_all('td')
results = []
for i in items:
t = [text for text in i.stripped_strings]
if t:
if t[0].find(':') > 0:
m = {
'name': t[1][:t[1].find(' (')],
'role': t[0][:-1],
'leg_id':
getLegislatorIDByName(t[1][:t[1].find(' (')]),
}
else:
m = {
'name': t[0][:t[0].find(' (')],
'role': 'member',
'leg_id':
getLegislatorIDByName(t[0][:t[0].find(' (')]),
}
results.append(m)
return title, results, meetings
def getSenateCommittees():
response=get_contents_of_url(mn_senate_base+'/committees/')
if response!=None:
soup=BeautifulSoup(response)
info = soup.find_all('div','HRDFlyer')
links = links=info[0].find_all('a')
name=''
committees=[]
count=0
for l in links:
if l.text.find('Members')>=0:
members=l['href']
elif l.text.find('Schedule')>=0:
schedule=l['href']
elif l.text.find('Audio/Video')>=0:
av=l['href']
committee={'committee':name,
'chamber':'upper',
'id': getCommitteeIDFromURL(l['href']),
'members_url':members,
'meetings_url':schedule,
'media_url':av,}
committees.append(committee)
else:
if l.text[1:].find(' ')>0:
name=l.text[1:l.text[1:].find(' ')]
else:
name=l.text[1:]
return committees
else:
return None
def fetchRawHousehPVIfeed(): response = get_contents_of_url(house_raw_hpvi_feed_url) data=json.loads(response[62:-2]) hpvi=[] for r in data['table']['rows']: hpvi.append(r['c'][0]['v']) return sorted(hpvi)
def fetchRawHousehPVIfeed():
response = get_contents_of_url(house_raw_hpvi_feed_url)
data = json.loads(response[62:-2])
hpvi = []
for r in data['table']['rows']:
hpvi.append(r['c'][0]['v'])
return sorted(hpvi)
def getSenateCommitteeMeetingsByID(com_id):
url=mn_senate_base+'/committees/committee_members.php?cmte_id='+com_id
response=get_contents_of_url(url)
if response!=None:
soup=BeautifulSoup(response)
meetings = soup.find('div','leg_Col1of4-Last HRDFlyer')
return meetings
return None
def getHouseCommitteeMeetings(url):
response=get_contents_of_url(url)
if response!=None:
soup=BeautifulSoup(response)
meeting_text = soup.find_all('div','leg_col1of3-Last')
if meeting_text[0]:
return meeting_text[0]
return None
def getSenateCommitteeMeetingsByID(com_id):
url = mn_senate_base + '/committees/committee_members.php?cmte_id=' + com_id
response = get_contents_of_url(url)
if response != None:
soup = BeautifulSoup(response)
meetings = soup.find('div', 'leg_Col1of4-Last HRDFlyer')
return meetings
return None
def sendGetRequest(url):
url = substitute_char(url, ' ', '%20')
response = get_contents_of_url(url)
if response:
data = json.loads(response)
return data
else:
return None
def getHouseCommitteeMeetings(url):
response = get_contents_of_url(url)
if response != None:
soup = BeautifulSoup(response)
meeting_text = soup.find_all('div', 'leg_col1of3-Last')
if meeting_text[0]:
return meeting_text[0]
return None
def sendGetRequest(url): url=substitute_char(url,' ','%20') response = get_contents_of_url(url) if response: data=json.loads(response) return data else: return None
def get_source(self, environment, template):
path = self.path+template
page=get_contents_of_url(path)
if not page:
raise jinja2.TemplateNotFound(template)
#mtime = os.path.getmtime(path)
#with file(path) as f:
source = page.decode('utf-8')
return source, path, lambda: False #lambda: mtime == os.path.getmtime(path)
def get_source(self, environment, template):
path = self.path + template
page = get_contents_of_url(path)
if not page:
raise jinja2.TemplateNotFound(template)
#mtime = os.path.getmtime(path)
#with file(path) as f:
source = page.decode('utf-8')
return source, path, lambda: False #lambda: mtime == os.path.getmtime(path)
def fetchElectionResults(url): response = get_contents_of_url(url) if response and response!=None: response = parseCSVfromURL(response,',') results=[] for r in response: results.append([r[1],r[2],r[3],r[4],r[5],r[6],r[7],r[8],r[9],r[10],r[11],r[12],r[13],r[14],r[15]]) return results else: return None
def fetchPrecinctTable(url): response = get_contents_of_url(url) if response and response!=None: response = parseCSVfromURL(response,';') results=[] for r in response: results.append([r[0],r[1],r[2],r[3],r[4],r[5],r[6],r[7],r[8],r[9]]) return results else: return None
def parseHouseCommitteeMeetings(url):
response=get_contents_of_url(url)
if response!=None:
soup=BeautifulSoup(response)
meeting_text = soup.find_all('div','leg_col1of3-Last')
if meeting_text:
if meeting_text[0].p.p:
m=meeting_text[0].p.p
return [text for text in m.stripped_strings]
return None
def parseHouseCommitteeMeetings(url):
response = get_contents_of_url(url)
if response != None:
soup = BeautifulSoup(response)
meeting_text = soup.find_all('div', 'leg_col1of3-Last')
if meeting_text:
if meeting_text[0].p.p:
m = meeting_text[0].p.p
return [text for text in m.stripped_strings]
return None
def getSenateCommitteeMeetingInfo(url):
results = []
response = get_contents_of_url(url)
if response != None:
soup = BeautifulSoup(response)
meetings = soup.find('div', 'leg_Col3of4')
meet = meetings.table
m = meet.find_all('table')
for i in m:
results.append([text for text in i.stripped_strings])
return results
def getSenateCommitteeMeetingInfo(url):
results=[]
response=get_contents_of_url(url)
if response!=None:
soup=BeautifulSoup(response)
meetings = soup.find('div','leg_Col3of4')
meet=meetings.table
m=meet.find_all('table')
for i in m:
results.append([text for text in i.stripped_strings])
return results
def fetchPrecinctTable(url):
response = get_contents_of_url(url)
if response and response != None:
response = parseCSVfromURL(response, ';')
results = []
for r in response:
results.append(
[r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9]])
return results
else:
return None
def get(self,chamber):
params=self.check_login('districts/'+chamber)
if 'loggedin_user' not in params:
self.redirect('/signup')
else:
page=get_contents_of_url(aws_output+'districts/'+chamber)
if page:
self.write(page)
else:
params['district_map']=get_chamber_name(chamber)
params['districts']=getAllDistrictsByID(params['district_map'])
self.render(all_districts_page, **params)
def fetchSenateElectionResults():
response=getFromCache('senate2012elections')
if not response:
response = get_contents_of_url(senate_2012_election_results)
if response and response!=None:
putInCache('senate2012elections',response)
else:
return None
response = parseCSVfromURL(response,';')
results=[]
for r in response:
results.append([r[5],r[4],r[12],r[15]," ".join([x.capitalize() for x in r[7].split(" ")]),r[10],r[14],r[13]])
return results
def get(self, chamber):
params = self.check_login('districts/' + chamber)
if 'loggedin_user' not in params:
self.redirect('/signup')
else:
page = get_contents_of_url(aws_output + 'districts/' + chamber)
if page:
self.write(page)
else:
params['district_map'] = get_chamber_name(chamber)
params['districts'] = getAllDistrictsByID(
params['district_map'])
self.render(all_districts_page, **params)
def fetchElectionResults(url):
response = get_contents_of_url(url)
if response and response != None:
response = parseCSVfromURL(response, ',')
results = []
for r in response:
results.append([
r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10],
r[11], r[12], r[13], r[14], r[15]
])
return results
else:
return None
def get(self):
params=self.check_login('committees')
chamber=self.request.get("q")
if chamber!="house" and chamber!="senate" and chamber:
self.redirect('/')
else:
params['chamber']='upper'
if self.request.get("q")=="house":
params['chamber']='lower'
page=get_contents_of_url(aws_output+'committees/'+params['chamber'])
if page:
self.write(page)
else:
params['committees']=getCommitteesByChamber(params['chamber'])
self.render(all_committees_page, **params)
def getBillText(url):
bill_page=get_contents_of_url(url)
clean_bill=substitute_char(bill_page,var_re,'')
clean_bill=substitute_char(clean_bill,markup_id_re,'')
soup=BeautifulSoup(clean_bill)
for e in soup.find_all('br'):
e.extract()
br = soup.new_tag('br')
bill_text = soup.find_all('div','xtend')
# for e in bill_text[0]('a'):
# bill_text[0].a.insert_before(br)
# first_link = bill_text[0].a
# first_link.find_next("a")
return bill_text[0]
def get(self):
params = self.check_login('committees')
chamber = self.request.get("q")
if chamber != "house" and chamber != "senate" and chamber:
self.redirect('/')
else:
params['chamber'] = 'upper'
if self.request.get("q") == "house":
params['chamber'] = 'lower'
page = get_contents_of_url(aws_output + 'committees/' +
params['chamber'])
if page:
self.write(page)
else:
params['committees'] = getCommitteesByChamber(
params['chamber'])
self.render(all_committees_page, **params)
def fetchHouseElectionResults():
response = getFromCache('house2012elections')
if not response:
response = get_contents_of_url(house_2012_election_results)
if response and response != None:
putInCache('house2012elections', response)
else:
return None
response = parseCSVfromURL(response, ';')
results = []
for r in response:
results.append([
r[5], r[4], r[12], r[15],
" ".join([x.capitalize() for x in r[7].split(" ")]), r[10], r[14],
r[13]
])
return results
def getBillText(url):
bill_page = get_contents_of_url(url)
clean_bill = substitute_char(bill_page, var_re, '')
clean_bill = substitute_char(clean_bill, markup_id_re, '')
soup = BeautifulSoup(clean_bill)
for e in soup.find_all('br'):
e.extract()
br = soup.new_tag('br')
bill_text = soup.find_all('div', 'xtend')
# for e in bill_text[0]('a'):
# bill_text[0].a.insert_before(br)
# first_link = bill_text[0].a
# first_link.find_next("a")
return bill_text[0]
def get(self):
params=self.check_login('legislators')
chamber=self.request.get("q")
if chamber!="house" and chamber!="senate" and chamber:
self.redirect('/')
else:
params['chamber']='upper'
params['chamber_name']="Senate"
if self.request.get("q")=="house":
params['chamber']='lower'
params['chamber_name']='House'
page=get_contents_of_url(aws_output+'legislators/'+params['chamber'])
if page:
self.write(page)
else:
params['legislators']=getLegislatorsByChamber(params['chamber'])
params['search_page']="True"
self.render(current_legislators_page, **params)
def get(self):
params = self.check_login('legislators')
chamber = self.request.get("q")
if chamber != "house" and chamber != "senate" and chamber:
self.redirect('/')
else:
params['chamber'] = 'upper'
params['chamber_name'] = "Senate"
if self.request.get("q") == "house":
params['chamber'] = 'lower'
params['chamber_name'] = 'House'
page = get_contents_of_url(aws_output + 'legislators/' +
params['chamber'])
if page:
self.write(page)
else:
params['legislators'] = getLegislatorsByChamber(
params['chamber'])
params['search_page'] = "True"
self.render(current_legislators_page, **params)
def fetchDistrictDemoData(district):
if district.find('u')>0: # senate district
district=district[-2:]
else: # house district
district=district[-3:].upper()
url=mnleg_district_demo_info+district+'.txt'
page = get_contents_of_url(url)
demographics={}
if page:
loop=True
while loop:
w,page=page[:page.find('\n')],page[page.find('\n')+1:]
v,page=page[:page.find('\n')],page[page.find('\n')+1:]
# y,page=page[:page.find('\n')],page[page.find('\n')+1:]
# z,page=page[:page.find('\n')],page[page.find('\n')+1:]
# results.append((w+': '+v,y+': '+z))
demographics[w]=v
if len(page)<=0:
loop=False
return demographics,url
def fetchDistrictDemoData(district):
if district.find('u') > 0: # senate district
district = district[-2:]
else: # house district
district = district[-3:].upper()
url = mnleg_district_demo_info + district + '.txt'
page = get_contents_of_url(url)
demographics = {}
if page:
loop = True
while loop:
w, page = page[:page.find('\n')], page[page.find('\n') + 1:]
v, page = page[:page.find('\n')], page[page.find('\n') + 1:]
# y,page=page[:page.find('\n')],page[page.find('\n')+1:]
# z,page=page[:page.find('\n')],page[page.find('\n')+1:]
# results.append((w+': '+v,y+': '+z))
demographics[w] = v
if len(page) <= 0:
loop = False
return demographics, url
def get(self):
params = self.check_login('/bills')
if 'loggedin_user' not in params:
self.redirect('/signup')
else:
keyword = self.request.get("k")
leg = self.request.get("l")
if keyword:
sort = self.request.get("s")
s = getSortValue(sort)
params['keyword'] = keyword
params['bills'] = getBillsbyKeyword(keyword, s)
self.render(bills_search_results_page, **params)
elif leg:
params['goodstring'] = 'no'
legs = getAllLegislators()
for l in legs:
if leg == l.name:
leg = l.leg_id
sort = self.request.get("s")
s = getSortValue(sort)
params['goodstring'] = 'yes'
params['author'] = leg
params['bills'] = getBillsbyAuthor(params['author'],
sort=s)
params['author_data'] = getLegislatorByID(leg)
break
if params['goodstring'] == 'no':
params['string'] = leg
self.render(bills_search_results_page, **params)
else:
page = get_contents_of_url(aws_output + 'bills/front')
if page:
self.write(page)
else:
params["sessions"], params["details"] = getSessionNames()
params['legislators'] = getAllLegislators()
params['search_page'] = "True"
self.render(bills_search_page, **params)
def get(self):
params=self.check_login('/bills')
if 'loggedin_user' not in params:
self.redirect('/signup')
else:
keyword=self.request.get("k")
leg=self.request.get("l")
if keyword:
sort=self.request.get("s")
s=getSortValue(sort)
params['keyword']=keyword
params['bills']=getBillsbyKeyword(keyword,s)
self.render(bills_search_results_page, **params)
elif leg:
params['goodstring']='no'
legs=getAllLegislators()
for l in legs:
if leg==l.name:
leg=l.leg_id
sort=self.request.get("s")
s=getSortValue(sort)
params['goodstring']='yes'
params['author']=leg
params['bills']=getBillsbyAuthor(params['author'],sort=s)
params['author_data']=getLegislatorByID(leg)
break
if params['goodstring']=='no':
params['string']=leg
self.render(bills_search_results_page, **params)
else:
page=get_contents_of_url(aws_output+'bills/front')
if page:
self.write(page)
else:
params["sessions"],params["details"]=getSessionNames()
params['legislators']=getAllLegislators()
params['search_page']="True"
self.render(bills_search_page, **params)
def getSenateCommitteeMembers(url):
response=get_contents_of_url(url)
if response!=None:
soup=BeautifulSoup(response)
title = soup.find('div','leg_PageContent').h2.text
title = title[:title.find('Membership')-1]
info = soup.find_all('div','leg_Col3of4-First HRDFlyer')
meetings = soup.find('div','leg_Col1of4-Last HRDFlyer')
items=info[0].find_all('td')
results=[]
for i in items:
t = [text for text in i.stripped_strings]
if t:
if t[0].find(':')>0:
m={'name': t[1][:t[1].find(' (')],
'role': t[0][:-1],
'leg_id': getLegislatorIDByName(t[1][:t[1].find(' (')]),}
else:
m={'name': t[0][:t[0].find(' (')],
'role': 'member',
'leg_id': getLegislatorIDByName(t[0][:t[0].find(' (')]),}
results.append(m)
return title, results, meetings
