def import_votes(session=None):
if session is None:
session = Session.objects.current()
votelisturl = VOTELIST_URL % {'parliamentnum' : session.parliamentnum, 'sessnum': session.sessnum}
votelistpage = urllib2.urlopen(votelisturl)
tree = etree.parse(votelistpage)
root = tree.getroot()
votelist = root.findall('Vote')
votelist.reverse() # We want to process earlier votes first, just for the order they show up in the activity feed
for vote in votelist:
votenumber = int(vote.attrib['number'])
if VoteQuestion.objects.filter(session=session, number=votenumber).count():
continue
print "Processing vote #%s" % votenumber
votequestion = VoteQuestion(
number=votenumber,
session=session,
date=datetime.date(*(int(x) for x in vote.attrib['date'].split('-'))),
yea_total=int(vote.find('TotalYeas').text),
nay_total=int(vote.find('TotalNays').text),
paired_total=int(vote.find('TotalPaired').text))
if sum((votequestion.yea_total, votequestion.nay_total)) < 100:
logger.error("Fewer than 100 votes on vote#%s" % votenumber)
decision = vote.find('Decision').text
if decision in ('Agreed to', 'Agreed To'):
votequestion.result = 'Y'
elif decision == 'Negatived':
votequestion.result = 'N'
elif decision == 'Tie':
votequestion.result = 'T'
else:
raise Exception("Couldn't process vote result %s in %s" % (decision, votelisturl))
if vote.find('RelatedBill') is not None:
billnumber = vote.find('RelatedBill').attrib['number']
try:
votequestion.bill = Bill.objects.get(sessions=session, number=billnumber)
except Bill.DoesNotExist:
votequestion.bill = Bill.objects.create_temporary_bill(session=session, number=billnumber)
logger.warning("Temporary bill %s created for vote %s" % (billnumber, votenumber))
# Now get the detailed results
def get_detail(lang):
votedetailurl = VOTEDETAIL_URL % {
'lang': lang,
'parliamentnum' : session.parliamentnum,
'sessnum': session.sessnum,
'votenum': votenumber
}
votedetailpage = urllib2.urlopen(votedetailurl)
detailtree = etree.parse(votedetailpage)
detailroot = detailtree.getroot()
return (detailroot, parsetools.etree_extract_text(detailroot.find('Context')).strip())
try:
detailroot_fr, votequestion.description_fr = get_detail('F')
detailroot, votequestion.description_en = get_detail('E')
except Exception as e:
logger.exception("Import error on vote #%s" % votenumber)
continue
# Okay, save the question, start processing members.
votequestion.save()
for voter in detailroot.findall('Participant'):
name = voter.find('FirstName').text + ' ' + voter.find('LastName').text
riding = Riding.objects.get_by_name(voter.find('Constituency').text)
pol = Politician.objects.get_by_name(name=name, session=session, riding=riding)
member = ElectedMember.objects.get_by_pol(politician=pol, date=votequestion.date)
rvote = voter.find('RecordedVote')
if rvote.find('Yea').text == '1':
ballot = 'Y'
elif rvote.find('Nay').text == '1':
ballot = 'N'
elif rvote.find('Paired').text == '1':
ballot = 'P'
else:
raise Exception("Couldn't parse RecordedVote for %s in vote %s" % (name, votenumber))
MemberVote(member=member, politician=pol, votequestion=votequestion, vote=ballot).save()
votequestion.label_absent_members()
votequestion.label_party_votes()
for mv in votequestion.membervote_set.all():
mv.save_activity()
return True
def import_votes(session=None):
if session is None:
session = Session.objects.current()
votelisturl = VOTELIST_URL % {
'parliamentnum': session.parliamentnum,
'sessnum': session.sessnum
}
votelistpage = urllib2.urlopen(votelisturl)
tree = etree.parse(votelistpage)
root = tree.getroot()
votelist = root.findall('Vote')
votelist.reverse(
) # We want to process earlier votes first, just for the order they show up in the activity feed
for vote in votelist:
votenumber = int(vote.attrib['number'])
if VoteQuestion.objects.filter(session=session,
number=votenumber).count():
continue
print "Processing vote #%s" % votenumber
votequestion = VoteQuestion(
number=votenumber,
session=session,
date=datetime.date(*(int(x)
for x in vote.attrib['date'].split('-'))),
yea_total=int(vote.find('TotalYeas').text),
nay_total=int(vote.find('TotalNays').text),
paired_total=int(vote.find('TotalPaired').text))
if sum((votequestion.yea_total, votequestion.nay_total)) < 100:
logger.error("Fewer than 100 votes on vote#%s" % votenumber)
decision = vote.find('Decision').text
if decision in ('Agreed to', 'Agreed To'):
votequestion.result = 'Y'
elif decision == 'Negatived':
votequestion.result = 'N'
elif decision == 'Tie':
votequestion.result = 'T'
else:
raise Exception("Couldn't process vote result %s in %s" %
(decision, votelisturl))
if vote.find('RelatedBill') is not None:
billnumber = vote.find('RelatedBill').attrib['number']
try:
votequestion.bill = Bill.objects.get(sessions=session,
number=billnumber)
except Bill.DoesNotExist:
votequestion.bill = Bill.objects.create_temporary_bill(
session=session, number=billnumber)
logger.warning("Temporary bill %s created for vote %s" %
(billnumber, votenumber))
# Now get the detailed results
def get_detail(lang):
votedetailurl = VOTEDETAIL_URL % {
'lang': lang,
'parliamentnum': session.parliamentnum,
'sessnum': session.sessnum,
'votenum': votenumber
}
votedetailpage = urllib2.urlopen(votedetailurl)
detailtree = etree.parse(votedetailpage)
detailroot = detailtree.getroot()
return (detailroot,
parsetools.etree_extract_text(
detailroot.find('Context')).strip())
try:
detailroot_fr, votequestion.description_fr = get_detail('F')
detailroot, votequestion.description_en = get_detail('E')
except Exception as e:
logger.exception("Import error on vote #%s" % votenumber)
continue
# Okay, save the question, start processing members.
votequestion.save()
for voter in detailroot.findall('Participant'):
name = voter.find('FirstName').text + ' ' + voter.find(
'LastName').text
riding = Riding.objects.get_by_name(
voter.find('Constituency').text)
pol = Politician.objects.get_by_name(name=name,
session=session,
riding=riding)
member = ElectedMember.objects.get_by_pol(politician=pol,
date=votequestion.date)
rvote = voter.find('RecordedVote')
if rvote.find('Yea').text == '1':
ballot = 'Y'
elif rvote.find('Nay').text == '1':
ballot = 'N'
elif rvote.find('Paired').text == '1':
ballot = 'P'
else:
raise Exception(
"Couldn't parse RecordedVote for %s in vote %s" %
(name, votenumber))
MemberVote(member=member,
politician=pol,
votequestion=votequestion,
vote=ballot).save()
votequestion.label_absent_members()
votequestion.label_party_votes()
for mv in votequestion.membervote_set.all():
mv.save_activity()
return True
def import_votes(session=None):
if session is None:
session = Session.objects.current()
votelisturl = VOTELIST_URL % {'parliamentnum' : session.parliamentnum, 'sessnum': session.sessnum}
votelistpage = urllib2.urlopen(votelisturl)
tree = etree.parse(votelistpage)
root = tree.getroot()
votelist = root.findall('Vote')
votelist.reverse() # We want to process earlier votes first, just for the order they show up in the activity feed
for vote in votelist:
votenumber = int(vote.attrib['number'])
if VoteQuestion.objects.filter(session=session, number=votenumber).count():
continue
print "Processing vote #%s" % votenumber
votequestion = VoteQuestion(
number=votenumber,
session=session,
date=datetime.date(*(int(x) for x in vote.attrib['date'].split('-'))),
yea_total=int(vote.find('TotalYeas').text),
nay_total=int(vote.find('TotalNays').text),
paired_total=int(vote.find('TotalPaired').text))
decision = vote.find('Decision').text
if decision == 'Agreed to':
votequestion.result = 'Y'
elif decision == 'Negatived':
votequestion.result = 'N'
elif decision == 'Tie':
votequestion.result = 'T'
else:
raise Exception("Couldn't process vote result %s in %s" % (decision, votelisturl))
if vote.find('RelatedBill') is not None:
try:
votequestion.bill = Bill.objects.get(sessions=session, number=vote.find('RelatedBill').attrib['number'])
except Bill.DoesNotExist:
print "ERROR: Could not find bill for vote %s" % votenumber
# Now get the detailed results
votedetailurl = VOTEDETAIL_URL % {'parliamentnum' : session.parliamentnum,
'sessnum': session.sessnum,
'votenum': votenumber }
try:
votedetailpage = urllib2.urlopen(votedetailurl)
detailtree = etree.parse(votedetailpage)
except Exception, e:
print "ERROR on %s" % votedetailurl
print e
continue
detailroot = detailtree.getroot()
votequestion.description = parsetools.etree_extract_text(detailroot.find('Context')).strip()
# Okay, save the question, start processing members.
votequestion.save()
for voter in detailroot.findall('Participant'):
name = voter.find('FirstName').text + ' ' + voter.find('LastName').text
riding = Riding.objects.getByName(voter.find('Constituency').text)
pol = Politician.objects.getByName(name=name, session=session, riding=riding)
member = ElectedMember.objects.get_by_pol(politician=pol, date=votequestion.date)
rvote = voter.find('RecordedVote')
if rvote.find('Yea').text == '1':
ballot = 'Y'
elif rvote.find('Nay').text == '1':
ballot = 'N'
elif rvote.find('Paired').text == '1':
ballot = 'P'
else:
raise Exception("Couldn't parse RecordedVote for %s in vote %s" % (name, votenumber))
MemberVote(member=member, politician=pol, votequestion=votequestion, vote=ballot).save()
votequestion.label_absent_members()
def import_votes(session=None):
if session is None:
session = Session.objects.current()
elif session != Session.objects.current():
raise Exception(
"FIXME only current session supported in VOTELIST_URL for now")
votelisturl_en = VOTELIST_URL.format(lang='en')
resp = requests.get(votelisturl_en)
resp.raise_for_status()
root = etree.fromstring(resp.content)
votelisturl_fr = VOTELIST_URL.format(lang='fr')
resp = requests.get(votelisturl_fr)
resp.raise_for_status()
root_fr = etree.fromstring(resp.content)
votelist = root.findall('VoteParticipant')
for vote in votelist:
votenumber = int(vote.findtext('DecisionDivisionNumber'))
if VoteQuestion.objects.filter(session=session,
number=votenumber).count():
continue
print "Processing vote #%s" % votenumber
date = vote.findtext('DecisionEventDateTime')
date = datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S').date()
votequestion = VoteQuestion(
number=votenumber,
session=session,
date=date,
yea_total=int(vote.findtext('DecisionDivisionNumberOfYeas')),
nay_total=int(vote.findtext('DecisionDivisionNumberOfNays')),
paired_total=int(vote.findtext('DecisionDivisionNumberOfPaired')))
if sum((votequestion.yea_total, votequestion.nay_total)) < 100:
logger.error("Fewer than 100 votes on vote#%s" % votenumber)
decision = vote.findtext('DecisionResultName')
if decision in ('Agreed to', 'Agreed To'):
votequestion.result = 'Y'
elif decision == 'Negatived':
votequestion.result = 'N'
elif decision == 'Tie':
votequestion.result = 'T'
else:
raise Exception("Couldn't process vote result %s in %s" %
(decision, votelisturl))
if vote.findtext('BillNumberCode'):
billnumber = vote.findtext('BillNumberCode')
try:
votequestion.bill = Bill.objects.get(sessions=session,
number=billnumber)
except Bill.DoesNotExist:
votequestion.bill = Bill.objects.create_temporary_bill(
session=session, number=billnumber)
logger.warning("Temporary bill %s created for vote %s" %
(billnumber, votenumber))
votequestion.description_en = vote.findtext('DecisionDivisionSubject')
try:
votequestion.description_fr = root_fr.xpath(
'VoteParticipant/DecisionDivisionNumber[text()=%s]/../DecisionDivisionSubject/text()'
% votenumber)[0]
except Exception:
logger.exception("Couldn't get french description for vote %s" %
votenumber)
# Okay, save the question, start processing members.
votequestion.save()
detailurl = VOTEDETAIL_URL.format(parliamentnum=session.parliamentnum,
sessnum=session.sessnum,
votenumber=votenumber)
resp = requests.get(detailurl)
resp.raise_for_status()
detailroot = etree.fromstring(resp.content)
for voter in detailroot.findall('VoteParticipant'):
name = voter.find('FirstName').text + ' ' + voter.find(
'LastName').text
riding = Riding.objects.get_by_name(
voter.find('ConstituencyName').text)
pol = Politician.objects.get_by_name(name=name,
session=session,
riding=riding)
member = ElectedMember.objects.get_by_pol(politician=pol,
date=votequestion.date)
if voter.find('Yea').text == '1':
ballot = 'Y'
elif voter.find('Nay').text == '1':
ballot = 'N'
elif voter.find('Paired').text == '1':
ballot = 'P'
else:
raise Exception(
"Couldn't parse RecordedVote for %s in vote %s" %
(name, votenumber))
MemberVote(member=member,
politician=pol,
votequestion=votequestion,
vote=ballot).save()
votequestion.label_absent_members()
votequestion.label_party_votes()
for mv in votequestion.membervote_set.all():
mv.save_activity()
return True
def import_votes(session=None):
if session is None:
session = Session.objects.current()
elif session != Session.objects.current():
raise Exception("FIXME only current session supported in VOTELIST_URL for now")
votelisturl_en = VOTELIST_URL.format(lang='en')
resp = requests.get(votelisturl_en)
resp.raise_for_status()
root = etree.fromstring(resp.content)
votelisturl_fr = VOTELIST_URL.format(lang='fr')
resp = requests.get(votelisturl_fr)
resp.raise_for_status()
root_fr = etree.fromstring(resp.content)
votelist = root.findall('VoteParticipant')
for vote in votelist:
votenumber = int(vote.findtext('DecisionDivisionNumber'))
if VoteQuestion.objects.filter(session=session, number=votenumber).count():
continue
print "Processing vote #%s" % votenumber
date = vote.findtext('DecisionEventDateTime')
date = datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S').date()
votequestion = VoteQuestion(
number=votenumber,
session=session,
date=date,
yea_total=int(vote.findtext('DecisionDivisionNumberOfYeas')),
nay_total=int(vote.findtext('DecisionDivisionNumberOfNays')),
paired_total=int(vote.findtext('DecisionDivisionNumberOfPaired')))
if sum((votequestion.yea_total, votequestion.nay_total)) < 100:
logger.error("Fewer than 100 votes on vote#%s" % votenumber)
decision = vote.findtext('DecisionResultName')
if decision in ('Agreed to', 'Agreed To'):
votequestion.result = 'Y'
elif decision == 'Negatived':
votequestion.result = 'N'
elif decision == 'Tie':
votequestion.result = 'T'
else:
raise Exception("Couldn't process vote result %s in %s" % (decision, votelisturl))
if vote.findtext('BillNumberCode'):
billnumber = vote.findtext('BillNumberCode')
try:
votequestion.bill = Bill.objects.get(sessions=session, number=billnumber)
except Bill.DoesNotExist:
votequestion.bill = Bill.objects.create_temporary_bill(session=session, number=billnumber)
logger.warning("Temporary bill %s created for vote %s" % (billnumber, votenumber))
votequestion.description_en = vote.findtext('DecisionDivisionSubject')
try:
votequestion.description_fr = root_fr.xpath(
'VoteParticipant/DecisionDivisionNumber[text()=%s]/../DecisionDivisionSubject/text()'
% votenumber)[0]
except Exception:
logger.exception("Couldn't get french description for vote %s" % votenumber)
# Okay, save the question, start processing members.
votequestion.save()
detailurl = VOTEDETAIL_URL.format(parliamentnum=session.parliamentnum,
sessnum=session.sessnum, votenumber=votenumber)
resp = requests.get(detailurl)
resp.raise_for_status()
detailroot = etree.fromstring(resp.content)
for voter in detailroot.findall('VoteParticipant'):
name = voter.find('FirstName').text + ' ' + voter.find('LastName').text
riding = Riding.objects.get_by_name(voter.find('ConstituencyName').text)
pol = Politician.objects.get_by_name(name=name, session=session, riding=riding)
member = ElectedMember.objects.get_by_pol(politician=pol, date=votequestion.date)
if voter.find('Yea').text == '1':
ballot = 'Y'
elif voter.find('Nay').text == '1':
ballot = 'N'
elif voter.find('Paired').text == '1':
ballot = 'P'
else:
raise Exception("Couldn't parse RecordedVote for %s in vote %s" % (name, votenumber))
MemberVote(member=member, politician=pol, votequestion=votequestion, vote=ballot).save()
votequestion.label_absent_members()
votequestion.label_party_votes()
for mv in votequestion.membervote_set.all():
mv.save_activity()
return True