コード例 #1
0
def import_votes(session=None):
    if session is None:
        session = Session.objects.current()
    votelisturl = VOTELIST_URL % {'parliamentnum' : session.parliamentnum, 'sessnum': session.sessnum}
    votelistpage = urllib2.urlopen(votelisturl)
    tree = etree.parse(votelistpage)
    root = tree.getroot()
    votelist = root.findall('Vote')
    votelist.reverse() # We want to process earlier votes first, just for the order they show up in the activity feed
    for vote in votelist:
        votenumber = int(vote.attrib['number'])
        if VoteQuestion.objects.filter(session=session, number=votenumber).count():
            continue
        print "Processing vote #%s" % votenumber
        votequestion = VoteQuestion(
            number=votenumber,
            session=session,
            date=datetime.date(*(int(x) for x in vote.attrib['date'].split('-'))),
            yea_total=int(vote.find('TotalYeas').text),
            nay_total=int(vote.find('TotalNays').text),
            paired_total=int(vote.find('TotalPaired').text))
        if sum((votequestion.yea_total, votequestion.nay_total)) < 100:
            logger.error("Fewer than 100 votes on vote#%s" % votenumber)
        decision = vote.find('Decision').text
        if decision in ('Agreed to', 'Agreed To'):
            votequestion.result = 'Y'
        elif decision == 'Negatived':
            votequestion.result = 'N'
        elif decision == 'Tie':
            votequestion.result = 'T'
        else:
            raise Exception("Couldn't process vote result %s in %s" % (decision, votelisturl))
        if vote.find('RelatedBill') is not None:
            billnumber = vote.find('RelatedBill').attrib['number']
            try:
                votequestion.bill = Bill.objects.get(sessions=session, number=billnumber)
            except Bill.DoesNotExist:
                votequestion.bill = Bill.objects.create_temporary_bill(session=session, number=billnumber)
                logger.warning("Temporary bill %s created for vote %s" % (billnumber, votenumber))

        # Now get the detailed results
        def get_detail(lang):
            votedetailurl = VOTEDETAIL_URL % {
                    'lang': lang,
                    'parliamentnum' : session.parliamentnum,
                    'sessnum': session.sessnum,
                    'votenum': votenumber 
            }
            votedetailpage = urllib2.urlopen(votedetailurl)
            detailtree = etree.parse(votedetailpage)
            detailroot = detailtree.getroot()
            return (detailroot, parsetools.etree_extract_text(detailroot.find('Context')).strip())
        try:
            detailroot_fr, votequestion.description_fr = get_detail('F')
            detailroot, votequestion.description_en = get_detail('E')
        except Exception as e:
            logger.exception("Import error on vote #%s" % votenumber)
            continue
        
        # Okay, save the question, start processing members.
        votequestion.save()
        for voter in detailroot.findall('Participant'):
            name = voter.find('FirstName').text + ' ' + voter.find('LastName').text
            riding = Riding.objects.get_by_name(voter.find('Constituency').text)
            pol = Politician.objects.get_by_name(name=name, session=session, riding=riding)
            member = ElectedMember.objects.get_by_pol(politician=pol, date=votequestion.date)
            rvote = voter.find('RecordedVote')
            if rvote.find('Yea').text == '1':
                ballot = 'Y'
            elif rvote.find('Nay').text == '1':
                ballot = 'N'
            elif rvote.find('Paired').text == '1':
                ballot = 'P'
            else:
                raise Exception("Couldn't parse RecordedVote for %s in vote %s" % (name, votenumber))
            MemberVote(member=member, politician=pol, votequestion=votequestion, vote=ballot).save()
        votequestion.label_absent_members()
        votequestion.label_party_votes()
        for mv in votequestion.membervote_set.all():
            mv.save_activity()
    return True
コード例 #2
0
def import_votes(session=None):
    if session is None:
        session = Session.objects.current()
    votelisturl = VOTELIST_URL % {
        'parliamentnum': session.parliamentnum,
        'sessnum': session.sessnum
    }
    votelistpage = urllib2.urlopen(votelisturl)
    tree = etree.parse(votelistpage)
    root = tree.getroot()
    votelist = root.findall('Vote')
    votelist.reverse(
    )  # We want to process earlier votes first, just for the order they show up in the activity feed
    for vote in votelist:
        votenumber = int(vote.attrib['number'])
        if VoteQuestion.objects.filter(session=session,
                                       number=votenumber).count():
            continue
        print "Processing vote #%s" % votenumber
        votequestion = VoteQuestion(
            number=votenumber,
            session=session,
            date=datetime.date(*(int(x)
                                 for x in vote.attrib['date'].split('-'))),
            yea_total=int(vote.find('TotalYeas').text),
            nay_total=int(vote.find('TotalNays').text),
            paired_total=int(vote.find('TotalPaired').text))
        if sum((votequestion.yea_total, votequestion.nay_total)) < 100:
            logger.error("Fewer than 100 votes on vote#%s" % votenumber)
        decision = vote.find('Decision').text
        if decision in ('Agreed to', 'Agreed To'):
            votequestion.result = 'Y'
        elif decision == 'Negatived':
            votequestion.result = 'N'
        elif decision == 'Tie':
            votequestion.result = 'T'
        else:
            raise Exception("Couldn't process vote result %s in %s" %
                            (decision, votelisturl))
        if vote.find('RelatedBill') is not None:
            billnumber = vote.find('RelatedBill').attrib['number']
            try:
                votequestion.bill = Bill.objects.get(sessions=session,
                                                     number=billnumber)
            except Bill.DoesNotExist:
                votequestion.bill = Bill.objects.create_temporary_bill(
                    session=session, number=billnumber)
                logger.warning("Temporary bill %s created for vote %s" %
                               (billnumber, votenumber))

        # Now get the detailed results
        def get_detail(lang):
            votedetailurl = VOTEDETAIL_URL % {
                'lang': lang,
                'parliamentnum': session.parliamentnum,
                'sessnum': session.sessnum,
                'votenum': votenumber
            }
            votedetailpage = urllib2.urlopen(votedetailurl)
            detailtree = etree.parse(votedetailpage)
            detailroot = detailtree.getroot()
            return (detailroot,
                    parsetools.etree_extract_text(
                        detailroot.find('Context')).strip())

        try:
            detailroot_fr, votequestion.description_fr = get_detail('F')
            detailroot, votequestion.description_en = get_detail('E')
        except Exception as e:
            logger.exception("Import error on vote #%s" % votenumber)
            continue

        # Okay, save the question, start processing members.
        votequestion.save()
        for voter in detailroot.findall('Participant'):
            name = voter.find('FirstName').text + ' ' + voter.find(
                'LastName').text
            riding = Riding.objects.get_by_name(
                voter.find('Constituency').text)
            pol = Politician.objects.get_by_name(name=name,
                                                 session=session,
                                                 riding=riding)
            member = ElectedMember.objects.get_by_pol(politician=pol,
                                                      date=votequestion.date)
            rvote = voter.find('RecordedVote')
            if rvote.find('Yea').text == '1':
                ballot = 'Y'
            elif rvote.find('Nay').text == '1':
                ballot = 'N'
            elif rvote.find('Paired').text == '1':
                ballot = 'P'
            else:
                raise Exception(
                    "Couldn't parse RecordedVote for %s in vote %s" %
                    (name, votenumber))
            MemberVote(member=member,
                       politician=pol,
                       votequestion=votequestion,
                       vote=ballot).save()
        votequestion.label_absent_members()
        votequestion.label_party_votes()
        for mv in votequestion.membervote_set.all():
            mv.save_activity()
    return True
コード例 #3
0
ファイル: parlvotes.py プロジェクト: xster/openparliament
def import_votes(session=None):
    if session is None:
        session = Session.objects.current()
    votelisturl = VOTELIST_URL % {'parliamentnum' : session.parliamentnum, 'sessnum': session.sessnum}
    votelistpage = urllib2.urlopen(votelisturl)
    tree = etree.parse(votelistpage)
    root = tree.getroot()
    votelist = root.findall('Vote')
    votelist.reverse() # We want to process earlier votes first, just for the order they show up in the activity feed
    for vote in votelist:
        votenumber = int(vote.attrib['number'])
        if VoteQuestion.objects.filter(session=session, number=votenumber).count():
            continue
        print "Processing vote #%s" % votenumber
        votequestion = VoteQuestion(
            number=votenumber,
            session=session,
            date=datetime.date(*(int(x) for x in vote.attrib['date'].split('-'))),
            yea_total=int(vote.find('TotalYeas').text),
            nay_total=int(vote.find('TotalNays').text),
            paired_total=int(vote.find('TotalPaired').text))
        decision = vote.find('Decision').text
        if decision == 'Agreed to':
            votequestion.result = 'Y'
        elif decision == 'Negatived':
            votequestion.result = 'N'
        elif decision == 'Tie':
            votequestion.result = 'T'
        else:
            raise Exception("Couldn't process vote result %s in %s" % (decision, votelisturl))
        if vote.find('RelatedBill') is not None:
            try:
                votequestion.bill = Bill.objects.get(sessions=session, number=vote.find('RelatedBill').attrib['number'])
            except Bill.DoesNotExist:
                print "ERROR: Could not find bill for vote %s" % votenumber
        
        # Now get the detailed results
        votedetailurl = VOTEDETAIL_URL % {'parliamentnum' : session.parliamentnum,
                'sessnum': session.sessnum,
                'votenum': votenumber }
        try:
            votedetailpage = urllib2.urlopen(votedetailurl)
            detailtree = etree.parse(votedetailpage)
        except Exception, e:
            print "ERROR on %s" % votedetailurl
            print e
            continue
        detailroot = detailtree.getroot()
        votequestion.description = parsetools.etree_extract_text(detailroot.find('Context')).strip()

        
        # Okay, save the question, start processing members.
        votequestion.save()
        for voter in detailroot.findall('Participant'):
            name = voter.find('FirstName').text + ' ' + voter.find('LastName').text
            riding = Riding.objects.getByName(voter.find('Constituency').text)
            pol = Politician.objects.getByName(name=name, session=session, riding=riding)
            member = ElectedMember.objects.get_by_pol(politician=pol, date=votequestion.date)
            rvote = voter.find('RecordedVote')
            if rvote.find('Yea').text == '1':
                ballot = 'Y'
            elif rvote.find('Nay').text == '1':
                ballot = 'N'
            elif rvote.find('Paired').text == '1':
                ballot = 'P'
            else:
                raise Exception("Couldn't parse RecordedVote for %s in vote %s" % (name, votenumber))
            MemberVote(member=member, politician=pol, votequestion=votequestion, vote=ballot).save()
        votequestion.label_absent_members()
コード例 #4
0
def import_votes(session=None):
    if session is None:
        session = Session.objects.current()
    elif session != Session.objects.current():
        raise Exception(
            "FIXME only current session supported in VOTELIST_URL for now")

    votelisturl_en = VOTELIST_URL.format(lang='en')
    resp = requests.get(votelisturl_en)
    resp.raise_for_status()
    root = etree.fromstring(resp.content)

    votelisturl_fr = VOTELIST_URL.format(lang='fr')
    resp = requests.get(votelisturl_fr)
    resp.raise_for_status()
    root_fr = etree.fromstring(resp.content)

    votelist = root.findall('VoteParticipant')
    for vote in votelist:
        votenumber = int(vote.findtext('DecisionDivisionNumber'))
        if VoteQuestion.objects.filter(session=session,
                                       number=votenumber).count():
            continue
        print "Processing vote #%s" % votenumber
        date = vote.findtext('DecisionEventDateTime')
        date = datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S').date()
        votequestion = VoteQuestion(
            number=votenumber,
            session=session,
            date=date,
            yea_total=int(vote.findtext('DecisionDivisionNumberOfYeas')),
            nay_total=int(vote.findtext('DecisionDivisionNumberOfNays')),
            paired_total=int(vote.findtext('DecisionDivisionNumberOfPaired')))
        if sum((votequestion.yea_total, votequestion.nay_total)) < 100:
            logger.error("Fewer than 100 votes on vote#%s" % votenumber)
        decision = vote.findtext('DecisionResultName')
        if decision in ('Agreed to', 'Agreed To'):
            votequestion.result = 'Y'
        elif decision == 'Negatived':
            votequestion.result = 'N'
        elif decision == 'Tie':
            votequestion.result = 'T'
        else:
            raise Exception("Couldn't process vote result %s in %s" %
                            (decision, votelisturl))
        if vote.findtext('BillNumberCode'):
            billnumber = vote.findtext('BillNumberCode')
            try:
                votequestion.bill = Bill.objects.get(sessions=session,
                                                     number=billnumber)
            except Bill.DoesNotExist:
                votequestion.bill = Bill.objects.create_temporary_bill(
                    session=session, number=billnumber)
                logger.warning("Temporary bill %s created for vote %s" %
                               (billnumber, votenumber))

        votequestion.description_en = vote.findtext('DecisionDivisionSubject')
        try:
            votequestion.description_fr = root_fr.xpath(
                'VoteParticipant/DecisionDivisionNumber[text()=%s]/../DecisionDivisionSubject/text()'
                % votenumber)[0]
        except Exception:
            logger.exception("Couldn't get french description for vote %s" %
                             votenumber)

        # Okay, save the question, start processing members.
        votequestion.save()

        detailurl = VOTEDETAIL_URL.format(parliamentnum=session.parliamentnum,
                                          sessnum=session.sessnum,
                                          votenumber=votenumber)
        resp = requests.get(detailurl)
        resp.raise_for_status()
        detailroot = etree.fromstring(resp.content)

        for voter in detailroot.findall('VoteParticipant'):
            name = voter.find('FirstName').text + ' ' + voter.find(
                'LastName').text
            riding = Riding.objects.get_by_name(
                voter.find('ConstituencyName').text)
            pol = Politician.objects.get_by_name(name=name,
                                                 session=session,
                                                 riding=riding)
            member = ElectedMember.objects.get_by_pol(politician=pol,
                                                      date=votequestion.date)
            if voter.find('Yea').text == '1':
                ballot = 'Y'
            elif voter.find('Nay').text == '1':
                ballot = 'N'
            elif voter.find('Paired').text == '1':
                ballot = 'P'
            else:
                raise Exception(
                    "Couldn't parse RecordedVote for %s in vote %s" %
                    (name, votenumber))
            MemberVote(member=member,
                       politician=pol,
                       votequestion=votequestion,
                       vote=ballot).save()
        votequestion.label_absent_members()
        votequestion.label_party_votes()
        for mv in votequestion.membervote_set.all():
            mv.save_activity()
    return True
コード例 #5
0
def import_votes(session=None):
    if session is None:
        session = Session.objects.current()
    elif session != Session.objects.current():
        raise Exception("FIXME only current session supported in VOTELIST_URL for now")
    
    votelisturl_en = VOTELIST_URL.format(lang='en')
    resp = requests.get(votelisturl_en)
    resp.raise_for_status()
    root = etree.fromstring(resp.content)

    votelisturl_fr = VOTELIST_URL.format(lang='fr')
    resp = requests.get(votelisturl_fr)
    resp.raise_for_status()
    root_fr = etree.fromstring(resp.content)

    votelist = root.findall('VoteParticipant')
    for vote in votelist:
        votenumber = int(vote.findtext('DecisionDivisionNumber'))
        if VoteQuestion.objects.filter(session=session, number=votenumber).count():
            continue
        print "Processing vote #%s" % votenumber
        date = vote.findtext('DecisionEventDateTime')
        date = datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S').date()
        votequestion = VoteQuestion(
            number=votenumber,
            session=session,
            date=date,
            yea_total=int(vote.findtext('DecisionDivisionNumberOfYeas')),
            nay_total=int(vote.findtext('DecisionDivisionNumberOfNays')),
            paired_total=int(vote.findtext('DecisionDivisionNumberOfPaired')))
        if sum((votequestion.yea_total, votequestion.nay_total)) < 100:
            logger.error("Fewer than 100 votes on vote#%s" % votenumber)
        decision = vote.findtext('DecisionResultName')
        if decision in ('Agreed to', 'Agreed To'):
            votequestion.result = 'Y'
        elif decision == 'Negatived':
            votequestion.result = 'N'
        elif decision == 'Tie':
            votequestion.result = 'T'
        else:
            raise Exception("Couldn't process vote result %s in %s" % (decision, votelisturl))
        if vote.findtext('BillNumberCode'):
            billnumber = vote.findtext('BillNumberCode')
            try:
                votequestion.bill = Bill.objects.get(sessions=session, number=billnumber)
            except Bill.DoesNotExist:
                votequestion.bill = Bill.objects.create_temporary_bill(session=session, number=billnumber)
                logger.warning("Temporary bill %s created for vote %s" % (billnumber, votenumber))

        votequestion.description_en = vote.findtext('DecisionDivisionSubject')
        try:
            votequestion.description_fr = root_fr.xpath(
                'VoteParticipant/DecisionDivisionNumber[text()=%s]/../DecisionDivisionSubject/text()'
                % votenumber)[0]
        except Exception:
            logger.exception("Couldn't get french description for vote %s" % votenumber)

        # Okay, save the question, start processing members.
        votequestion.save()

        detailurl = VOTEDETAIL_URL.format(parliamentnum=session.parliamentnum,
            sessnum=session.sessnum, votenumber=votenumber)
        resp = requests.get(detailurl)
        resp.raise_for_status()
        detailroot = etree.fromstring(resp.content)

        for voter in detailroot.findall('VoteParticipant'):
            name = voter.find('FirstName').text + ' ' + voter.find('LastName').text
            riding = Riding.objects.get_by_name(voter.find('ConstituencyName').text)
            pol = Politician.objects.get_by_name(name=name, session=session, riding=riding)
            member = ElectedMember.objects.get_by_pol(politician=pol, date=votequestion.date)
            if voter.find('Yea').text == '1':
                ballot = 'Y'
            elif voter.find('Nay').text == '1':
                ballot = 'N'
            elif voter.find('Paired').text == '1':
                ballot = 'P'
            else:
                raise Exception("Couldn't parse RecordedVote for %s in vote %s" % (name, votenumber))
            MemberVote(member=member, politician=pol, votequestion=votequestion, vote=ballot).save()
        votequestion.label_absent_members()
        votequestion.label_party_votes()
        for mv in votequestion.membervote_set.all():
            mv.save_activity()
    return True