Exemplo n.º 1
0
def _import_bill(lbill, session, previous_session=None):
    #lbill should be an ElementTree Element for the Bill tag

    if previous_session is None:
        previous_session = _get_previous_session(session)

    lbillnumber = lbill.xpath('BillNumber')[0]
    billnumber = (lbillnumber.get('prefix') + '-' + lbillnumber.get('number') +
                  lbillnumber.get('suffix', ''))
    try:
        bill = Bill.objects.get(number=billnumber, sessions=session)
        bis = bill.billinsession_set.get(session=session)
    except Bill.DoesNotExist:
        bill = Bill(number=billnumber)
        bis = BillInSession(bill=bill, session=session)
        bill._changed = True
        bis._changed = True
        bill.set_temporary_session(session)

    _update(bill, 'name_en',
            lbill.xpath('BillTitle/Title[@language="en"]')[0].text)

    if not bill.status_code:
        # This is presumably our first import of the bill; check if this
        # looks like a reintroduced bill and we want to merge with an
        # older Bill object.
        bill._newbill = True
        try:
            if previous_session:
                mergebill = Bill.objects.get(sessions=previous_session,
                                             number=bill.number,
                                             name_en__iexact=bill.name_en)

                if bill.id:
                    # If the new bill has already been saved, let's not try
                    # to merge automatically
                    logger.error("Bill %s may need to be merged. IDs: %s %s" %
                                 (bill.number, bill.id, mergebill.id))
                else:
                    logger.warning("Merging bill %s" % bill.number)
                    bill = mergebill
                    bis.bill = bill
        except Bill.DoesNotExist:
            # Nothing to merge
            pass

    _update(bill, 'name_fr',
            lbill.xpath('BillTitle/Title[@language="fr"]')[0].text)
    _update(bill, 'short_title_en',
            lbill.xpath('ShortTitle/Title[@language="en"]')[0].text)
    _update(bill, 'short_title_fr',
            lbill.xpath('ShortTitle/Title[@language="fr"]')[0].text)

    if not bis.sponsor_politician and bill.number[0] == 'C' and lbill.xpath(
            'SponsorAffiliation/@id'):
        # We don't deal with Senate sponsors yet
        pol_id = int(lbill.xpath('SponsorAffiliation/@id')[0])
        try:
            bis.sponsor_politician = Politician.objects.get_by_parl_id(pol_id)
        except Politician.DoesNotExist:
            logger.error(
                "Couldn't find sponsor politician for bill %s, pol ID %s" %
                (bill.number, pol_id))
        bis._changed = True
        try:
            bis.sponsor_member = ElectedMember.objects.get_by_pol(
                politician=bis.sponsor_politician, session=session)
        except Exception:
            logger.error("Couldn't find ElectedMember for bill %s, pol %r" %
                         (bill.number, bis.sponsor_politician))
        if not bill.sponsor_politician:
            bill.sponsor_politician = bis.sponsor_politician
            bill.sponsor_member = bis.sponsor_member
            bill._changed = True

    _update(bis, 'introduced',
            _parse_date(lbill.xpath('BillIntroducedDate')[0].text))
    if not bill.introduced:
        bill.introduced = bis.introduced

    try:
        status_code = lbill.xpath('Events')[0].get('laagCurrentStage')
        if status_code == '':
            status_code = 'Introduced'
        _update(bill, 'status_code', status_code)
        if status_code not in Bill.STATUS_CODES:
            logger.error("Unknown bill status code %s" % status_code)
        #_update(bill, 'status_date', _parse_date(
        #    lbill.xpath('Events/LastMajorStageEvent/Event/@date')[0]))
        status_dates = [
            _parse_date(d)
            for d in lbill.xpath('Events/LegislativeEvents/Event/@date')
        ]
        _update(bill, 'status_date', max(status_dates))
    except IndexError:
        # Some older bills don't have status information
        pass

    try:
        _update(bill, 'text_docid',
                int(lbill.xpath('Publications/Publication/@id')[-1]))
    except IndexError:
        pass

    _update(bis, 'legisinfo_id', int(lbill.get('id')))

    if getattr(bill, '_changed', False):
        bill.save()
    if getattr(bis, '_changed', False):
        bis.bill = bis.bill  # bizarrely, the django orm makes you do this
        bis.save()

    for levent in lbill.xpath('Events/LegislativeEvents/Event'):
        source_id = int(levent.get('id'))
        if BillEvent.objects.filter(source_id=source_id).exists():
            continue

        event = BillEvent(
            source_id=source_id,
            bis=bis,
            date=_parse_date(levent.get('date')),
            institution='S' if levent.get('chamber') == 'SEN' else 'C',
            status_en=levent.xpath('Status/Title[@language="en"]/text()')[0],
            status_fr=levent.xpath('Status/Title[@language="fr"]/text()')[0])

        if event.institution == 'C':
            hansard_num = levent.get('meetingNumber')
            try:
                event.debate = Document.debates.get(session=bis.session,
                                                    number=hansard_num)
            except Document.DoesNotExist:
                logger.info(
                    u"Could not associate BillEvent for %s with Hansard#%s" %
                    (bill, hansard_num))
                continue

            for lcommittee in levent.xpath('Committee'):
                acronym = lcommittee.get('accronym')
                if acronym and acronym != 'WHOL':
                    event.save()
                    try:
                        committee = Committee.objects.get_by_acronym(
                            acronym, bis.session)
                        for number in lcommittee.xpath(
                                'CommitteeMeetings/CommitteeMeeting/@number'):
                            event.committee_meetings.add(
                                CommitteeMeeting.objects.get(
                                    committee=committee,
                                    number=int(number),
                                    session=bis.session))
                    except ObjectDoesNotExist:
                        logger.exception(
                            "Could not import committee meetings: %s" %
                            etree.tostring(lcommittee))
                        continue
        event.save()

    if getattr(bill, '_newbill', False) and not session.end:
        bill.save_sponsor_activity()

    if bill.text_docid and not BillText.objects.filter(
            docid=bill.text_docid).exists():
        try:
            BillText.objects.create(bill=bill,
                                    docid=bill.text_docid,
                                    text_en=get_plain_bill_text(bill))
            bill.save()  # to trigger search indexing
        except CannotScrapeException:
            logger.warning(u"Could not get bill text for %s" % bill)

    return bill
Exemplo n.º 2
0
def _import_bill(lbill, session, previous_session=None):
    #lbill should be an ElementTree Element for the Bill tag

    if previous_session is None:
        previous_session = _get_previous_session(session)

    lbillnumber = lbill.xpath('BillNumber')[0]
    billnumber = (lbillnumber.get('prefix') + '-' + lbillnumber.get('number')
        + lbillnumber.get('suffix', ''))
    try:
        bill = Bill.objects.get(number=billnumber, sessions=session)
        bis = bill.billinsession_set.get(session=session)
    except Bill.DoesNotExist:
        bill = Bill(number=billnumber)
        bis = BillInSession(bill=bill, session=session)
        bill._changed = True
        bis._changed = True
        bill.set_temporary_session(session)

    _update(bill, 'name', lbill.xpath('BillTitle/Title[@language="en"]')[0].text)

    if not bill.status:
        # This is presumably our first import of the bill; check if this
        # looks like a reintroduced bill and we want to merge with an
        # older Bill object.
        bill._newbill = True
        try:
            if previous_session:
                mergebill = Bill.objects.get(sessions=previous_session,
                                             number=bill.number,
                                             name__iexact=bill.name)

                if bill.id:
                    # If the new bill has already been saved, let's not try
                    # to merge automatically
                    logger.error("Bill %s may need to be merged. IDs: %s %s" %
                                 (bill.number, bill.id, mergebill.id))
                else:
                    logger.warning("Merging bill %s" % bill.number)
                    bill = mergebill
                    bis.bill = bill
        except Bill.DoesNotExist:
            # Nothing to merge
            pass

    _update(bill, 'name_fr', lbill.xpath('BillTitle/Title[@language="fr"]')[0].text)
    _update(bill, 'short_title_en', lbill.xpath('ShortTitle/Title[@language="en"]')[0].text)
    _update(bill, 'short_title_fr', lbill.xpath('ShortTitle/Title[@language="fr"]')[0].text)

    if not bis.sponsor_politician and bill.number[0] == 'C' and lbill.xpath('SponsorAffiliation/@id'):
        # We don't deal with Senate sponsors yet
        pol_id = int(lbill.xpath('SponsorAffiliation/@id')[0])
        try:
            bis.sponsor_politician = Politician.objects.get_by_parl_id(pol_id)
        except Politician.DoesNotExist:
            logger.error("Couldn't find sponsor politician for bill %s, pol ID %s" % (bill.number, pol_id))
        bis._changed = True
        try:
            bis.sponsor_member = ElectedMember.objects.get_by_pol(politician=bis.sponsor_politician,
                                                                   session=session)
        except Exception:
            logger.error("Couldn't find ElectedMember for bill %s, pol %r" %
                         (bill.number, bis.sponsor_politician))
        if not bill.sponsor_politician:
            bill.sponsor_politician = bis.sponsor_politician
            bill.sponsor_member = bis.sponsor_member
            bill._changed = True

    _update(bis, 'introduced', _parse_date(lbill.xpath('BillIntroducedDate')[0].text))
    if not bill.introduced:
        bill.introduced = bis.introduced

    try:
        _update(bill, 'status',
            lbill.xpath('Events/LastMajorStageEvent/Event/Status/Title[@language="en"]')[0].text)
        _update(bill, 'status_fr',
            lbill.xpath('Events/LastMajorStageEvent/Event/Status/Title[@language="fr"]')[0].text)
        _update(bill, 'status_date', _parse_date(
            lbill.xpath('Events/LastMajorStageEvent/Event/@date')[0]))
    except IndexError:
        # Some older bills don't have status information
        pass

    try:
        _update(bill, 'text_docid', int(
            lbill.xpath('Publications/Publication/@id')[-1]))
    except IndexError:
        pass

    _update(bis, 'legisinfo_id', int(lbill.get('id')))

    if getattr(bill, '_changed', False):
        bill.save()
    if getattr(bis, '_changed', False):
        bis.bill = bis.bill # bizarrely, the django orm makes you do this
        bis.save()

    if getattr(bill, '_newbill', False) and not session.end:
        bill.save_sponsor_activity()

    if bill.text_docid and not BillText.objects.filter(docid=bill.text_docid).exists():
        try:
            BillText.objects.create(
                bill=bill,
                docid=bill.text_docid,
                text_en=get_plain_bill_text(bill)
            )
            bill.save()  # to trigger search indexing
        except CannotScrapeException:
            logger.warning(u"Could not get bill text for %s" % bill)

    return bill
            
Exemplo n.º 3
0
def _import_bill(lbill, session, previous_session=None):
    #lbill should be an ElementTree Element for the Bill tag

    if previous_session is None:
        previous_session = _get_previous_session(session)

    lbillnumber = lbill.xpath('BillNumber')[0]
    billnumber = (lbillnumber.get('prefix') + '-' + lbillnumber.get('number')
        + lbillnumber.get('suffix', ''))
    try:
        bill = Bill.objects.get(number=billnumber, sessions=session)
        bis = bill.billinsession_set.get(session=session)
    except Bill.DoesNotExist:
        bill = Bill(number=billnumber)
        bis = BillInSession(bill=bill, session=session)
        bill._changed = True
        bis._changed = True
        bill.set_temporary_session(session)

    _update(bill, 'name_en', lbill.xpath('BillTitle/Title[@language="en"]')[0].text)

    if not bill.status_code:
        # This is presumably our first import of the bill; check if this
        # looks like a reintroduced bill and we want to merge with an
        # older Bill object.
        bill._newbill = True
        try:
            if previous_session:
                mergebill = Bill.objects.get(sessions=previous_session,
                                             number=bill.number,
                                             name_en__iexact=bill.name_en)

                if bill.id:
                    # If the new bill has already been saved, let's not try
                    # to merge automatically
                    logger.error("Bill %s may need to be merged. IDs: %s %s" %
                                 (bill.number, bill.id, mergebill.id))
                else:
                    logger.warning("Merging bill %s" % bill.number)
                    bill = mergebill
                    bis.bill = bill
        except Bill.DoesNotExist:
            # Nothing to merge
            pass

    _update(bill, 'name_fr', lbill.xpath('BillTitle/Title[@language="fr"]')[0].text)
    _update(bill, 'short_title_en', lbill.xpath('ShortTitle/Title[@language="en"]')[0].text)
    _update(bill, 'short_title_fr', lbill.xpath('ShortTitle/Title[@language="fr"]')[0].text)

    if not bis.sponsor_politician and bill.number[0] == 'C' and lbill.xpath('SponsorAffiliation/@id'):
        # We don't deal with Senate sponsors yet
        pol_id = int(lbill.xpath('SponsorAffiliation/@id')[0])
        try:
            bis.sponsor_politician = Politician.objects.get_by_parl_id(pol_id)
        except Politician.DoesNotExist:
            logger.error("Couldn't find sponsor politician for bill %s, pol ID %s" % (bill.number, pol_id))
        bis._changed = True
        try:
            bis.sponsor_member = ElectedMember.objects.get_by_pol(politician=bis.sponsor_politician,
                                                                   session=session)
        except Exception:
            logger.error("Couldn't find ElectedMember for bill %s, pol %r" %
                         (bill.number, bis.sponsor_politician))
        if not bill.sponsor_politician:
            bill.sponsor_politician = bis.sponsor_politician
            bill.sponsor_member = bis.sponsor_member
            bill._changed = True

    _update(bis, 'introduced', _parse_date(lbill.xpath('BillIntroducedDate')[0].text))
    if not bill.introduced:
        bill.introduced = bis.introduced

    try:
        status_code = lbill.xpath('Events')[0].get('laagCurrentStage')
        if status_code == '':
            status_code = 'Introduced'
        _update(bill, 'status_code', status_code)
        if status_code not in Bill.STATUS_CODES:
            logger.error("Unknown bill status code %s" % status_code)
        #_update(bill, 'status_date', _parse_date(
        #    lbill.xpath('Events/LastMajorStageEvent/Event/@date')[0]))
        status_dates = [_parse_date(d) for d in lbill.xpath('Events/LegislativeEvents/Event/@date')]
        _update(bill, 'status_date', max(status_dates))
    except IndexError:
        # Some older bills don't have status information
        pass

    try:
        _update(bill, 'text_docid', int(
            lbill.xpath('Publications/Publication/@id')[-1]))
    except IndexError:
        pass

    _update(bis, 'legisinfo_id', int(lbill.get('id')))

    if getattr(bill, '_changed', False):
        bill.save()
    if getattr(bis, '_changed', False):
        bis.bill = bis.bill # bizarrely, the django orm makes you do this
        bis.save()        

    for levent in lbill.xpath('Events/LegislativeEvents/Event'):
        source_id = int(levent.get('id'))
        if BillEvent.objects.filter(source_id=source_id).exists():
            continue

        try:
            status_en = levent.xpath('Status/Title[@language="en"]/text()')[0]
            status_fr = levent.xpath('Status/Title[@language="fr"]/text()')[0]
        except IndexError:
            logger.debug("No status present in billevent: %s", etree.tostring(levent))
            continue

        event = BillEvent(
            source_id=source_id,
            bis=bis,
            date=_parse_date(levent.get('date')),
            institution='S' if levent.get('chamber') == 'SEN' else 'C',
            status_en=status_en,
            status_fr=status_fr
        )

        if event.institution == 'C':
            hansard_num = levent.get('meetingNumber')
            try:
                event.debate = Document.debates.get(session=bis.session, number=hansard_num)
            except Document.DoesNotExist:
                logger.info(u"Could not associate BillEvent for %s with Hansard#%s" % (bill, hansard_num))
                continue

            for lcommittee in levent.xpath('Committee'):
                acronym = lcommittee.get('accronym')
                if acronym and acronym != 'WHOL':
                    event.save()
                    try:
                        committee = Committee.objects.get_by_acronym(acronym, bis.session)
                        for number in lcommittee.xpath('CommitteeMeetings/CommitteeMeeting/@number'):
                            event.committee_meetings.add(
                                CommitteeMeeting.objects.get(committee=committee, number=int(number), session=bis.session)
                            )
                    except ObjectDoesNotExist:
                        logger.exception("Could not import committee meetings: %s" % etree.tostring(lcommittee))
                        continue
        event.save()

    if getattr(bill, '_newbill', False) and not session.end:
        bill.save_sponsor_activity()

    if bill.text_docid and not BillText.objects.filter(docid=bill.text_docid).exists():
        try:
            BillText.objects.create(
                bill=bill,
                docid=bill.text_docid,
                text_en=get_plain_bill_text(bill)
            )
            bill.save()  # to trigger search indexing
        except CannotScrapeException:
            logger.warning(u"Could not get bill text for %s" % bill)

    return bill