class Party(models.Model): """A federal political party.""" name_en = models.CharField(max_length=100) name_fr = models.CharField(max_length=100, blank=True) short_name_en = models.CharField(max_length=100, blank=True) short_name_fr = models.CharField(max_length=100, blank=True) slug = models.CharField(max_length=10, blank=True) name = language_property('name') short_name = language_property('short_name') objects = PartyManager() class Meta: verbose_name_plural = 'Parties' def __init__(self, *args, **kwargs): # If we're creating a new object, set a flag to save the name to the alternate-names table. super(Party, self).__init__(*args, **kwargs) self._saveAlternate = True def save(self): if not self.name_fr: self.name_fr = self.name_en if not self.short_name_en: self.short_name_en = self.name_en if not self.short_name_fr: self.short_name_fr = self.name_fr super(Party, self).save() if getattr(self, '_saveAlternate', False): self.add_alternate_name(self.name_en) self.add_alternate_name(self.name_fr) def delete(self): InternalXref.objects.filter(schema='party_names', target_id=self.id).delete() super(Party, self).delete() def add_alternate_name(self, name): name = name.strip().lower() # check if exists x = InternalXref.objects.filter(schema='party_names', text_value=name) if len(x) == 0: InternalXref(schema='party_names', target_id=self.id, text_value=name).save() else: if x[0].target_id != self.id: raise Exception("Name %s already points to a different party" % name.strip().lower()) def __unicode__(self): return self.name
class BillText(models.Model): bill = models.ForeignKey(Bill) docid = models.PositiveIntegerField(unique=True, db_index=True) created = models.DateTimeField(default=datetime.datetime.now) text_en = models.TextField() text_fr = models.TextField(blank=True) text = language_property('text') def __unicode__(self): return u"Document #%d for %s" % (self.docid, self.bill) @property def summary(self): match = re.search(r'SUMMARY\n([\s\S]+?)(Also a|A)vailable on', self.text_en) return match.group(1).strip() if match else None @property def summary_html(self): summary = self.summary if not summary: return '' return mark_safe('<p>' + summary.replace('\n', '</p><p>') + '</p>')
class BillEvent(models.Model): bis = models.ForeignKey(BillInSession) date = models.DateField(db_index=True) source_id = models.PositiveIntegerField(unique=True, db_index=True) institution = models.CharField(max_length=1, choices=Bill.CHAMBERS) status_en = models.TextField() status_fr = models.TextField(blank=True) debate = models.ForeignKey('hansards.Document', blank=True, null=True, on_delete=models.SET_NULL) committee_meetings = models.ManyToManyField('committees.CommitteeMeeting', blank=True) status = language_property('status') def __unicode__(self): return u"%s: %s, %s" % (self.status, self.bis.bill.number, self.date) @property def bill_number(self): return self.bis.bill.number
class Riding(models.Model): "A federal riding." name_en = models.CharField(max_length=200) name_fr = models.CharField(blank=True, max_length=200) province = models.CharField(max_length=2, choices=PROVINCE_CHOICES) slug = models.CharField(max_length=60, unique=True, db_index=True) edid = models.IntegerField(blank=True, null=True, db_index=True) current = models.BooleanField(blank=True, default=False) objects = RidingManager() name = language_property('name') class Meta: ordering = ('province', 'name_en') def save(self): if not self.slug: self.slug = parsetools.slugify(self.name_en) super(Riding, self).save() @property def dashed_name(self): return self.name.replace('--', '—') def __unicode__(self): return "%s (%s)" % (self.dashed_name, self.get_province_display())
class CommitteeReport(models.Model): committee = models.ForeignKey(Committee) session = models.ForeignKey(Session) number = models.SmallIntegerField(blank=True, null=True) # watch this become a char name_en = models.CharField(max_length=500) name_fr = models.CharField(max_length=500, blank=True) source_id = models.IntegerField(unique=True, db_index=True) adopted_date = models.DateField(blank=True, null=True) presented_date = models.DateField(blank=True, null=True) government_response = models.BooleanField(default=False) parent = models.ForeignKey('self', null=True, blank=True, related_name='children') name = language_property('name') def __unicode__(self): return u"%s report #%s" % (self.committee, self.number)
class CommitteeActivity(models.Model): committee = models.ForeignKey(Committee) name_en = models.CharField(max_length=500) name_fr = models.CharField(max_length=500) study = models.BooleanField(default=False) # study or activity name = language_property('name') def __unicode__(self): return self.name @models.permalink def get_absolute_url(self): return ('committee_activity', [], {'activity_id': self.id}) def get_source_url(self): return self.committeeactivityinsession_set.order_by( '-session__start')[0].get_source_url() @property def type(self): return 'Study' if self.study else 'Activity' class Meta: verbose_name_plural = 'Committee activities'
class Bill(models.Model): CHAMBERS = ( ('C', 'House'), ('S', 'Senate'), ) STATUS_CODES = { u'BillNotActive': 'Not active', u'WillNotBeProceededWith': 'Dead', u'RoyalAssentAwaiting': 'Awaiting royal assent', u'BillDefeated': 'Defeated', u'HouseAtReportStage': 'Report stage (House)', u'RoyalAssentGiven': 'Law (royal assent given)', u'SenateAt1stReading': 'First reading (Senate)', u'HouseAt1stReading': 'First reading (House)', u'HouseAtReferralToCommitteeBeforeSecondReading': 'Referral to committee before 2nd reading (House)', u'HouseAt2ndReading': 'Second reading (House)', u'HouseAtReportStageAndSecondReading': 'Report stage and second reading (House)', u'SenateAt2ndReading': 'Second reading (Senate)', u'SenateAt3rdReading': 'Third reading (Senate)', u'HouseAt3rdReading': 'Third reading (House)', u'HouseInCommittee': 'In committee (House)', u'SenateInCommittee': 'In committee (Senate)', u'SenateConsiderationOfCommitteeReport': 'Considering committee report (Senate)', u'HouseConsiderationOfCommitteeReport': 'Considering committee report (House)', u'SenateConsiderationOfAmendments': 'Considering amendments (Senate)', u'HouseConsiderationOfAmendments': 'Considering amendments (House)', u'Introduced': 'Introduced' } name_en = models.TextField(blank=True) name_fr = models.TextField(blank=True) short_title_en = models.TextField(blank=True) short_title_fr = models.TextField(blank=True) number = models.CharField(max_length=10) number_only = models.SmallIntegerField() institution = models.CharField(max_length=1, db_index=True, choices=CHAMBERS) sessions = models.ManyToManyField(Session, through='BillInSession') privatemember = models.NullBooleanField() sponsor_member = models.ForeignKey(ElectedMember, blank=True, null=True) sponsor_politician = models.ForeignKey(Politician, blank=True, null=True) law = models.NullBooleanField() status_date = models.DateField(blank=True, null=True, db_index=True) status_code = models.CharField(max_length=50, blank=True) added = models.DateField(default=datetime.date.today, db_index=True) introduced = models.DateField(blank=True, null=True) text_docid = models.IntegerField( blank=True, null=True, help_text= "The parl.gc.ca document ID of the latest version of the bill's text") objects = BillManager() name = language_property('name') short_title = language_property('short_title') class Meta: ordering = ('privatemember', 'institution', 'number_only') def __unicode__(self): return "%s - %s" % (self.number, self.name) def get_absolute_url(self): return self.url_for_session(self.session) def url_for_session(self, session): return urlresolvers.reverse('bill', kwargs={ 'session_id': session.id, 'bill_number': self.number }) def get_legisinfo_url(self, lang='E'): return LEGISINFO_BILL_URL % { 'lang': lang, 'bill': self.number.replace('-', ''), 'parliament': self.session.parliamentnum, 'session': self.session.sessnum } legisinfo_url = property(get_legisinfo_url) def get_billtext_url(self, lang='E', single_page=False): if not self.text_docid: return None url = PARLIAMENT_DOCVIEWER_URL % { 'lang': lang, 'docid': self.text_docid } if single_page: url += '&File=4&Col=1' return url def get_text_object(self): if not self.text_docid: raise BillText.DoesNotExist return BillText.objects.get(bill=self, docid=self.text_docid) def get_text(self, language=settings.LANGUAGE_CODE): try: return getattr(self.get_text_object(), 'text_' + language) except BillText.DoesNotExist: return '' def get_summary(self): try: return self.get_text_object().summary_html except BillText.DoesNotExist: return '' def get_related_debates(self): return Document.objects.filter(billinsession__bill=self) def get_committee_meetings(self): return CommitteeMeeting.objects.filter(billevent__bis__bill=self) def get_major_speeches(self): doc_ids = list(self.get_related_debates().values_list('id', flat=True)) if self.short_title_en: qs = Statement.objects.filter(h2_en__iexact=self.short_title_en, wordcount__gt=50) else: qs = self.statement_set.filter(wordcount__gt=100) return qs.filter(document__in=doc_ids, procedural=False) @property def latest_date(self): return self.status_date if self.status_date else self.introduced def save(self, *args, **kwargs): if not self.number_only: self.number_only = int(re.sub(r'\D', '', self.number)) if getattr(self, 'privatemember', None) is None: self.privatemember = bool(self.number_only >= 200) if not self.institution: self.institution = self.number[0] if not self.law and self.status_code == 'RoyalAssentGiven': self.law = True super(Bill, self).save(*args, **kwargs) def save_sponsor_activity(self): if self.sponsor_politician: activity.save_activity( obj=self, politician=self.sponsor_politician, date=self.introduced if self.introduced else (self.added - datetime.timedelta(days=1)), variety='billsponsor', ) def get_session(self): """Returns the most recent session this bill belongs to.""" try: self.__dict__['session'] = s = self.sessions.all().order_by( '-start')[0] return s except (IndexError, ValueError): return getattr(self, '_session', None) def set_temporary_session(self, session): """To deal with tricky save logic, saves a session to the object for cases when self.sessions.all() won't get exist in the DB.""" self._session = session session = property(get_session) @property def status(self): return self.STATUS_CODES.get(self.status_code, 'Unknown') @property def dead(self): return self.status_code in ('BillNotActive', 'WillNotBeProceededWith', 'BillDefeated') @property def dormant(self): return (self.status_date and (datetime.date.today() - self.status_date).days > 150)
class VoteQuestion(models.Model): bill = models.ForeignKey(Bill, blank=True, null=True) session = models.ForeignKey(Session) number = models.PositiveIntegerField() date = models.DateField(db_index=True) description_en = models.TextField() description_fr = models.TextField(blank=True) result = models.CharField(max_length=1, choices=VOTE_RESULT_CHOICES) yea_total = models.SmallIntegerField() nay_total = models.SmallIntegerField() paired_total = models.SmallIntegerField() context_statement = models.ForeignKey('hansards.Statement', blank=True, null=True, on_delete=models.SET_NULL) description = language_property('description') def __unicode__(self): return u"Vote #%s on %s" % (self.number, self.date) class Meta: ordering = ('-date', '-number') def to_api_dict(self, representation): r = { 'bill_url': self.bill.get_absolute_url() if self.bill else None, 'session': self.session_id, 'number': self.number, 'date': unicode(self.date), 'description': { 'en': self.description_en, 'fr': self.description_fr }, 'result': self.get_result_display(), 'yea_total': self.yea_total, 'nay_total': self.nay_total, 'paired_total': self.paired_total, } if representation == 'detail': r.update( context_statement=self.context_statement.get_absolute_url() if self.context_statement else None, party_votes=[{ 'vote': pv.get_vote_display(), 'disagreement': pv.disagreement, 'party': { 'name': { 'en': pv.party.name }, 'short_name': { 'en': pv.party.short_name } }, } for pv in self.partyvote_set.all()]) return r def label_absent_members(self): for member in ElectedMember.objects.on_date( self.date).exclude(membervote__votequestion=self): MemberVote(votequestion=self, member=member, politician_id=member.politician_id, vote='A').save() def label_party_votes(self): """Create PartyVote objects representing the party-line vote; label individual dissenting votes.""" membervotes = self.membervote_set.select_related( 'member', 'member__party').all() parties = defaultdict(lambda: defaultdict(int)) for mv in membervotes: if mv.member.party.name != 'Independent': parties[mv.member.party][mv.vote] += 1 partyvotes = {} for party in parties: # Find the most common vote votes = sorted(parties[party].items(), key=lambda i: i[1]) partyvotes[party] = votes[-1][0] # Find how many people voted with the majority yn = (parties[party]['Y'], parties[party]['N']) try: disagreement = float(min(yn)) / sum(yn) except ZeroDivisionError: disagreement = 0.0 # If more than 15% of the party voted against the party majority, # label this as a free vote. if disagreement >= 0.15: partyvotes[party] = 'F' PartyVote.objects.filter(party=party, votequestion=self).delete() PartyVote.objects.create(party=party, votequestion=self, vote=partyvotes[party], disagreement=disagreement) for mv in membervotes: if mv.member.party.name != 'Independent' \ and mv.vote != partyvotes[mv.member.party] \ and mv.vote in ('Y', 'N') \ and partyvotes[mv.member.party] in ('Y', 'N'): mv.dissent = True mv.save() @models.permalink def get_absolute_url(self): return ('vote', [], { 'session_id': self.session_id, 'number': self.number })
class Committee(models.Model): name_en = models.TextField() short_name_en = models.TextField() name_fr = models.TextField(blank=True) short_name_fr = models.TextField(blank=True) slug = models.SlugField(unique=True) parent = models.ForeignKey('self', related_name='subcommittees', blank=True, null=True) sessions = models.ManyToManyField(Session, through='CommitteeInSession') joint = models.BooleanField('Joint committee?', default=False) display = models.BooleanField('Display on site?', db_index=True, default=True) objects = CommitteeManager() name = language_property('name') short_name = language_property('short_name') class Meta: ordering = ['name_en'] def __unicode__(self): return self.name def save(self, *args, **kwargs): if not self.short_name_en: self.short_name_en = self.name_en if not self.short_name_fr: self.short_name_fr = self.name_fr if not self.slug: self.slug = slugify(self.short_name_en, allow_numbers=True) if self.parent: self.slug = self.parent.slug + '-' + self.slug self.slug = self.slug[:46] while Committee.objects.filter(slug=self.slug).exists(): self.slug += '-' + random.choice(string.lowercase) super(Committee, self).save(*args, **kwargs) @models.permalink def get_absolute_url(self): return ('committee', [], {'slug': self.slug}) def get_source_url(self): return self.committeeinsession_set.order_by( '-session__start')[0].get_source_url() def get_acronym(self, session): return CommitteeInSession.objects.get(committee=self, session=session).acronym def latest_session(self): return self.sessions.order_by('-start')[0] @property def title(self): if 'committee' in self.name_en.lower(): return self.name else: return self.name + u' Committee' def to_api_dict(self, representation): d = dict( name={ 'en': self.name_en, 'fr': self.name_fr }, short_name={ 'en': self.short_name_en, 'fr': self.short_name_fr }, slug=self.slug, parent_url=self.parent.get_absolute_url() if self.parent else None, ) if representation == 'detail': d['sessions'] = [{ 'session': cis.session_id, 'acronym': cis.acronym, 'source_url': cis.get_source_url(), } for cis in self.committeeinsession_set.all().order_by( '-session__end').select_related('session')] d['subcommittees'] = [ c.get_absolute_url() for c in self.subcommittees.all() ] return d
class Statement(models.Model): document = models.ForeignKey(Document) time = models.DateTimeField(db_index=True) source_id = models.CharField(max_length=15, blank=True) slug = models.SlugField(max_length=100, blank=True) urlcache = models.CharField(max_length=200, blank=True) h1_en = models.CharField(max_length=300, blank=True) h2_en = models.CharField(max_length=300, blank=True) h3_en = models.CharField(max_length=300, blank=True) h1_fr = models.CharField(max_length=400, blank=True) h2_fr = models.CharField(max_length=400, blank=True) h3_fr = models.CharField(max_length=400, blank=True) member = models.ForeignKey(ElectedMember, blank=True, null=True) politician = models.ForeignKey(Politician, blank=True, null=True) # a shortcut -- should == member.politician who_en = models.CharField(max_length=300, blank=True) who_fr = models.CharField(max_length=500, blank=True) who_hocid = models.PositiveIntegerField(blank=True, null=True, db_index=True) who_context_en = models.CharField(max_length=300, blank=True) who_context_fr = models.CharField(max_length=500, blank=True) content_en = models.TextField() content_fr = models.TextField(blank=True) sequence = models.IntegerField(db_index=True) wordcount = models.IntegerField() wordcount_en = models.PositiveSmallIntegerField(null=True, help_text="# words originally spoken in English") procedural = models.BooleanField(default=False, db_index=True) written_question = models.CharField(max_length=1, blank=True, choices=( ('Q', 'Question'), ('R', 'Response') )) statement_type = models.CharField(max_length=35, blank=True) bills = models.ManyToManyField('bills.Bill', blank=True) mentioned_politicians = models.ManyToManyField(Politician, blank=True, related_name='statements_with_mentions') class Meta: ordering = ('sequence',) unique_together = ( ('document', 'slug') ) h1 = language_property('h1') h2 = language_property('h2') h3 = language_property('h3') who = language_property('who') who_context = language_property('who_context') def save(self, *args, **kwargs): self.content_en = self.content_en.replace('\n', '').replace('</p>', '</p>\n').strip() self.content_fr = self.content_fr.replace('\n', '').replace('</p>', '</p>\n').strip() if self.wordcount_en is None: self._generate_wordcounts() if ((not self.procedural) and self.wordcount <= 300 and ( (parsetools.r_notamember.search(self.who) and re.search(r'(Speaker|Chair|président)', self.who)) or (not self.who) or not any(p for p in self.content_en.split('\n') if 'class="procedural"' not in p) )): # Some form of routine, procedural statement (e.g. somethng short by the speaker) self.procedural = True if not self.urlcache: self.generate_url() super(Statement, self).save(*args, **kwargs) @property def date(self): return datetime.date(self.time.year, self.time.month, self.time.day) def generate_url(self): self.urlcache = "%s%s/" % ( self.document.get_absolute_url(), (self.slug if self.slug else self.sequence)) def get_absolute_url(self): if not self.urlcache: self.generate_url() return self.urlcache def __unicode__ (self): return u"%s speaking about %s around %s" % (self.who, self.topic, self.time) def content_floor(self): if not self.content_fr: return self.content_en el, fl = self.content_en.split('\n'), self.content_fr.split('\n') if len(el) != len(fl): logger.error("Different en/fr paragraphs in %s" % self.get_absolute_url()) return self.content_en r = [] for e, f in zip(el, fl): idx = e.find('data-originallang="') if idx and e[idx+19:idx+21] == 'fr': r.append(f) else: r.append(e) return u"\n".join(r) def content_floor_if_necessary(self): """Returns text spoken in the original language(s), but only if that would be different than the content in the default language.""" if not (self.content_en and self.content_fr): return '' lang_matches = re.finditer(r'data-originallang="(\w\w)"', getattr(self, 'content_' + settings.LANGUAGE_CODE)) if any(m.group(1) != settings.LANGUAGE_CODE for m in lang_matches): return self.content_floor() return '' def text_html(self, language=settings.LANGUAGE_CODE): return mark_safe(getattr(self, 'content_' + language)) def text_plain(self, language=settings.LANGUAGE_CODE): return self.html_to_text(getattr(self, 'content_' + language)) @staticmethod def html_to_text(text): return strip_tags( text .replace('\n', '') .replace('<br>', '\n') .replace('</p>', '\n\n') ).strip() def _generate_wordcounts(self): paragraphs = [ [], # english [], # french [] # procedural ] for para in self.content_en.split('\n'): idx = para.find('data-originallang="') if idx == -1: paragraphs[2].append(para) else: lang = para[idx+19:idx+21] if lang == 'en': paragraphs[0].append(para) elif lang == 'fr': paragraphs[1].append(para) else: paragraphs[0].append(para) logger.warning("Unrecognized language %s", lang) counts = [ len(self.html_to_text(' '.join(p)).split()) for p in paragraphs ] self.wordcount = counts[0] + counts[1] self.wordcount_en = counts[0] #self.wordcount_procedural = counts[2] # temp compatibility @property def heading(self): return self.h1 @property def topic(self): return self.h2 def to_api_dict(self, representation): d = dict( time=unicode(self.time) if self.time else None, attribution={'en': self.who_en, 'fr': self.who_fr}, content={'en': self.content_en, 'fr': self.content_fr}, url=self.get_absolute_url(), politician_url=self.politician.get_absolute_url() if self.politician else None, politician_membership_url=urlresolvers.reverse('politician_membership', kwargs={'member_id': self.member_id}) if self.member_id else None, procedural=self.procedural, source_id=self.source_id ) for h in ('h1', 'h2', 'h3'): if getattr(self, h): d[h] = {'en': getattr(self, h + '_en'), 'fr': getattr(self, h + '_fr')} d['document_url'] = d['url'][:d['url'].rstrip('/').rfind('/')+1] return d @property @memoize_property def name_info(self): info = { 'post': None, 'named': True } if not self.member: info['display_name'] = parsetools.r_mister.sub('', self.who) if self.who_context: if self.who_context in self.who: info['display_name'] = parsetools.r_parens.sub('', info['display_name']) info['post'] = self.who_context else: info['post_reminder'] = self.who_context if self.who_hocid: info['url'] = '/search/?q=Witness%%3A+%%22%s%%22' % self.who_hocid else: info['url'] = self.member.politician.get_absolute_url() if parsetools.r_notamember.search(self.who): info['display_name'] = self.who if self.member.politician.name in self.who: info['display_name'] = re.sub(r'\(.+\)', '', self.who) info['named'] = False elif not '(' in self.who or not parsetools.r_politicalpost.search(self.who): info['display_name'] = self.member.politician.name else: post_match = re.search(r'\((.+)\)', self.who) if post_match: info['post'] = post_match.group(1).split(',')[0] info['display_name'] = self.member.politician.name return info @staticmethod def set_slugs(statements): counter = defaultdict(int) for statement in statements: slug = slugify(statement.name_info['display_name'])[:50] if not slug: slug = 'procedural' counter[slug] += 1 statement.slug = slug + '-%s' % counter[slug] @property def committee_name(self): if self.document.document_type != Document.EVIDENCE: return '' return self.document.committeemeeting.committee.short_name @property def committee_slug(self): if self.document.document_type != Document.EVIDENCE: return '' return self.document.committeemeeting.committee.slug