class Party(models.Model):
"""A federal political party."""
name_en = models.CharField(max_length=100)
name_fr = models.CharField(max_length=100, blank=True)
short_name_en = models.CharField(max_length=100, blank=True)
short_name_fr = models.CharField(max_length=100, blank=True)
slug = models.CharField(max_length=10, blank=True)
name = language_property('name')
short_name = language_property('short_name')
objects = PartyManager()
class Meta:
verbose_name_plural = 'Parties'
def __init__(self, *args, **kwargs):
# If we're creating a new object, set a flag to save the name to the alternate-names table.
super(Party, self).__init__(*args, **kwargs)
self._saveAlternate = True
def save(self):
if not self.name_fr:
self.name_fr = self.name_en
if not self.short_name_en:
self.short_name_en = self.name_en
if not self.short_name_fr:
self.short_name_fr = self.name_fr
super(Party, self).save()
if getattr(self, '_saveAlternate', False):
self.add_alternate_name(self.name_en)
self.add_alternate_name(self.name_fr)
def delete(self):
InternalXref.objects.filter(schema='party_names',
target_id=self.id).delete()
super(Party, self).delete()
def add_alternate_name(self, name):
name = name.strip().lower()
# check if exists
x = InternalXref.objects.filter(schema='party_names', text_value=name)
if len(x) == 0:
InternalXref(schema='party_names',
target_id=self.id,
text_value=name).save()
else:
if x[0].target_id != self.id:
raise Exception("Name %s already points to a different party" %
name.strip().lower())
def __unicode__(self):
return self.name
class BillText(models.Model):
bill = models.ForeignKey(Bill)
docid = models.PositiveIntegerField(unique=True, db_index=True)
created = models.DateTimeField(default=datetime.datetime.now)
text_en = models.TextField()
text_fr = models.TextField(blank=True)
text = language_property('text')
def __unicode__(self):
return u"Document #%d for %s" % (self.docid, self.bill)
@property
def summary(self):
match = re.search(r'SUMMARY\n([\s\S]+?)(Also a|A)vailable on',
self.text_en)
return match.group(1).strip() if match else None
@property
def summary_html(self):
summary = self.summary
if not summary:
return ''
return mark_safe('<p>' + summary.replace('\n', '</p><p>') + '</p>')
class BillEvent(models.Model):
bis = models.ForeignKey(BillInSession)
date = models.DateField(db_index=True)
source_id = models.PositiveIntegerField(unique=True, db_index=True)
institution = models.CharField(max_length=1, choices=Bill.CHAMBERS)
status_en = models.TextField()
status_fr = models.TextField(blank=True)
debate = models.ForeignKey('hansards.Document',
blank=True,
null=True,
on_delete=models.SET_NULL)
committee_meetings = models.ManyToManyField('committees.CommitteeMeeting',
blank=True)
status = language_property('status')
def __unicode__(self):
return u"%s: %s, %s" % (self.status, self.bis.bill.number, self.date)
@property
def bill_number(self):
return self.bis.bill.number
class Riding(models.Model):
"A federal riding."
name_en = models.CharField(max_length=200)
name_fr = models.CharField(blank=True, max_length=200)
province = models.CharField(max_length=2, choices=PROVINCE_CHOICES)
slug = models.CharField(max_length=60, unique=True, db_index=True)
edid = models.IntegerField(blank=True, null=True, db_index=True)
current = models.BooleanField(blank=True, default=False)
objects = RidingManager()
name = language_property('name')
class Meta:
ordering = ('province', 'name_en')
def save(self):
if not self.slug:
self.slug = parsetools.slugify(self.name_en)
super(Riding, self).save()
@property
def dashed_name(self):
return self.name.replace('--', '—')
def __unicode__(self):
return "%s (%s)" % (self.dashed_name, self.get_province_display())
class CommitteeReport(models.Model):
committee = models.ForeignKey(Committee)
session = models.ForeignKey(Session)
number = models.SmallIntegerField(blank=True,
null=True) # watch this become a char
name_en = models.CharField(max_length=500)
name_fr = models.CharField(max_length=500, blank=True)
source_id = models.IntegerField(unique=True, db_index=True)
adopted_date = models.DateField(blank=True, null=True)
presented_date = models.DateField(blank=True, null=True)
government_response = models.BooleanField(default=False)
parent = models.ForeignKey('self',
null=True,
blank=True,
related_name='children')
name = language_property('name')
def __unicode__(self):
return u"%s report #%s" % (self.committee, self.number)
class CommitteeActivity(models.Model):
committee = models.ForeignKey(Committee)
name_en = models.CharField(max_length=500)
name_fr = models.CharField(max_length=500)
study = models.BooleanField(default=False) # study or activity
name = language_property('name')
def __unicode__(self):
return self.name
@models.permalink
def get_absolute_url(self):
return ('committee_activity', [], {'activity_id': self.id})
def get_source_url(self):
return self.committeeactivityinsession_set.order_by(
'-session__start')[0].get_source_url()
@property
def type(self):
return 'Study' if self.study else 'Activity'
class Meta:
verbose_name_plural = 'Committee activities'
class Bill(models.Model):
CHAMBERS = (
('C', 'House'),
('S', 'Senate'),
)
STATUS_CODES = {
u'BillNotActive': 'Not active',
u'WillNotBeProceededWith': 'Dead',
u'RoyalAssentAwaiting': 'Awaiting royal assent',
u'BillDefeated': 'Defeated',
u'HouseAtReportStage': 'Report stage (House)',
u'RoyalAssentGiven': 'Law (royal assent given)',
u'SenateAt1stReading': 'First reading (Senate)',
u'HouseAt1stReading': 'First reading (House)',
u'HouseAtReferralToCommitteeBeforeSecondReading':
'Referral to committee before 2nd reading (House)',
u'HouseAt2ndReading': 'Second reading (House)',
u'HouseAtReportStageAndSecondReading':
'Report stage and second reading (House)',
u'SenateAt2ndReading': 'Second reading (Senate)',
u'SenateAt3rdReading': 'Third reading (Senate)',
u'HouseAt3rdReading': 'Third reading (House)',
u'HouseInCommittee': 'In committee (House)',
u'SenateInCommittee': 'In committee (Senate)',
u'SenateConsiderationOfCommitteeReport':
'Considering committee report (Senate)',
u'HouseConsiderationOfCommitteeReport':
'Considering committee report (House)',
u'SenateConsiderationOfAmendments': 'Considering amendments (Senate)',
u'HouseConsiderationOfAmendments': 'Considering amendments (House)',
u'Introduced': 'Introduced'
}
name_en = models.TextField(blank=True)
name_fr = models.TextField(blank=True)
short_title_en = models.TextField(blank=True)
short_title_fr = models.TextField(blank=True)
number = models.CharField(max_length=10)
number_only = models.SmallIntegerField()
institution = models.CharField(max_length=1,
db_index=True,
choices=CHAMBERS)
sessions = models.ManyToManyField(Session, through='BillInSession')
privatemember = models.NullBooleanField()
sponsor_member = models.ForeignKey(ElectedMember, blank=True, null=True)
sponsor_politician = models.ForeignKey(Politician, blank=True, null=True)
law = models.NullBooleanField()
status_date = models.DateField(blank=True, null=True, db_index=True)
status_code = models.CharField(max_length=50, blank=True)
added = models.DateField(default=datetime.date.today, db_index=True)
introduced = models.DateField(blank=True, null=True)
text_docid = models.IntegerField(
blank=True,
null=True,
help_text=
"The parl.gc.ca document ID of the latest version of the bill's text")
objects = BillManager()
name = language_property('name')
short_title = language_property('short_title')
class Meta:
ordering = ('privatemember', 'institution', 'number_only')
def __unicode__(self):
return "%s - %s" % (self.number, self.name)
def get_absolute_url(self):
return self.url_for_session(self.session)
def url_for_session(self, session):
return urlresolvers.reverse('bill',
kwargs={
'session_id': session.id,
'bill_number': self.number
})
def get_legisinfo_url(self, lang='E'):
return LEGISINFO_BILL_URL % {
'lang': lang,
'bill': self.number.replace('-', ''),
'parliament': self.session.parliamentnum,
'session': self.session.sessnum
}
legisinfo_url = property(get_legisinfo_url)
def get_billtext_url(self, lang='E', single_page=False):
if not self.text_docid:
return None
url = PARLIAMENT_DOCVIEWER_URL % {
'lang': lang,
'docid': self.text_docid
}
if single_page:
url += '&File=4&Col=1'
return url
def get_text_object(self):
if not self.text_docid:
raise BillText.DoesNotExist
return BillText.objects.get(bill=self, docid=self.text_docid)
def get_text(self, language=settings.LANGUAGE_CODE):
try:
return getattr(self.get_text_object(), 'text_' + language)
except BillText.DoesNotExist:
return ''
def get_summary(self):
try:
return self.get_text_object().summary_html
except BillText.DoesNotExist:
return ''
def get_related_debates(self):
return Document.objects.filter(billinsession__bill=self)
def get_committee_meetings(self):
return CommitteeMeeting.objects.filter(billevent__bis__bill=self)
def get_major_speeches(self):
doc_ids = list(self.get_related_debates().values_list('id', flat=True))
if self.short_title_en:
qs = Statement.objects.filter(h2_en__iexact=self.short_title_en,
wordcount__gt=50)
else:
qs = self.statement_set.filter(wordcount__gt=100)
return qs.filter(document__in=doc_ids, procedural=False)
@property
def latest_date(self):
return self.status_date if self.status_date else self.introduced
def save(self, *args, **kwargs):
if not self.number_only:
self.number_only = int(re.sub(r'\D', '', self.number))
if getattr(self, 'privatemember', None) is None:
self.privatemember = bool(self.number_only >= 200)
if not self.institution:
self.institution = self.number[0]
if not self.law and self.status_code == 'RoyalAssentGiven':
self.law = True
super(Bill, self).save(*args, **kwargs)
def save_sponsor_activity(self):
if self.sponsor_politician:
activity.save_activity(
obj=self,
politician=self.sponsor_politician,
date=self.introduced if self.introduced else
(self.added - datetime.timedelta(days=1)),
variety='billsponsor',
)
def get_session(self):
"""Returns the most recent session this bill belongs to."""
try:
self.__dict__['session'] = s = self.sessions.all().order_by(
'-start')[0]
return s
except (IndexError, ValueError):
return getattr(self, '_session', None)
def set_temporary_session(self, session):
"""To deal with tricky save logic, saves a session to the object for cases
when self.sessions.all() won't get exist in the DB."""
self._session = session
session = property(get_session)
@property
def status(self):
return self.STATUS_CODES.get(self.status_code, 'Unknown')
@property
def dead(self):
return self.status_code in ('BillNotActive', 'WillNotBeProceededWith',
'BillDefeated')
@property
def dormant(self):
return (self.status_date
and (datetime.date.today() - self.status_date).days > 150)
class VoteQuestion(models.Model):
bill = models.ForeignKey(Bill, blank=True, null=True)
session = models.ForeignKey(Session)
number = models.PositiveIntegerField()
date = models.DateField(db_index=True)
description_en = models.TextField()
description_fr = models.TextField(blank=True)
result = models.CharField(max_length=1, choices=VOTE_RESULT_CHOICES)
yea_total = models.SmallIntegerField()
nay_total = models.SmallIntegerField()
paired_total = models.SmallIntegerField()
context_statement = models.ForeignKey('hansards.Statement',
blank=True,
null=True,
on_delete=models.SET_NULL)
description = language_property('description')
def __unicode__(self):
return u"Vote #%s on %s" % (self.number, self.date)
class Meta:
ordering = ('-date', '-number')
def to_api_dict(self, representation):
r = {
'bill_url': self.bill.get_absolute_url() if self.bill else None,
'session': self.session_id,
'number': self.number,
'date': unicode(self.date),
'description': {
'en': self.description_en,
'fr': self.description_fr
},
'result': self.get_result_display(),
'yea_total': self.yea_total,
'nay_total': self.nay_total,
'paired_total': self.paired_total,
}
if representation == 'detail':
r.update(
context_statement=self.context_statement.get_absolute_url()
if self.context_statement else None,
party_votes=[{
'vote': pv.get_vote_display(),
'disagreement': pv.disagreement,
'party': {
'name': {
'en': pv.party.name
},
'short_name': {
'en': pv.party.short_name
}
},
} for pv in self.partyvote_set.all()])
return r
def label_absent_members(self):
for member in ElectedMember.objects.on_date(
self.date).exclude(membervote__votequestion=self):
MemberVote(votequestion=self,
member=member,
politician_id=member.politician_id,
vote='A').save()
def label_party_votes(self):
"""Create PartyVote objects representing the party-line vote; label individual dissenting votes."""
membervotes = self.membervote_set.select_related(
'member', 'member__party').all()
parties = defaultdict(lambda: defaultdict(int))
for mv in membervotes:
if mv.member.party.name != 'Independent':
parties[mv.member.party][mv.vote] += 1
partyvotes = {}
for party in parties:
# Find the most common vote
votes = sorted(parties[party].items(), key=lambda i: i[1])
partyvotes[party] = votes[-1][0]
# Find how many people voted with the majority
yn = (parties[party]['Y'], parties[party]['N'])
try:
disagreement = float(min(yn)) / sum(yn)
except ZeroDivisionError:
disagreement = 0.0
# If more than 15% of the party voted against the party majority,
# label this as a free vote.
if disagreement >= 0.15:
partyvotes[party] = 'F'
PartyVote.objects.filter(party=party, votequestion=self).delete()
PartyVote.objects.create(party=party,
votequestion=self,
vote=partyvotes[party],
disagreement=disagreement)
for mv in membervotes:
if mv.member.party.name != 'Independent' \
and mv.vote != partyvotes[mv.member.party] \
and mv.vote in ('Y', 'N') \
and partyvotes[mv.member.party] in ('Y', 'N'):
mv.dissent = True
mv.save()
@models.permalink
def get_absolute_url(self):
return ('vote', [], {
'session_id': self.session_id,
'number': self.number
})
class Committee(models.Model):
name_en = models.TextField()
short_name_en = models.TextField()
name_fr = models.TextField(blank=True)
short_name_fr = models.TextField(blank=True)
slug = models.SlugField(unique=True)
parent = models.ForeignKey('self',
related_name='subcommittees',
blank=True,
null=True)
sessions = models.ManyToManyField(Session, through='CommitteeInSession')
joint = models.BooleanField('Joint committee?', default=False)
display = models.BooleanField('Display on site?',
db_index=True,
default=True)
objects = CommitteeManager()
name = language_property('name')
short_name = language_property('short_name')
class Meta:
ordering = ['name_en']
def __unicode__(self):
return self.name
def save(self, *args, **kwargs):
if not self.short_name_en:
self.short_name_en = self.name_en
if not self.short_name_fr:
self.short_name_fr = self.name_fr
if not self.slug:
self.slug = slugify(self.short_name_en, allow_numbers=True)
if self.parent:
self.slug = self.parent.slug + '-' + self.slug
self.slug = self.slug[:46]
while Committee.objects.filter(slug=self.slug).exists():
self.slug += '-' + random.choice(string.lowercase)
super(Committee, self).save(*args, **kwargs)
@models.permalink
def get_absolute_url(self):
return ('committee', [], {'slug': self.slug})
def get_source_url(self):
return self.committeeinsession_set.order_by(
'-session__start')[0].get_source_url()
def get_acronym(self, session):
return CommitteeInSession.objects.get(committee=self,
session=session).acronym
def latest_session(self):
return self.sessions.order_by('-start')[0]
@property
def title(self):
if 'committee' in self.name_en.lower():
return self.name
else:
return self.name + u' Committee'
def to_api_dict(self, representation):
d = dict(
name={
'en': self.name_en,
'fr': self.name_fr
},
short_name={
'en': self.short_name_en,
'fr': self.short_name_fr
},
slug=self.slug,
parent_url=self.parent.get_absolute_url() if self.parent else None,
)
if representation == 'detail':
d['sessions'] = [{
'session': cis.session_id,
'acronym': cis.acronym,
'source_url': cis.get_source_url(),
} for cis in self.committeeinsession_set.all().order_by(
'-session__end').select_related('session')]
d['subcommittees'] = [
c.get_absolute_url() for c in self.subcommittees.all()
]
return d
class Statement(models.Model):
document = models.ForeignKey(Document)
time = models.DateTimeField(db_index=True)
source_id = models.CharField(max_length=15, blank=True)
slug = models.SlugField(max_length=100, blank=True)
urlcache = models.CharField(max_length=200, blank=True)
h1_en = models.CharField(max_length=300, blank=True)
h2_en = models.CharField(max_length=300, blank=True)
h3_en = models.CharField(max_length=300, blank=True)
h1_fr = models.CharField(max_length=400, blank=True)
h2_fr = models.CharField(max_length=400, blank=True)
h3_fr = models.CharField(max_length=400, blank=True)
member = models.ForeignKey(ElectedMember, blank=True, null=True)
politician = models.ForeignKey(Politician, blank=True, null=True) # a shortcut -- should == member.politician
who_en = models.CharField(max_length=300, blank=True)
who_fr = models.CharField(max_length=500, blank=True)
who_hocid = models.PositiveIntegerField(blank=True, null=True, db_index=True)
who_context_en = models.CharField(max_length=300, blank=True)
who_context_fr = models.CharField(max_length=500, blank=True)
content_en = models.TextField()
content_fr = models.TextField(blank=True)
sequence = models.IntegerField(db_index=True)
wordcount = models.IntegerField()
wordcount_en = models.PositiveSmallIntegerField(null=True,
help_text="# words originally spoken in English")
procedural = models.BooleanField(default=False, db_index=True)
written_question = models.CharField(max_length=1, blank=True, choices=(
('Q', 'Question'),
('R', 'Response')
))
statement_type = models.CharField(max_length=35, blank=True)
bills = models.ManyToManyField('bills.Bill', blank=True)
mentioned_politicians = models.ManyToManyField(Politician, blank=True, related_name='statements_with_mentions')
class Meta:
ordering = ('sequence',)
unique_together = (
('document', 'slug')
)
h1 = language_property('h1')
h2 = language_property('h2')
h3 = language_property('h3')
who = language_property('who')
who_context = language_property('who_context')
def save(self, *args, **kwargs):
self.content_en = self.content_en.replace('\n', '').replace('</p>', '</p>\n').strip()
self.content_fr = self.content_fr.replace('\n', '').replace('</p>', '</p>\n').strip()
if self.wordcount_en is None:
self._generate_wordcounts()
if ((not self.procedural) and self.wordcount <= 300
and (
(parsetools.r_notamember.search(self.who) and re.search(r'(Speaker|Chair|président)', self.who))
or (not self.who)
or not any(p for p in self.content_en.split('\n') if 'class="procedural"' not in p)
)):
# Some form of routine, procedural statement (e.g. somethng short by the speaker)
self.procedural = True
if not self.urlcache:
self.generate_url()
super(Statement, self).save(*args, **kwargs)
@property
def date(self):
return datetime.date(self.time.year, self.time.month, self.time.day)
def generate_url(self):
self.urlcache = "%s%s/" % (
self.document.get_absolute_url(),
(self.slug if self.slug else self.sequence))
def get_absolute_url(self):
if not self.urlcache:
self.generate_url()
return self.urlcache
def __unicode__ (self):
return u"%s speaking about %s around %s" % (self.who, self.topic, self.time)
def content_floor(self):
if not self.content_fr:
return self.content_en
el, fl = self.content_en.split('\n'), self.content_fr.split('\n')
if len(el) != len(fl):
logger.error("Different en/fr paragraphs in %s" % self.get_absolute_url())
return self.content_en
r = []
for e, f in zip(el, fl):
idx = e.find('data-originallang="')
if idx and e[idx+19:idx+21] == 'fr':
r.append(f)
else:
r.append(e)
return u"\n".join(r)
def content_floor_if_necessary(self):
"""Returns text spoken in the original language(s), but only if that would
be different than the content in the default language."""
if not (self.content_en and self.content_fr):
return ''
lang_matches = re.finditer(r'data-originallang="(\w\w)"',
getattr(self, 'content_' + settings.LANGUAGE_CODE))
if any(m.group(1) != settings.LANGUAGE_CODE for m in lang_matches):
return self.content_floor()
return ''
def text_html(self, language=settings.LANGUAGE_CODE):
return mark_safe(getattr(self, 'content_' + language))
def text_plain(self, language=settings.LANGUAGE_CODE):
return self.html_to_text(getattr(self, 'content_' + language))
@staticmethod
def html_to_text(text):
return strip_tags(
text
.replace('\n', '')
.replace('<br>', '\n')
.replace('</p>', '\n\n')
).strip()
def _generate_wordcounts(self):
paragraphs = [
[], # english
[], # french
[] # procedural
]
for para in self.content_en.split('\n'):
idx = para.find('data-originallang="')
if idx == -1:
paragraphs[2].append(para)
else:
lang = para[idx+19:idx+21]
if lang == 'en':
paragraphs[0].append(para)
elif lang == 'fr':
paragraphs[1].append(para)
else:
paragraphs[0].append(para)
logger.warning("Unrecognized language %s", lang)
counts = [
len(self.html_to_text(' '.join(p)).split())
for p in paragraphs
]
self.wordcount = counts[0] + counts[1]
self.wordcount_en = counts[0]
#self.wordcount_procedural = counts[2]
# temp compatibility
@property
def heading(self):
return self.h1
@property
def topic(self):
return self.h2
def to_api_dict(self, representation):
d = dict(
time=unicode(self.time) if self.time else None,
attribution={'en': self.who_en, 'fr': self.who_fr},
content={'en': self.content_en, 'fr': self.content_fr},
url=self.get_absolute_url(),
politician_url=self.politician.get_absolute_url() if self.politician else None,
politician_membership_url=urlresolvers.reverse('politician_membership',
kwargs={'member_id': self.member_id}) if self.member_id else None,
procedural=self.procedural,
source_id=self.source_id
)
for h in ('h1', 'h2', 'h3'):
if getattr(self, h):
d[h] = {'en': getattr(self, h + '_en'), 'fr': getattr(self, h + '_fr')}
d['document_url'] = d['url'][:d['url'].rstrip('/').rfind('/')+1]
return d
@property
@memoize_property
def name_info(self):
info = {
'post': None,
'named': True
}
if not self.member:
info['display_name'] = parsetools.r_mister.sub('', self.who)
if self.who_context:
if self.who_context in self.who:
info['display_name'] = parsetools.r_parens.sub('', info['display_name'])
info['post'] = self.who_context
else:
info['post_reminder'] = self.who_context
if self.who_hocid:
info['url'] = '/search/?q=Witness%%3A+%%22%s%%22' % self.who_hocid
else:
info['url'] = self.member.politician.get_absolute_url()
if parsetools.r_notamember.search(self.who):
info['display_name'] = self.who
if self.member.politician.name in self.who:
info['display_name'] = re.sub(r'\(.+\)', '', self.who)
info['named'] = False
elif not '(' in self.who or not parsetools.r_politicalpost.search(self.who):
info['display_name'] = self.member.politician.name
else:
post_match = re.search(r'\((.+)\)', self.who)
if post_match:
info['post'] = post_match.group(1).split(',')[0]
info['display_name'] = self.member.politician.name
return info
@staticmethod
def set_slugs(statements):
counter = defaultdict(int)
for statement in statements:
slug = slugify(statement.name_info['display_name'])[:50]
if not slug:
slug = 'procedural'
counter[slug] += 1
statement.slug = slug + '-%s' % counter[slug]
@property
def committee_name(self):
if self.document.document_type != Document.EVIDENCE:
return ''
return self.document.committeemeeting.committee.short_name
@property
def committee_slug(self):
if self.document.document_type != Document.EVIDENCE:
return ''
return self.document.committeemeeting.committee.slug
