class User(BaseModel
): # TODO: subclass Django/Djangae user base model, or something
""" A public user who is registered on the site. """
constituency = CharField(choices=CONSTITUENCIES.choices)
# The MP can be calculated from constituency, but shortcut field for speed and convenience:
mp = models.ForeignKey(MP, related_name="constituents")
follows_mps = RelatedSetField(
MP,
help_text=
"MPs which this user wants to get/see info/updates/notifications about",
blank=True,
related_name="followers",
)
follows_keywords = SetField(
CharField(),
help_text=
"Keywords which this user wants to know about when an MP says them",
blank=True,
)
def save(self, *args, **kwargs):
# Make the user follow their own MP. # TODO: maybe don't be so forceful!
self.follows_mps_ids.add(self.mp_id)
return super(User, self).save(*args, **kwargs)
class Speech(BaseModel):
""" The words spoken by an MP in a single "speech".
A "speech" will probably be the words spoken in a single instance of an MP standing up, but
it might depend on how Hansard publishes the data.
"""
mp = models.ForeignKey(MP)
timestamp = models.DateTimeField(
help_text=
"The time when the MP spoke these words (not when this was saved to the DB)"
)
hansard_url = models.URLField(
help_text=
"The URL of where this speech is located on the Hansard website")
text = models.TextField()
keywords = SetField(
CharField(),
# Not sure if this should be the non-filler words, or the words which users are actually
# following at the time when it's saved. TBD.
help_text="Words from the speech which are relevant for querying by")
# These fields store the number of users that have like/disliked/flagged this speech.
# For each of these counts there is a separate "through" table between the users and the
# speeches, but because we're on a non-relational DB and can't actually JOIN those tables, we
# store these plain counts directly on the Speech model.
# These may need sharded counters if we get a lot of users liking/disliking things at the same
# time, but keeping it simple for now.
like_count = models.PositiveIntegerField(default=0)
dislike_count = models.PositiveIntegerField(default=0)
is_filibustering_count = models.PositiveIntegerField(default=0)
fact_check_request_count = models.PositiveIntegerField(default=0)
class InstanceIndex(models.Model):
@classmethod
def calc_id(cls, term, instance):
source = u"|".join(
[term, instance.__class__._meta.db_table,
unicode(instance.pk)])
return hashlib.md5(source.encode("utf-8")).hexdigest()
id = models.CharField(max_length=500, primary_key=True)
iexact = models.CharField(max_length=1024)
instance_db_table = models.CharField(max_length=1024)
instance_pk = models.PositiveIntegerField(default=0)
count = models.PositiveIntegerField(default=0)
partials = SetField(models.CharField(max_length=500))
class Meta:
unique_together = [('iexact', 'instance_db_table', 'instance_pk')]
def _generate_partials(self):
"""
Partials are anything we want to match when doing fuzzy matching
change this logic if you can think of more possibilities!
"""
partials = set([
self.iexact
]) #We always include the term itself for easier querying
length = len(self.iexact)
for i in xrange(int(math.floor(float(length) / 2.0)), length):
s = self.iexact[:i]
# We want to match the first half of the word always
# but be fuzzy with the last half
partials.add(s)
# Now, just add the term with characters missing
for j in xrange(1, len(self.iexact)):
partials.add(self.iexact[:j] + self.iexact[j + 1:])
# And swap out vowels
vowels = "aeiou"
for i, vowel in enumerate(vowels):
others = vowels[:i] + vowels[i + 1:]
for other in others:
s = self.iexact
while vowel in s:
s = s.replace(vowel, other, 1)
partials.add(s)
return partials
def save(self, *args, **kwargs):
self.partials = self._generate_partials()
return super(InstanceIndex, self).save(*args, **kwargs)
def test_set_field(self):
instance = IterableFieldModel.objects.create()
self.assertEqual(set(), instance.set_field)
instance.set_field.add("One")
self.assertEqual(set(["One"]), instance.set_field)
instance.save()
self.assertEqual(set(["One"]), instance.set_field)
instance = IterableFieldModel.objects.get(pk=instance.pk)
self.assertEqual(set(["One"]), instance.set_field)
self.assertEqual({1, 2}, SetField(models.IntegerField).to_python("{1, 2}"))
def test_set_field(self):
instance = IterableFieldModel.objects.create()
self.assertEqual(set(), instance.set_field)
instance.set_field.add("One")
self.assertEqual(set(["One"]), instance.set_field)
instance.save()
self.assertEqual(set(["One"]), instance.set_field)
instance = IterableFieldModel.objects.get(pk=instance.pk)
self.assertEqual(set(["One"]), instance.set_field)
instance.set_field = None
# Or anything else for that matter!
with self.assertRaises(ValueError):
instance.set_field = "Bananas"
instance.save()
self.assertEqual({1, 2}, SetField(models.IntegerField).to_python("{1, 2}"))
class IterableFieldModel(models.Model):
set_field = SetField(models.CharField(max_length=1))
list_field = ListField(models.CharField(max_length=1))
class Meta:
app_label = "djangae"
class MasterTranslation(models.Model):
id = models.CharField(max_length=64, primary_key=True)
text = models.TextField()
text_for_ordering = ComputedCharField(lambda instance: instance.text[:500], max_length=500)
plural_text = models.TextField(blank=True)
hint = models.CharField(max_length=500, default="", blank=True)
language_code = models.CharField(
max_length=8,
choices=settings.LANGUAGES,
default=settings.LANGUAGE_CODE
)
translations_by_language_code = JSONField()
translations = RelatedSetField(Translation)
translated_into_languages = SetField(models.CharField(max_length=8), editable=False)
# Was this master translation updated or created by make messages?
used_in_code_or_templates = models.BooleanField(default=False, blank=True, editable=False)
# Were any groups specified in the trans tags?
used_by_groups_in_code_or_templates = SetField(models.CharField(max_length=64), blank=True)
# Record the ID of the last scan which updated this instance (if any)
last_updated_by_scan_uuid = models.CharField(max_length=64, blank=True, default="")
first_letter = models.CharField(max_length=1, editable=False)
@property
def is_plural(self):
return bool(self.plural_text)
def __unicode__(self):
return u"{} ({}{})".format(self.text, self.language_code, ' plural' if self.is_plural else '')
def __repr__(self):
"""
Define an ASCII string safe representation of the master translation.
"""
return str("{}".format(self.id))
def get_display(self):
from fluent.trans import _get_trans
result = _get_trans(self.text, self.hint)
return result
def text_for_language_code(self, lang_code):
new_code = find_closest_supported_language(lang_code)
if new_code not in self.translations_by_language_code.keys():
# we don't have a translation for this language
return self.text
translation_id = self.translations_by_language_code[new_code]
translation = Translation.objects.get(id=translation_id)
return translation.text
@classmethod
def find_by_groups(cls, groups):
from .fields import find_installed_translatable_fields
translatable_fields_by_model = find_installed_translatable_fields(with_groups=groups)
# Go through all Translatable(Char|Text)Fields or TextFields marked with the specified group and get
# all the master translation IDs which are set to them
master_translation_ids = []
for model, fields in translatable_fields_by_model.items():
master_translation_ids.extend(chain(*model.objects.values_list(*[field.attname for field in fields])))
master_translation_ids = list(set(master_translation_ids))
# Now get all the master translations with a group specified in the templates
master_translation_ids.extend(
list(MasterTranslation.objects.filter(used_by_groups_in_code_or_templates__overlap=groups)
.values_list("pk", flat=True))
)
# Make sure master translation ids don't include None values or duplicates
master_translation_ids = set(master_translation_ids)
master_translation_ids = master_translation_ids - {None}
# Return them all!
return MasterTranslation.objects.filter(pk__in=master_translation_ids)
@classmethod
def find_by_group(cls, group_name):
return cls.find_by_groups([group_name])
@staticmethod
def generate_key(text, hint, language_code):
assert text
assert hint is not None
assert language_code
result = md5()
for x in (text.encode("utf-8"), hint.encode("utf-8"), language_code):
result.update(x)
return result.hexdigest()
def save(self, *args, **kwargs):
assert self.text
assert self.language_code
# Always store the first letter for querying
self.first_letter = self.text.strip()[:1]
# Generate the appropriate key on creation
if self._state.adding:
self.pk = MasterTranslation.generate_key(
self.text, self.hint, self.language_code
)
# If we are adding for the first time, then create a counterpart
# translation for the master language.
# Note that this Translation will be complete and correct only for the languages that
# only require 2 plural forms - for others this language needs to be explicitly translated
# or updated later.
if self._state.adding:
with transaction.atomic(xg=True):
singular_form = get_plural_index(self.language_code, 1)
plural_form = get_plural_index(self.language_code, 2)
plurals = {singular_form: self.text}
if self.is_plural:
plurals[plural_form] = self.plural_text
# if len(LANGUAGE_LOOKUPS[self.language_code].plurals_needed) > len(plurals):
# FIXME: We can detect that we're dealing with a language that needs more plurals
# What should we do? mark the translation as incomplete?
# Don't create the translation object at all?
new_trans = Translation.objects.create(
master_translation=self,
language_code=self.language_code,
plural_texts=plurals,
denorm_master_text=self.text,
denorm_master_hint=self.hint
)
self.translations_by_language_code[self.language_code] = new_trans.pk
self.translations.add(new_trans)
self.translated_into_languages = set(self.translations_by_language_code.keys())
return super(MasterTranslation, self).save(*args, **kwargs)
else:
# Otherwise just do a normal save
self.translated_into_languages = set(self.translations_by_language_code.keys())
return super(MasterTranslation, self).save(*args, **kwargs)
def create_or_update_translation(self, language_code, singular_text=None, plural_texts=None, validate=False):
if language_code not in dict(settings.LANGUAGES).keys():
return ["'{}' is not included as a language in your settings file".format(language_code)]
with transaction.atomic(xg=True):
trans = None
if language_code in self.translations_by_language_code:
# We already have a translation for this language, update it!
try:
trans = Translation.objects.get(pk=self.translations_by_language_code[language_code])
created = False
except Translation.DoesNotExist:
trans = None
if not trans:
# OK, create the translation object and add it to the respective fields
trans = Translation(
master_translation_id=self.pk,
language_code=language_code,
denorm_master_hint=self.hint,
denorm_master_text=self.text
)
created = True
if plural_texts:
trans.plural_texts = plural_texts
else:
trans.text = singular_text
if validate:
errors = validate_translation_texts(trans, self)
if errors:
return errors
trans.master_translation = self
trans.save()
if created:
self.refresh_from_db()
self.translations_by_language_code[language_code] = trans.pk
self.translations.add(trans)
self.save()
class Meta:
app_label = "fluent"
class IterableFieldsWithValidatorsModel(models.Model):
set_field = SetField(models.CharField(max_length=100), min_length=2, max_length=3, blank=False)
list_field = ListField(models.CharField(max_length=100), min_length=2, max_length=3, blank=False)
related_set = RelatedSetField(ISOther, min_length=2, max_length=3, blank=False)
related_list = RelatedListField(ISOther, related_name="iterable_list", min_length=2, max_length=3, blank=False)