class VideoComment(models.Model):
"""
Represents a user's comment on a Video
"""
video = models.ForeignKey(Video)
# overall comment sentiment analysis
sentiment = models.FloatField(default=0)
magnitude = models.FloatField(default=0)
analyzed_comment = JSONField(blank=True, null=True)
analysis_failed = models.BooleanField(default=False)
# you tube specific data
youtube_id = models.CharField(max_length=100)
author_display_name = models.CharField(max_length=100)
author_profile_image_url = models.CharField(max_length=255)
comment_raw = models.TextField()
comment_rich = models.TextField()
published = models.DateTimeField()
updated = models.DateTimeField()
def __unicode__(self):
return u'Video: {} Comment: {}'.format(self.video_id, self.comment_raw)
@property
def analysis_complete(self):
"""
If our comment has an analyzed comment, then analysis was successful
"""
if self.analyzed_comment:
return True
return False
class Meta:
ordering = ['-updated', '-published']
class Video(models.Model):
"""
Represents a video within a project
"""
owner = models.ForeignKey(get_user_model())
project = models.ForeignKey(Project)
# overall video sentiment analysis
sentiment = models.FloatField(default=0)
magnitude = models.FloatField(default=0)
analyzed_transcript = JSONField(blank=True, null=True)
transcript_failed = models.BooleanField(default=False)
# you tube specific data
youtube_id = models.CharField(max_length=25)
name = models.CharField(max_length=255)
description = models.TextField(blank=True)
transcript = models.TextField(blank=True)
published = models.DateTimeField()
thumbnail_default = models.CharField(max_length=255)
thumbnail_medium = models.CharField(max_length=255)
thumbnail_high = models.CharField(max_length=255)
likes = models.PositiveIntegerField(default=0)
dislikes = models.PositiveIntegerField(default=0)
comment_count = models.PositiveIntegerField(default=0)
def __unicode__(self):
return u'{}'.format(self.name)
@property
def analysis_complete(self):
if self.analyzed_transcript:
return True
return False
@property
def can_be_analyzed(self):
"""
Videos can only be analyzed if they have a transcript available.
"""
if self.transcript:
return True
return False
@property
def comment_analysis_complete(self):
"""
Returns True if all the videos comments have been analyzed
"""
for comment in self.videocomment_set.all():
if not comment.analysis_complete:
return False
return True
class Meta:
ordering = ['-published']
class ThumbnailAbstract(models.Model):
thumbnails = JSONField()
def get_thumbnail(self, size=DEFAULT_SIZE):
if size in self.thumbnails:
return self.thumbnails[size]
elif DEFAULT_SIZE in self.thumbnails:
return self.thumbnails[DEFAULT_SIZE]
elif len(self.thumbnails) > 0:
return self.thumbnails.values()[0]
else:
return ""
class Meta:
abstract = True
class TestResult(models.Model):
name = models.CharField(max_length=500, editable=False)
django_version = models.CharField(max_length=10, editable=False)
djangae_version = models.CharField(max_length=10, editable=False)
class Meta:
unique_together = [("name", "django_version", "djangae_version")]
last_modified = models.DateField(auto_now=True, editable=False)
status = models.CharField(
max_length=50,
choices=test_result_choices,
default=test_result_choices[0][0],
editable=False,
)
score = models.FloatField(default=-1, editable=False)
data = JSONField(default=dict, editable=False)
objects = TestResultManager()
def __unicode__(self):
return self.name
class JSONFieldWithDefaultModel(models.Model):
json_field = JSONField(use_ordered_dict=True)
class JSONFieldModel(models.Model):
json_field = JSONField(use_ordered_dict=True)
def contribute_to_class(self, cls, name, virtual_only=False):
try:
cls._meta.get_field(MASTERS_CACHE_ATTR)
except FieldDoesNotExist:
cache_field = JSONField(blank=True)
cache_field.contribute_to_class(cls, MASTERS_CACHE_ATTR,
virtual_only)
# Do whatever foreignkey does
super(TranslatableCharField,
self).contribute_to_class(cls, name, virtual_only)
# Get the klass of the descriptor that it used
klass = getattr(cls, name).__class__
CACHE_ATTRIBUTE = "{}_content".format(self.name)
# Now, subclass it so we can add our own magic
class TranslatableFieldDescriptor(klass):
def __get__(self, instance, instance_type):
# First, do we have a content attribute or non-None default already,
# if so, return it
existing = getattr(instance, CACHE_ATTRIBUTE,
self.field.get_default())
if existing:
return existing
master_translation = None
master_id = getattr(instance, self.field.attname)
instance_cache = getattr(instance, MASTERS_CACHE_ATTR)
master_data = instance_cache.get(master_id, None)
# If there's a master_id assigned but it's not in the instance cache yet,
# attempt to retrieve the related MasterTranslation
if master_id and not master_data:
master_translation = super(TranslatableFieldDescriptor,
self).__get__(
instance, instance_type)
if master_translation:
master_data = instance_cache[master_id] = {
'text': master_translation.text,
'hint': master_translation.hint,
'lang': master_translation.language_code,
}
if master_data:
# When master_data is coming from the instance cache the resulting
# TranslatableContent is crippled (the master_translation_cache is None),
# but it works fine for translations.
new_content = TranslatableContent()
new_content._master_translation_id = master_id
new_content._master_translation_cache = master_translation
new_content._hint = master_data['hint']
new_content._text = master_data['text']
new_content._language_code = master_data['lang']
else:
new_content = TranslatableContent(hint=self.field.hint)
setattr(instance, CACHE_ATTRIBUTE, new_content)
return new_content
def __set__(self, instance, value):
if not isinstance(value, TranslatableContent):
raise ValueError("Must be a TranslatableContent instance")
# If no hint is specified, but we have a default, then set it
value.hint = value.hint or self.field.hint
# Replace the content attribute
setattr(instance, CACHE_ATTRIBUTE, value)
# If this is new, never before seen content, then _master_translation_id
# will be None, so we don't want to set anything in the master translation
# cache field
if value._master_translation_id:
# Update the instance master translation cache
getattr(
instance,
MASTERS_CACHE_ATTR)[value._master_translation_id] = {
'hint': value.hint,
'text': value.text,
'lang': value.language_code,
}
# Make sure we update the underlying master translation appropriately
super(TranslatableFieldDescriptor,
self).__set__(instance, value._master_translation_id)
setattr(cls, self.name, TranslatableFieldDescriptor(self))
def contribute_to_class(self, cls, name, virtual_only=False):
try:
cls._meta.get_field(MASTERS_CACHE_ATTR)
except FieldDoesNotExist:
cache_field = JSONField(blank=True)
cache_field.contribute_to_class(cls, MASTERS_CACHE_ATTR, virtual_only)
# Do whatever foreignkey does
super(TranslatableCharField, self).contribute_to_class(cls, name, virtual_only)
# Get the klass of the descriptor that it used
klass = getattr(cls, name).__class__
CACHE_ATTRIBUTE = "{}_content".format(self.name)
# Now, subclass it so we can add our own magic
class TranslatableFieldDescriptor(klass):
def __get__(self, instance, instance_type):
# First, do we have a content attribute or non-None default already,
# if so, return it
existing = getattr(instance, CACHE_ATTRIBUTE, self.field.get_default())
if existing:
return existing
master_translation = None
master_id = getattr(instance, self.field.attname)
instance_cache = getattr(instance, MASTERS_CACHE_ATTR)
master_data = instance_cache.get(master_id, None)
# If there's a master_id assigned but it's not in the instance cache yet,
# attempt to retrieve the related MasterTranslation
if master_id and not master_data:
master_translation = super(TranslatableFieldDescriptor, self).__get__(instance, instance_type)
if master_translation:
master_data = instance_cache[master_id] = {
'text': master_translation.text,
'hint': master_translation.hint,
'lang': master_translation.language_code,
}
if master_data:
# When master_data is coming from the instance cache the resulting
# TranslatableContent is crippled (the master_translation_cache is None),
# but it works fine for translations.
new_content = TranslatableContent()
new_content._master_translation_id = master_id
new_content._master_translation_cache = master_translation
new_content._hint = master_data['hint']
new_content._text = master_data['text']
new_content._language_code = master_data['lang']
else:
new_content = TranslatableContent(hint=self.field.hint)
setattr(instance, CACHE_ATTRIBUTE, new_content)
return new_content
def __set__(self, instance, value):
if not isinstance(value, TranslatableContent):
raise ValueError("Must be a TranslatableContent instance")
# If no hint is specified, but we have a default, then set it
value.hint = value.hint or self.field.hint
# Replace the content attribute
setattr(instance, CACHE_ATTRIBUTE, value)
# If this is new, never before seen content, then _master_translation_id
# will be None, so we don't want to set anything in the master translation
# cache field
if value._master_translation_id:
# Update the instance master translation cache
getattr(instance, MASTERS_CACHE_ATTR)[value._master_translation_id] = {
'hint': value.hint,
'text': value.text,
'lang': value.language_code,
}
# Make sure we update the underlying master translation appropriately
super(TranslatableFieldDescriptor, self).__set__(instance, value._master_translation_id)
setattr(cls, self.name, TranslatableFieldDescriptor(self))
class Translation(models.Model):
master_translation = models.ForeignKey("fluent.MasterTranslation", editable=False, related_name="+")
language_code = models.CharField(max_length=8, blank=False)
plural_texts = JSONField(blank=True) # These are the various plural translations depending on the language
denorm_master_text = models.TextField(editable=False)
denorm_master_hint = models.CharField(max_length=500, editable=False)
denorm_master_language = models.CharField(max_length=8, editable=False)
master_text_hint_hash = models.CharField(max_length=64)
class Meta:
app_label = "fluent"
@property
def text(self):
singular_form = get_plural_index(self.language_code, 1)
try:
return self.plural_texts[singular_form]
except KeyError:
# Some kind of corrupt data, so just return the source language
return self.denorm_master_text
@text.setter
def text(self, value):
singular_form = get_plural_index(self.language_code, 1)
self.plural_texts[singular_form] = value
def clean(self):
msgs = validate_translation_texts(self)
if msgs:
raise ValidationError([err for err, _orig, _trans in msgs])
def __unicode__(self):
return u"Translation of {} for {}".format(self.denorm_master_text, self.language_code)
def __repr__(self):
"""
Define an ASCII string safe representation of the translation.
"""
return str("{}".format(self.id))
@staticmethod
def generate_hash(master_text, master_hint):
assert master_text
assert master_hint is not None
result = md5()
for x in (master_text, master_hint):
x = x.encode('utf-8')
result.update(x)
return result.hexdigest()
def save(self, *args, **kwargs):
assert self.language_code
assert self.master_translation_id
assert len(self.plural_texts)
self.denorm_master_text = self.master_translation.text
self.denorm_master_hint = self.master_translation.hint
self.denorm_master_language = self.master_translation.language_code
# For querying (you can't query for text on the datastore)
self.master_text_hint_hash = Translation.generate_hash(
self.denorm_master_text,
self.denorm_master_hint
)
return super(Translation, self).save(*args, **kwargs)
class MasterTranslation(models.Model):
id = models.CharField(max_length=64, primary_key=True)
text = models.TextField()
text_for_ordering = ComputedCharField(lambda instance: instance.text[:500], max_length=500)
plural_text = models.TextField(blank=True)
hint = models.CharField(max_length=500, default="", blank=True)
language_code = models.CharField(
max_length=8,
choices=settings.LANGUAGES,
default=settings.LANGUAGE_CODE
)
translations_by_language_code = JSONField()
translations = RelatedSetField(Translation)
translated_into_languages = SetField(models.CharField(max_length=8), editable=False)
# Was this master translation updated or created by make messages?
used_in_code_or_templates = models.BooleanField(default=False, blank=True, editable=False)
# Were any groups specified in the trans tags?
used_by_groups_in_code_or_templates = SetField(models.CharField(max_length=64), blank=True)
# Record the ID of the last scan which updated this instance (if any)
last_updated_by_scan_uuid = models.CharField(max_length=64, blank=True, default="")
first_letter = models.CharField(max_length=1, editable=False)
@property
def is_plural(self):
return bool(self.plural_text)
def __unicode__(self):
return u"{} ({}{})".format(self.text, self.language_code, ' plural' if self.is_plural else '')
def __repr__(self):
"""
Define an ASCII string safe representation of the master translation.
"""
return str("{}".format(self.id))
def get_display(self):
from fluent.trans import _get_trans
result = _get_trans(self.text, self.hint)
return result
def text_for_language_code(self, lang_code):
new_code = find_closest_supported_language(lang_code)
if new_code not in self.translations_by_language_code.keys():
# we don't have a translation for this language
return self.text
translation_id = self.translations_by_language_code[new_code]
translation = Translation.objects.get(id=translation_id)
return translation.text
@classmethod
def find_by_groups(cls, groups):
from .fields import find_installed_translatable_fields
translatable_fields_by_model = find_installed_translatable_fields(with_groups=groups)
# Go through all Translatable(Char|Text)Fields or TextFields marked with the specified group and get
# all the master translation IDs which are set to them
master_translation_ids = []
for model, fields in translatable_fields_by_model.items():
master_translation_ids.extend(chain(*model.objects.values_list(*[field.attname for field in fields])))
master_translation_ids = list(set(master_translation_ids))
# Now get all the master translations with a group specified in the templates
master_translation_ids.extend(
list(MasterTranslation.objects.filter(used_by_groups_in_code_or_templates__overlap=groups)
.values_list("pk", flat=True))
)
# Make sure master translation ids don't include None values or duplicates
master_translation_ids = set(master_translation_ids)
master_translation_ids = master_translation_ids - {None}
# Return them all!
return MasterTranslation.objects.filter(pk__in=master_translation_ids)
@classmethod
def find_by_group(cls, group_name):
return cls.find_by_groups([group_name])
@staticmethod
def generate_key(text, hint, language_code):
assert text
assert hint is not None
assert language_code
result = md5()
for x in (text.encode("utf-8"), hint.encode("utf-8"), language_code):
result.update(x)
return result.hexdigest()
def save(self, *args, **kwargs):
assert self.text
assert self.language_code
# Always store the first letter for querying
self.first_letter = self.text.strip()[:1]
# Generate the appropriate key on creation
if self._state.adding:
self.pk = MasterTranslation.generate_key(
self.text, self.hint, self.language_code
)
# If we are adding for the first time, then create a counterpart
# translation for the master language.
# Note that this Translation will be complete and correct only for the languages that
# only require 2 plural forms - for others this language needs to be explicitly translated
# or updated later.
if self._state.adding:
with transaction.atomic(xg=True):
singular_form = get_plural_index(self.language_code, 1)
plural_form = get_plural_index(self.language_code, 2)
plurals = {singular_form: self.text}
if self.is_plural:
plurals[plural_form] = self.plural_text
# if len(LANGUAGE_LOOKUPS[self.language_code].plurals_needed) > len(plurals):
# FIXME: We can detect that we're dealing with a language that needs more plurals
# What should we do? mark the translation as incomplete?
# Don't create the translation object at all?
new_trans = Translation.objects.create(
master_translation=self,
language_code=self.language_code,
plural_texts=plurals,
denorm_master_text=self.text,
denorm_master_hint=self.hint
)
self.translations_by_language_code[self.language_code] = new_trans.pk
self.translations.add(new_trans)
self.translated_into_languages = set(self.translations_by_language_code.keys())
return super(MasterTranslation, self).save(*args, **kwargs)
else:
# Otherwise just do a normal save
self.translated_into_languages = set(self.translations_by_language_code.keys())
return super(MasterTranslation, self).save(*args, **kwargs)
def create_or_update_translation(self, language_code, singular_text=None, plural_texts=None, validate=False):
if language_code not in dict(settings.LANGUAGES).keys():
return ["'{}' is not included as a language in your settings file".format(language_code)]
with transaction.atomic(xg=True):
trans = None
if language_code in self.translations_by_language_code:
# We already have a translation for this language, update it!
try:
trans = Translation.objects.get(pk=self.translations_by_language_code[language_code])
created = False
except Translation.DoesNotExist:
trans = None
if not trans:
# OK, create the translation object and add it to the respective fields
trans = Translation(
master_translation_id=self.pk,
language_code=language_code,
denorm_master_hint=self.hint,
denorm_master_text=self.text
)
created = True
if plural_texts:
trans.plural_texts = plural_texts
else:
trans.text = singular_text
if validate:
errors = validate_translation_texts(trans, self)
if errors:
return errors
trans.master_translation = self
trans.save()
if created:
self.refresh_from_db()
self.translations_by_language_code[language_code] = trans.pk
self.translations.add(trans)
self.save()
class Meta:
app_label = "fluent"
class Dog(models.Model):
name = models.CharField(max_length=32)
data = JSONField()
class NullableJSONFieldModel(models.Model):
json_field = JSONField(null=True, blank=True)
