class VideoComment(models.Model): """ Represents a user's comment on a Video """ video = models.ForeignKey(Video) # overall comment sentiment analysis sentiment = models.FloatField(default=0) magnitude = models.FloatField(default=0) analyzed_comment = JSONField(blank=True, null=True) analysis_failed = models.BooleanField(default=False) # you tube specific data youtube_id = models.CharField(max_length=100) author_display_name = models.CharField(max_length=100) author_profile_image_url = models.CharField(max_length=255) comment_raw = models.TextField() comment_rich = models.TextField() published = models.DateTimeField() updated = models.DateTimeField() def __unicode__(self): return u'Video: {} Comment: {}'.format(self.video_id, self.comment_raw) @property def analysis_complete(self): """ If our comment has an analyzed comment, then analysis was successful """ if self.analyzed_comment: return True return False class Meta: ordering = ['-updated', '-published']
class Video(models.Model): """ Represents a video within a project """ owner = models.ForeignKey(get_user_model()) project = models.ForeignKey(Project) # overall video sentiment analysis sentiment = models.FloatField(default=0) magnitude = models.FloatField(default=0) analyzed_transcript = JSONField(blank=True, null=True) transcript_failed = models.BooleanField(default=False) # you tube specific data youtube_id = models.CharField(max_length=25) name = models.CharField(max_length=255) description = models.TextField(blank=True) transcript = models.TextField(blank=True) published = models.DateTimeField() thumbnail_default = models.CharField(max_length=255) thumbnail_medium = models.CharField(max_length=255) thumbnail_high = models.CharField(max_length=255) likes = models.PositiveIntegerField(default=0) dislikes = models.PositiveIntegerField(default=0) comment_count = models.PositiveIntegerField(default=0) def __unicode__(self): return u'{}'.format(self.name) @property def analysis_complete(self): if self.analyzed_transcript: return True return False @property def can_be_analyzed(self): """ Videos can only be analyzed if they have a transcript available. """ if self.transcript: return True return False @property def comment_analysis_complete(self): """ Returns True if all the videos comments have been analyzed """ for comment in self.videocomment_set.all(): if not comment.analysis_complete: return False return True class Meta: ordering = ['-published']
class ThumbnailAbstract(models.Model): thumbnails = JSONField() def get_thumbnail(self, size=DEFAULT_SIZE): if size in self.thumbnails: return self.thumbnails[size] elif DEFAULT_SIZE in self.thumbnails: return self.thumbnails[DEFAULT_SIZE] elif len(self.thumbnails) > 0: return self.thumbnails.values()[0] else: return "" class Meta: abstract = True
class TestResult(models.Model): name = models.CharField(max_length=500, editable=False) django_version = models.CharField(max_length=10, editable=False) djangae_version = models.CharField(max_length=10, editable=False) class Meta: unique_together = [("name", "django_version", "djangae_version")] last_modified = models.DateField(auto_now=True, editable=False) status = models.CharField( max_length=50, choices=test_result_choices, default=test_result_choices[0][0], editable=False, ) score = models.FloatField(default=-1, editable=False) data = JSONField(default=dict, editable=False) objects = TestResultManager() def __unicode__(self): return self.name
class JSONFieldWithDefaultModel(models.Model): json_field = JSONField(use_ordered_dict=True)
class JSONFieldModel(models.Model): json_field = JSONField(use_ordered_dict=True)
def contribute_to_class(self, cls, name, virtual_only=False): try: cls._meta.get_field(MASTERS_CACHE_ATTR) except FieldDoesNotExist: cache_field = JSONField(blank=True) cache_field.contribute_to_class(cls, MASTERS_CACHE_ATTR, virtual_only) # Do whatever foreignkey does super(TranslatableCharField, self).contribute_to_class(cls, name, virtual_only) # Get the klass of the descriptor that it used klass = getattr(cls, name).__class__ CACHE_ATTRIBUTE = "{}_content".format(self.name) # Now, subclass it so we can add our own magic class TranslatableFieldDescriptor(klass): def __get__(self, instance, instance_type): # First, do we have a content attribute or non-None default already, # if so, return it existing = getattr(instance, CACHE_ATTRIBUTE, self.field.get_default()) if existing: return existing master_translation = None master_id = getattr(instance, self.field.attname) instance_cache = getattr(instance, MASTERS_CACHE_ATTR) master_data = instance_cache.get(master_id, None) # If there's a master_id assigned but it's not in the instance cache yet, # attempt to retrieve the related MasterTranslation if master_id and not master_data: master_translation = super(TranslatableFieldDescriptor, self).__get__( instance, instance_type) if master_translation: master_data = instance_cache[master_id] = { 'text': master_translation.text, 'hint': master_translation.hint, 'lang': master_translation.language_code, } if master_data: # When master_data is coming from the instance cache the resulting # TranslatableContent is crippled (the master_translation_cache is None), # but it works fine for translations. new_content = TranslatableContent() new_content._master_translation_id = master_id new_content._master_translation_cache = master_translation new_content._hint = master_data['hint'] new_content._text = master_data['text'] new_content._language_code = master_data['lang'] else: new_content = TranslatableContent(hint=self.field.hint) setattr(instance, CACHE_ATTRIBUTE, new_content) return new_content def __set__(self, instance, value): if not isinstance(value, TranslatableContent): raise ValueError("Must be a TranslatableContent instance") # If no hint is specified, but we have a default, then set it value.hint = value.hint or self.field.hint # Replace the content attribute setattr(instance, CACHE_ATTRIBUTE, value) # If this is new, never before seen content, then _master_translation_id # will be None, so we don't want to set anything in the master translation # cache field if value._master_translation_id: # Update the instance master translation cache getattr( instance, MASTERS_CACHE_ATTR)[value._master_translation_id] = { 'hint': value.hint, 'text': value.text, 'lang': value.language_code, } # Make sure we update the underlying master translation appropriately super(TranslatableFieldDescriptor, self).__set__(instance, value._master_translation_id) setattr(cls, self.name, TranslatableFieldDescriptor(self))
def contribute_to_class(self, cls, name, virtual_only=False): try: cls._meta.get_field(MASTERS_CACHE_ATTR) except FieldDoesNotExist: cache_field = JSONField(blank=True) cache_field.contribute_to_class(cls, MASTERS_CACHE_ATTR, virtual_only) # Do whatever foreignkey does super(TranslatableCharField, self).contribute_to_class(cls, name, virtual_only) # Get the klass of the descriptor that it used klass = getattr(cls, name).__class__ CACHE_ATTRIBUTE = "{}_content".format(self.name) # Now, subclass it so we can add our own magic class TranslatableFieldDescriptor(klass): def __get__(self, instance, instance_type): # First, do we have a content attribute or non-None default already, # if so, return it existing = getattr(instance, CACHE_ATTRIBUTE, self.field.get_default()) if existing: return existing master_translation = None master_id = getattr(instance, self.field.attname) instance_cache = getattr(instance, MASTERS_CACHE_ATTR) master_data = instance_cache.get(master_id, None) # If there's a master_id assigned but it's not in the instance cache yet, # attempt to retrieve the related MasterTranslation if master_id and not master_data: master_translation = super(TranslatableFieldDescriptor, self).__get__(instance, instance_type) if master_translation: master_data = instance_cache[master_id] = { 'text': master_translation.text, 'hint': master_translation.hint, 'lang': master_translation.language_code, } if master_data: # When master_data is coming from the instance cache the resulting # TranslatableContent is crippled (the master_translation_cache is None), # but it works fine for translations. new_content = TranslatableContent() new_content._master_translation_id = master_id new_content._master_translation_cache = master_translation new_content._hint = master_data['hint'] new_content._text = master_data['text'] new_content._language_code = master_data['lang'] else: new_content = TranslatableContent(hint=self.field.hint) setattr(instance, CACHE_ATTRIBUTE, new_content) return new_content def __set__(self, instance, value): if not isinstance(value, TranslatableContent): raise ValueError("Must be a TranslatableContent instance") # If no hint is specified, but we have a default, then set it value.hint = value.hint or self.field.hint # Replace the content attribute setattr(instance, CACHE_ATTRIBUTE, value) # If this is new, never before seen content, then _master_translation_id # will be None, so we don't want to set anything in the master translation # cache field if value._master_translation_id: # Update the instance master translation cache getattr(instance, MASTERS_CACHE_ATTR)[value._master_translation_id] = { 'hint': value.hint, 'text': value.text, 'lang': value.language_code, } # Make sure we update the underlying master translation appropriately super(TranslatableFieldDescriptor, self).__set__(instance, value._master_translation_id) setattr(cls, self.name, TranslatableFieldDescriptor(self))
class Translation(models.Model): master_translation = models.ForeignKey("fluent.MasterTranslation", editable=False, related_name="+") language_code = models.CharField(max_length=8, blank=False) plural_texts = JSONField(blank=True) # These are the various plural translations depending on the language denorm_master_text = models.TextField(editable=False) denorm_master_hint = models.CharField(max_length=500, editable=False) denorm_master_language = models.CharField(max_length=8, editable=False) master_text_hint_hash = models.CharField(max_length=64) class Meta: app_label = "fluent" @property def text(self): singular_form = get_plural_index(self.language_code, 1) try: return self.plural_texts[singular_form] except KeyError: # Some kind of corrupt data, so just return the source language return self.denorm_master_text @text.setter def text(self, value): singular_form = get_plural_index(self.language_code, 1) self.plural_texts[singular_form] = value def clean(self): msgs = validate_translation_texts(self) if msgs: raise ValidationError([err for err, _orig, _trans in msgs]) def __unicode__(self): return u"Translation of {} for {}".format(self.denorm_master_text, self.language_code) def __repr__(self): """ Define an ASCII string safe representation of the translation. """ return str("{}".format(self.id)) @staticmethod def generate_hash(master_text, master_hint): assert master_text assert master_hint is not None result = md5() for x in (master_text, master_hint): x = x.encode('utf-8') result.update(x) return result.hexdigest() def save(self, *args, **kwargs): assert self.language_code assert self.master_translation_id assert len(self.plural_texts) self.denorm_master_text = self.master_translation.text self.denorm_master_hint = self.master_translation.hint self.denorm_master_language = self.master_translation.language_code # For querying (you can't query for text on the datastore) self.master_text_hint_hash = Translation.generate_hash( self.denorm_master_text, self.denorm_master_hint ) return super(Translation, self).save(*args, **kwargs)
class MasterTranslation(models.Model): id = models.CharField(max_length=64, primary_key=True) text = models.TextField() text_for_ordering = ComputedCharField(lambda instance: instance.text[:500], max_length=500) plural_text = models.TextField(blank=True) hint = models.CharField(max_length=500, default="", blank=True) language_code = models.CharField( max_length=8, choices=settings.LANGUAGES, default=settings.LANGUAGE_CODE ) translations_by_language_code = JSONField() translations = RelatedSetField(Translation) translated_into_languages = SetField(models.CharField(max_length=8), editable=False) # Was this master translation updated or created by make messages? used_in_code_or_templates = models.BooleanField(default=False, blank=True, editable=False) # Were any groups specified in the trans tags? used_by_groups_in_code_or_templates = SetField(models.CharField(max_length=64), blank=True) # Record the ID of the last scan which updated this instance (if any) last_updated_by_scan_uuid = models.CharField(max_length=64, blank=True, default="") first_letter = models.CharField(max_length=1, editable=False) @property def is_plural(self): return bool(self.plural_text) def __unicode__(self): return u"{} ({}{})".format(self.text, self.language_code, ' plural' if self.is_plural else '') def __repr__(self): """ Define an ASCII string safe representation of the master translation. """ return str("{}".format(self.id)) def get_display(self): from fluent.trans import _get_trans result = _get_trans(self.text, self.hint) return result def text_for_language_code(self, lang_code): new_code = find_closest_supported_language(lang_code) if new_code not in self.translations_by_language_code.keys(): # we don't have a translation for this language return self.text translation_id = self.translations_by_language_code[new_code] translation = Translation.objects.get(id=translation_id) return translation.text @classmethod def find_by_groups(cls, groups): from .fields import find_installed_translatable_fields translatable_fields_by_model = find_installed_translatable_fields(with_groups=groups) # Go through all Translatable(Char|Text)Fields or TextFields marked with the specified group and get # all the master translation IDs which are set to them master_translation_ids = [] for model, fields in translatable_fields_by_model.items(): master_translation_ids.extend(chain(*model.objects.values_list(*[field.attname for field in fields]))) master_translation_ids = list(set(master_translation_ids)) # Now get all the master translations with a group specified in the templates master_translation_ids.extend( list(MasterTranslation.objects.filter(used_by_groups_in_code_or_templates__overlap=groups) .values_list("pk", flat=True)) ) # Make sure master translation ids don't include None values or duplicates master_translation_ids = set(master_translation_ids) master_translation_ids = master_translation_ids - {None} # Return them all! return MasterTranslation.objects.filter(pk__in=master_translation_ids) @classmethod def find_by_group(cls, group_name): return cls.find_by_groups([group_name]) @staticmethod def generate_key(text, hint, language_code): assert text assert hint is not None assert language_code result = md5() for x in (text.encode("utf-8"), hint.encode("utf-8"), language_code): result.update(x) return result.hexdigest() def save(self, *args, **kwargs): assert self.text assert self.language_code # Always store the first letter for querying self.first_letter = self.text.strip()[:1] # Generate the appropriate key on creation if self._state.adding: self.pk = MasterTranslation.generate_key( self.text, self.hint, self.language_code ) # If we are adding for the first time, then create a counterpart # translation for the master language. # Note that this Translation will be complete and correct only for the languages that # only require 2 plural forms - for others this language needs to be explicitly translated # or updated later. if self._state.adding: with transaction.atomic(xg=True): singular_form = get_plural_index(self.language_code, 1) plural_form = get_plural_index(self.language_code, 2) plurals = {singular_form: self.text} if self.is_plural: plurals[plural_form] = self.plural_text # if len(LANGUAGE_LOOKUPS[self.language_code].plurals_needed) > len(plurals): # FIXME: We can detect that we're dealing with a language that needs more plurals # What should we do? mark the translation as incomplete? # Don't create the translation object at all? new_trans = Translation.objects.create( master_translation=self, language_code=self.language_code, plural_texts=plurals, denorm_master_text=self.text, denorm_master_hint=self.hint ) self.translations_by_language_code[self.language_code] = new_trans.pk self.translations.add(new_trans) self.translated_into_languages = set(self.translations_by_language_code.keys()) return super(MasterTranslation, self).save(*args, **kwargs) else: # Otherwise just do a normal save self.translated_into_languages = set(self.translations_by_language_code.keys()) return super(MasterTranslation, self).save(*args, **kwargs) def create_or_update_translation(self, language_code, singular_text=None, plural_texts=None, validate=False): if language_code not in dict(settings.LANGUAGES).keys(): return ["'{}' is not included as a language in your settings file".format(language_code)] with transaction.atomic(xg=True): trans = None if language_code in self.translations_by_language_code: # We already have a translation for this language, update it! try: trans = Translation.objects.get(pk=self.translations_by_language_code[language_code]) created = False except Translation.DoesNotExist: trans = None if not trans: # OK, create the translation object and add it to the respective fields trans = Translation( master_translation_id=self.pk, language_code=language_code, denorm_master_hint=self.hint, denorm_master_text=self.text ) created = True if plural_texts: trans.plural_texts = plural_texts else: trans.text = singular_text if validate: errors = validate_translation_texts(trans, self) if errors: return errors trans.master_translation = self trans.save() if created: self.refresh_from_db() self.translations_by_language_code[language_code] = trans.pk self.translations.add(trans) self.save() class Meta: app_label = "fluent"
class Dog(models.Model): name = models.CharField(max_length=32) data = JSONField()
class NullableJSONFieldModel(models.Model): json_field = JSONField(null=True, blank=True)