class UserProfile(models.Model): user = models.OneToOneField(User) profile_id = UUIDField(auto=True) # Notification preferences # Right now these represent e-mail contact # Floodlight updates notify_updates = models.BooleanField( "Floodlight Updates", default=True, help_text= "Updates about new Floodlight features, events and storytelling tips") notify_admin = models.BooleanField( "Administrative Updates", default=True, help_text="Administrative account updates") notify_digest = models.BooleanField( "Monthly Digest", default=True, help_text="A monthly digest of featured Floodlight stories") # Notifications about my stories notify_story_featured = models.BooleanField( "Homepage Notification", default=True, help_text= "One of my stories is featured on the Floodlight homepage or newsletter" ) notify_story_comment = models.BooleanField( "Comment Notification", default=True, help_text="Someone comments on one of my stories") def __unicode__(self): return unicode(self.user)
class Place(node_factory('PlaceRelation')): """ A larger scale geographic area such as a neighborhood or zip code Places are related hierachically using a directed graph as a place can have multiple parents. """ name = ShortTextField(_("Name")) geolevel = models.ForeignKey(GeoLevel, null=True, blank=True, related_name='places', verbose_name=_("GeoLevel")) boundary = models.MultiPolygonField(blank=True, null=True, verbose_name=_("Boundary")) place_id = UUIDField(auto=True, verbose_name=_("Place ID"), db_index=True) slug = models.SlugField(blank=True) def get_absolute_url(self): return reverse('place_stories', kwargs={'slug': self.slug}) def __unicode__(self): return self.name
class Help(TranslatedModel): help_id = UUIDField(auto=True) slug = models.SlugField(blank=True) searchable = models.BooleanField(default=False) objects = HelpManager() translated_fields = ['body', 'title'] translation_set = 'helptranslation_set' translation_class = HelpTranslation class Meta: verbose_name_plural = "help items" def __unicode__(self): if self.title: return self.title return _("Help Item") + " " + self.help_id def natural_key(self): return (self.help_id,) @models.permalink def get_absolute_url(self): """Calculate the canonical URL for a Help item""" if self.slug: return ('help_detail', [self.slug]) return ('help_detail', [self.help_id])
class SectionLayout(TranslatedModel): TEMPLATE_CHOICES = [(name, name) for name in settings.STORYBASE_LAYOUT_TEMPLATES] layout_id = UUIDField(auto=True) template = models.CharField(_("template"), max_length=100, choices=TEMPLATE_CHOICES) containers = models.ManyToManyField('Container', related_name='layouts', blank=True) objects = SectionLayoutManager() # Class attributes to handle translation translated_fields = ['name'] translation_set = 'sectionlayouttranslation_set' translation_class = SectionLayoutTranslation def __unicode__(self): return self.name def get_template_filename(self): return "storybase_story/sectionlayouts/%s" % (self.template) def get_template_contents(self): template_filename = self.get_template_filename() return render_to_string(template_filename) def natural_key(self): return (self.layout_id, )
class StoryTemplate(TranslatedModel): """Metadata for a template used to create new stories""" TIME_NEEDED_CHOICES = ( ('5 minutes', _('5 minutes')), ('30 minutes', _('30 minutes')), ) template_id = UUIDField(auto=True) # The structure of the template comes from a story model instance story = models.ForeignKey('Story', blank=True, null=True) # The amount of time needed to create a story of this type time_needed = models.CharField(max_length=140, choices=TIME_NEEDED_CHOICES, blank=True) objects = StoryTemplateManager() # Class attributes to handle translation translated_fields = ['title', 'description', 'tag_line'] translation_set = 'storytemplatetranslation_set' translation_class = StoryTemplateTranslation def __unicode__(self): return self.title def natural_key(self): return (self.template_id, )
class FileUpload(amo.models.ModelBase): """Created when a file is uploaded for validation/submission.""" uuid = UUIDField(primary_key=True, auto=True) path = models.CharField(max_length=255, default='') name = models.CharField(max_length=255, default='', help_text="The user's original filename") hash = models.CharField(max_length=255, default='') user = models.ForeignKey('users.UserProfile', null=True) valid = models.BooleanField(default=False) is_webapp = models.BooleanField(default=False) validation = models.TextField(null=True) compat_with_app = models.ForeignKey(Application, null=True, related_name='uploads_compat_for_app') compat_with_appver = models.ForeignKey( AppVersion, null=True, related_name='uploads_compat_for_appver') task_error = models.TextField(null=True) objects = amo.models.UncachedManagerBase() class Meta(amo.models.ModelBase.Meta): db_table = 'file_uploads' def __unicode__(self): return self.uuid def save(self, *args, **kw): if self.validation: try: if json.loads(self.validation)['errors'] == 0: self.valid = True except Exception: log.error('Invalid validation json: %r' % self) super(FileUpload, self).save() def add_file(self, chunks, filename, size, is_webapp=False): filename = smart_str(filename) loc = os.path.join(settings.ADDONS_PATH, 'temp', uuid.uuid4().hex) base, ext = os.path.splitext(amo.utils.smart_path(filename)) if ext in EXTENSIONS: loc += ext log.info('UPLOAD: %r (%s bytes) to %r' % (filename, size, loc)) hash = hashlib.sha256() with storage.open(loc, 'wb') as fd: for chunk in chunks: hash.update(chunk) fd.write(chunk) self.path = loc self.name = filename self.hash = 'sha256:%s' % hash.hexdigest() self.is_webapp = is_webapp self.save() @classmethod def from_post(cls, chunks, filename, size, is_webapp=False): fu = FileUpload() fu.add_file(chunks, filename, size, is_webapp) return fu
class FileUpload(ModelBase): """Created when a file is uploaded for validation/submission.""" uuid = UUIDField(primary_key=True, auto=True) path = models.CharField(max_length=255, default='') name = models.CharField(max_length=255, default='', help_text="The user's original filename") hash = models.CharField(max_length=255, default='') user = models.ForeignKey('users.UserProfile', null=True) valid = models.BooleanField(default=False) validation = models.TextField(null=True) task_error = models.TextField(null=True) objects = UncachedManagerBase() class Meta(ModelBase.Meta): db_table = 'file_uploads' def __unicode__(self): return self.uuid def save(self, *args, **kw): if self.validation: try: if json.loads(self.validation)['errors'] == 0: self.valid = True except Exception: log.error('Invalid validation json: %r' % self) super(FileUpload, self).save() def add_file(self, chunks, filename, size): filename = smart_str(filename) loc = os.path.join(settings.ADDONS_PATH, 'temp', uuid.uuid4().hex) base, ext = os.path.splitext(amo.utils.smart_path(filename)) if ext in EXTENSIONS: loc += ext log.info('UPLOAD: %r (%s bytes) to %r' % (filename, size, loc)) hash = hashlib.sha256() # The buffer might have been read before, so rewind back at the start. if hasattr(chunks, 'seek'): chunks.seek(0) with storage.open(loc, 'wb') as fd: for chunk in chunks: hash.update(chunk) fd.write(chunk) self.path = loc self.name = filename self.hash = 'sha256:%s' % hash.hexdigest() self.save() @classmethod def from_post(cls, chunks, filename, size, **kwargs): fu = FileUpload(**kwargs) fu.add_file(chunks, filename, size) return fu @property def processed(self): return bool(self.valid or self.validation)
def handle(self, *args, **options): poolrooms = Poolroom.objects.filter(Q(uuid__isnull=True) | Q(uuid='')) for poolroom in poolrooms: self.stdout.write('Generating UUID for poolroom "%s".\n' %(poolroom.name)) poolroom.uuid = UUIDField(hyphenate=True)._create_uuid() poolroom.save() self.stdout.write('New UUID for poolroom "%s" is "%s".\n' %(poolroom.name, poolroom.uuid))
class RssKey(models.Model): key = UUIDField(db_column='rsskey', auto=True, unique=True) addon = models.ForeignKey(Addon, null=True, unique=True) user = models.ForeignKey(UserProfile, null=True, unique=True) created = models.DateField(default=datetime.now) class Meta: db_table = 'hubrsskeys'
class Tag(TagPermission, TagBase): tag_id = UUIDField(auto=True) def get_absolute_url(self): return reverse('tag_stories', kwargs={'slug': self.slug}) class Meta: verbose_name = _("Tag") verbose_name_plural = _("Tags")
class TranslationModel(models.Model): """Base class for model that encapsulates translated fields""" translation_id = UUIDField(auto=True) language = models.CharField(max_length=15, choices=settings.LANGUAGES, default=settings.LANGUAGE_CODE) class Meta: """Model metadata options""" abstract = True
class Project(TranslatedModel, TimestampedModel): """ A project that collects related stories. Users can also be related to projects. """ project_id = UUIDField(auto=True) slug = models.SlugField(blank=True) website_url = models.URLField(blank=True) organizations = models.ManyToManyField(Organization, related_name='projects', blank=True) members = models.ManyToManyField(User, related_name='projects', blank=True) curated_stories = models.ManyToManyField( 'storybase_story.Story', related_name='curated_in_projects', blank=True, through='ProjectStory') on_homepage = models.BooleanField(_("Featured on homepage"), default=False) objects = FeaturedManager() translated_fields = ['name', 'description'] translation_set = 'projecttranslation_set' def __unicode__(self): return self.name @models.permalink def get_absolute_url(self): return ('project_detail', [self.slug]) def add_story(self, story, weight=0): """ Associate a story with the Project Arguments: story -- The Story model instance object to be associated weight -- The ordering of the story relative to other stories """ ProjectStory.objects.create(project=self, story=story, weight=weight) def ordered_stories(self): """ Return sorted curated stories This is a helper method to make it easy to access a sorted list of stories associated with the project in a template. Sorts first by weight, then by when a story was associated with the project in reverse chronological order. """ return self.curated_stories.order_by('projectstory__weight', '-projectstory__added')
class StoryRelation(StoryRelationPermission, models.Model): """Relationship between two stories""" RELATION_TYPES = (('connected', u"Connected Story"), ) DEFAULT_TYPE = 'connected' relation_id = UUIDField(auto=True) relation_type = models.CharField(max_length=25, choices=RELATION_TYPES, default=DEFAULT_TYPE) source = models.ForeignKey(Story, related_name="target") target = models.ForeignKey(Story, related_name="source")
class Location(LocationPermission, DirtyFieldsMixin, models.Model): """A location with a specific address or latitude and longitude""" location_id = UUIDField(auto=True, verbose_name=_("Location ID"), db_index=True) name = ShortTextField(_("Name"), blank=True) address = ShortTextField(_("Address"), blank=True) address2 = ShortTextField(_("Address 2"), blank=True) city = models.CharField(_("City"), max_length=255, blank=True) state = models.CharField(_("State"), max_length=255, blank=True, choices=STATE_CHOICES) postcode = models.CharField(_("Postal Code"), max_length=255, blank=True) lat = models.FloatField(_("Latitude"), blank=True, null=True) lng = models.FloatField(_("Longitude"), blank=True, null=True) point = models.PointField(_("Point"), blank=True, null=True) # I'm not sure what the best solution for parsing addresses is, or # what the best geocoder is for our application, or how users are # going to use this feature. So rather than spending a bunch of time # writing/testing an address parser (or picking a particular geocoder # that breaks an address into pieces), just have a place to store # the raw address provided by the user. This will, at the very least, # give us a domain-specific set of addresses to test against. raw = models.TextField(_("Raw Address"), blank=True) owner = models.ForeignKey(User, related_name="locations", blank=True, null=True) objects = models.GeoManager() def __unicode__(self): if self.name: unicode_rep = u"%s" % self.name elif self.address or self.city or self.state or self.postcode: unicode_rep = u", ".join([self.address, self.city, self.state]) unicode_rep = u" ".join([unicode_rep, self.postcode]) else: return u"Location %s" % self.location_id return unicode_rep def _geocode(self, address): point = None geocoder = get_geocoder() # There might be more than one matching location. For now, just # assume the first one. results = list(geocoder.geocode(address, exactly_one=False)) if results: place, (lat, lng) = results[0] point = (lat, lng) return point
class UserGroup(models.Model): id = UUIDField(auto=True, primary_key=True) name = models.CharField(max_length=16, blank=False, verbose_name=u'名称') need_credits = models.PositiveIntegerField(verbose_name=u'需要积分', default=0, unique=True) icon = models.CharField(max_length=200, verbose_name=u'图标', default='') read_level = models.PositiveSmallIntegerField(default=1, verbose_name=u'阅读权限') # 1为初始注册用户权限 can_ip = models.BooleanField(default=False, verbose_name=u'查看IP') objects = UserGroupManager() class Meta: db_table = 'user_group'
class StoryTemplate(TranslatedModel): """Metadata for a template used to create new stories""" TIME_NEEDED_CHOICES = ( ('5 minutes', _('5 minutes')), ('30 minutes', _('30 minutes')), ) LEVEL_CHOICES = (('beginner', _("Beginner")), ) template_id = UUIDField(auto=True, db_index=True) story = models.ForeignKey( 'Story', blank=True, null=True, help_text=_("The story that provides the structure for this " "template")) time_needed = models.CharField( max_length=140, choices=TIME_NEEDED_CHOICES, blank=True, help_text=_("The amount of time needed to create a story of this " "type")) level = models.CharField( max_length=140, choices=LEVEL_CHOICES, blank=True, help_text=_("The level of storytelling experience suggested to " "create stories with this template")) slug = models.SlugField(unique=True, help_text=_("A human-readable unique identifier")) examples = models.ManyToManyField( 'Story', blank=True, null=True, help_text=_("Stories that are examples of this template"), related_name="example_for") objects = StoryTemplateManager() # Class attributes to handle translation translated_fields = [ 'title', 'description', 'tag_line', 'ingredients', 'best_for', 'tip' ] translation_set = 'storytemplatetranslation_set' translation_class = StoryTemplateTranslation def __unicode__(self): return self.title def natural_key(self): return (self.template_id, )
class DataSet(TranslatedModel, PublishedModel, TimestampedModel, DataSetPermission): """ A set of data related to a story or used to produce a visualization included in a story This is a base class that provides common metadata for the data set. However, it does not provide the fields that specify the content itself. When creating a data set, one shouldn't instatniate this class, but instead use one of the model classes that inherits from DataSet. """ dataset_id = UUIDField(auto=True, db_index=True) source = models.TextField(blank=True) attribution = models.TextField(blank=True) links_to_file = models.BooleanField(_("Links to file"), default=True) """ Whether the dataset links to a file that can be downloaded or to a view of the data or a page describing the data. """ owner = models.ForeignKey(User, related_name="datasets", blank=True, null=True) # dataset_created is when the data set itself was created dataset_created = models.DateTimeField(blank=True, null=True) """ When the data set itself was created (possibly in non-digital form) """ translation_set = 'storybase_asset_datasettranslation_related' translated_fields = ['title', 'description'] translation_class = DataSetTranslation # Use InheritanceManager from django-model-utils to make # fetching of subclassed objects easier objects = InheritanceManager() def __unicode__(self): return self.title @models.permalink def get_absolute_url(self): return ('dataset_detail', [str(self.dataset_id)]) @property def download_url(self): """Returns the URL to the downloadable version of the data set""" raise NotImplemented
class FileUpload(amo.models.ModelBase): """Created when a file is uploaded for validation/submission.""" uuid = UUIDField(primary_key=True, auto=True) path = models.CharField(max_length=255) name = models.CharField(max_length=255, help_text="The user's original filename") hash = models.CharField(max_length=255, default='') user = models.ForeignKey('users.UserProfile', null=True) valid = models.BooleanField(default=False) validation = models.TextField(null=True) task_error = models.TextField(null=True) objects = amo.models.UncachedManagerBase() class Meta(amo.models.ModelBase.Meta): db_table = 'file_uploads' def __unicode__(self): return self.uuid def save(self, *args, **kw): if self.validation: try: if json.loads(self.validation)['errors'] == 0: self.valid = True except Exception: log.error('Invalid validation json: %r' % self) super(FileUpload, self).save() @classmethod def from_post(cls, chunks, filename, size): filename = smart_str(filename) loc = path.path(settings.ADDONS_PATH) / 'temp' / uuid.uuid4().hex if not loc.dirname().exists(): loc.dirname().makedirs() ext = path.path(filename).ext if ext in EXTENSIONS: loc += ext log.info('UPLOAD: %r (%s bytes) to %r' % (filename, size, loc)) hash = hashlib.sha256() with open(loc, 'wb') as fd: for chunk in chunks: hash.update(chunk) fd.write(chunk) return cls.objects.create(path=loc, name=filename, hash='sha256:%s' % hash.hexdigest())
class ContainerTemplate(models.Model): """Per-asset configuration for template assets in builder""" container_template_id = UUIDField(auto=True, db_index=True) template = models.ForeignKey('StoryTemplate') section = models.ForeignKey('Section') container = models.ForeignKey('Container') asset_type = models.CharField(max_length=10, choices=ASSET_TYPES, blank=True, help_text=_("Default asset type")) can_change_asset_type = models.BooleanField( default=False, help_text=_("User can change the asset type from the default")) help = models.ForeignKey(Help, blank=True, null=True) def __unicode__(self): return "%s / %s / %s" % (self.template.title, self.section.title, self.container.name)
class Help(TranslatedModel): help_id = UUIDField(auto=True) slug = models.SlugField(blank=True) objects = HelpManager() translated_fields = ['body', 'title'] translation_set = 'helptranslation_set' translation_class = HelpTranslation class Meta: verbose_name_plural = "help items" def __unicode__(self): if self.title: return self.title return _("Help Item") + " " + self.help_id def natural_key(self): return (self.help_id, )
class CommunicationThreadToken(amo.models.ModelBase): thread = models.ForeignKey(CommunicationThread, related_name='token') user = models.ForeignKey('users.UserProfile', related_name='comm_thread_tokens') uuid = UUIDField(unique=True, auto=True) use_count = models.IntegerField(default=0, help_text='Stores the number of times the token has been used') class Meta: db_table = 'comm_thread_tokens' unique_together = ('thread', 'user') def is_valid(self): # TODO: Confirm the expiration and max use count values. timedelta = datetime.now() - self.modified return (timedelta.days <= comm.THREAD_TOKEN_EXPIRY and self.use_count < comm.MAX_TOKEN_USE_COUNT) def reset_uuid(self): # Generate a new UUID. self.uuid = UUIDField()._create_uuid().hex
class FileUpload(amo.models.ModelBase): """Created when a file is uploaded for validation/submission.""" uuid = UUIDField(primary_key=True, auto=True) path = models.CharField(max_length=255, default='') name = models.CharField(max_length=255, default='', help_text="The user's original filename") hash = models.CharField(max_length=255, default='') user = models.ForeignKey('users.UserProfile', null=True) valid = models.BooleanField(default=False) validation = models.TextField(null=True) automated_signing = models.BooleanField(default=False) compat_with_app = models.PositiveIntegerField( choices=amo.APPS_CHOICES, db_column="compat_with_app_id", null=True) compat_with_appver = models.ForeignKey( AppVersion, null=True, related_name='uploads_compat_for_appver') objects = amo.models.UncachedManagerBase() class Meta(amo.models.ModelBase.Meta): db_table = 'file_uploads' def __unicode__(self): return self.uuid def save(self, *args, **kw): if self.validation: if json.loads(self.validation)['errors'] == 0: self.valid = True super(FileUpload, self).save() def add_file(self, chunks, filename, size): filename = smart_str(filename) loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex) base, ext = os.path.splitext(amo.utils.smart_path(filename)) if ext in EXTENSIONS: loc += ext log.info('UPLOAD: %r (%s bytes) to %r' % (filename, size, loc)) hash = hashlib.sha256() with storage.open(loc, 'wb') as fd: for chunk in chunks: hash.update(chunk) fd.write(chunk) self.path = loc self.name = filename self.hash = 'sha256:%s' % hash.hexdigest() self.save() @classmethod def from_post(cls, chunks, filename, size): fu = FileUpload() fu.add_file(chunks, filename, size) return fu @property def processed(self): return bool(self.valid or self.validation) @property def validation_timeout(self): if self.processed: validation = json.loads(self.validation) messages = validation['messages'] timeout_id = [ 'validator', 'unexpected_exception', 'validation_timeout' ] return any(msg['id'] == timeout_id for msg in messages) else: return False @property def processed_validation(self): """Return processed validation results as expected by the frontend.""" if self.validation: # Import loop. from devhub.utils import process_validation validation = json.loads(self.validation) is_compatibility = self.compat_with_app is not None return process_validation(validation, is_compatibility, self.hash)
class Extension(ModelBase): # Automatically handled fields. deleted = models.BooleanField(default=False, editable=False) icon_hash = models.CharField(max_length=8, blank=True) last_updated = models.DateTimeField(blank=True, null=True, editable=False) status = models.PositiveSmallIntegerField(choices=STATUS_CHOICES.items(), default=STATUS_NULL, editable=False) uuid = UUIDField(auto=True, editable=False) # Fields for which the manifest is the source of truth - can't be # overridden by the API. author = models.CharField(default='', editable=False, max_length=128) default_language = models.CharField(default=settings.LANGUAGE_CODE, editable=False, max_length=10) description = TranslatedField(default=None, editable=False) name = TranslatedField(default=None, editable=False) # Fields that can be modified using the API. authors = models.ManyToManyField('users.UserProfile') disabled = models.BooleanField(default=False) slug = models.CharField(max_length=35, null=True, unique=True) objects = ManagerBase.from_queryset(ExtensionQuerySet)() manifest_is_source_of_truth_fields = ('author', 'description', 'default_language', 'name') class Meta: ordering = ('-id', ) index_together = (('deleted', 'disabled', 'status'), ) @cached_property(writable=True) def latest_public_version(self): return self.versions.without_deleted().public().latest('pk') @cached_property(writable=True) def latest_version(self): return self.versions.without_deleted().latest('pk') def block(self): """Block this Extension. When in this state the Extension should not be editable by the developers at all; not visible publicly; not searchable by users; but should be shown in the developer's dashboard, as 'Blocked'.""" self.update(status=STATUS_BLOCKED) def clean_slug(self): return clean_slug(self, slug_field='slug') def delete(self, *args, **kwargs): """Delete this instance. By default, a soft-delete is performed, only hiding the instance from the custom manager methods without actually removing it from the database. pre_delete and post_delete signals are *not* sent in that case. The slug will be set to None during the process. Can be overridden by passing `hard_delete=True` keyword argument, in which case it behaves like a regular delete() call instead.""" if self.is_blocked(): raise BlockedExtensionError hard_delete = kwargs.pop('hard_delete', False) if hard_delete: # Real, hard delete. return super(Extension, self).delete(*args, **kwargs) # Soft delete. # Since we have a unique constraint with slug, set it to None when # deleting. Undelete should re-generate it - it might differ from the # original slug, but that's why you should be careful when deleting... self.update(deleted=True, slug=None) @property def devices(self): """Device ids the Extension is compatible with. For now, hardcoded to only return Firefox OS.""" return [DEVICE_GAIA.id] @property def device_names(self): """Device names the Extension is compatible with. Used by the API.""" return [DEVICE_TYPES[device_id].api_name for device_id in self.devices] @classmethod def extract_and_validate_upload(cls, upload): """Validate and extract manifest from a FileUpload instance. Can raise ParseError.""" with private_storage.open(upload.path) as file_obj: # The file will already have been uploaded at this point, so force # the content type to make the ExtensionValidator happy. We just # need to validate the contents. file_obj.content_type = 'application/zip' manifest_contents = ExtensionValidator(file_obj).validate() return manifest_contents @classmethod def extract_manifest_fields(cls, manifest_data, fields=None): """Extract the specified `fields` from `manifest_data`, applying transformations if necessary. If `fields` is absent, then use `cls.manifest_is_source_of_truth_fields`.""" if fields is None: fields = cls.manifest_is_source_of_truth_fields data = {k: manifest_data[k] for k in fields if k in manifest_data} # Determine default language to use for translations. # Web Extensions Manifest contains locales (e.g. "en_US"), not # languages (e.g. "en-US"). The field is also called differently as a # result (default_locale vs default_language), so we need to transform # both the key and the value before adding it to data. A default value # needs to be set to correctly generate the translated fields below. default_language = to_language( manifest_data.get('default_locale', cls._meta.get_field('default_language').default)) if 'default_language' in fields: data['default_language'] = default_language # Be nice and strip leading / trailing whitespace chars from # strings. for key, value in data.items(): if isinstance(value, basestring): data[key] = value.strip() # Translated fields should not be extracted as simple strings, # otherwise we end up setting a locale on the translation that is # dependent on the locale of the thread. Use dicts instead, always # setting default_language as the language for now (since we don't # support i18n in web extensions yet). for field in cls._meta.translated_fields: field_name = field.name if field_name in data: data[field_name] = { default_language: manifest_data[field_name] } return data @classmethod def from_upload(cls, upload, user=None): """Handle creating/editing the Extension instance and saving it to db, as well as file operations, from a FileUpload instance. Can throw a ParseError or SigningError, so should always be called within a try/except.""" manifest_contents = cls.extract_and_validate_upload(upload) data = cls.extract_manifest_fields(manifest_contents) # Check for name collision in the same locale for the same user. default_language = data['default_language'] name = data['name'][default_language] if user.extension_set.without_deleted().filter( name__locale=default_language, name__localized_string=name).exists(): raise ParseError( _(u'An Add-on with the same name already exists in your ' u'submissions.')) # Build a new instance. instance = cls.objects.create(**data) # Now that the instance has been saved, we can add the author and start # saving version data. If everything checks out, a status will be set # on the ExtensionVersion we're creating which will automatically be # replicated on the Extension instance. instance.authors.add(user) version = ExtensionVersion.from_upload( upload, parent=instance, manifest_contents=manifest_contents) # Trigger icon fetch task asynchronously if necessary now that we have # an extension and a version. if 'icons' in manifest_contents: fetch_icon.delay(instance.pk, version.pk) return instance @classmethod def get_fallback(cls): """Class method returning the field holding the default language to use in translations for this instance. *Needs* to be called get_fallback() and *needs* to be a classmethod, that's what the translations app requires.""" return cls._meta.get_field('default_language') def get_icon_dir(self): return os.path.join(settings.EXTENSION_ICONS_PATH, str(self.pk / 1000)) def get_icon_url(self, size): return get_icon_url(static_url('EXTENSION_ICON_URL'), self, size) @classmethod def get_indexer(cls): return ExtensionIndexer def get_url_path(self): return reverse('extension.detail', kwargs={'app_slug': self.slug}) @property def icon_type(self): return 'png' if self.icon_hash else '' def is_blocked(self): return self.status == STATUS_BLOCKED def is_dummy_content_for_qa(self): """ Returns whether this extension is a dummy extension used for testing only or not. Used by mkt.search.utils.extract_popularity_trending_boost() - the method needs to exist, but we are not using it yet. """ return False def is_public(self): return (not self.deleted and not self.disabled and self.status == STATUS_PUBLIC) @property def mini_manifest(self): """Mini-manifest used for install/update on FxOS devices, in dict form. It follows the Mozilla App Manifest format (because that's what FxOS requires to install/update add-ons), *not* the Web Extension manifest format. """ if self.is_blocked(): return {} # Platform "translates" back the mini-manifest into an app manifest and # verifies that some specific key properties in the real manifest match # what's found in the mini-manifest. To prevent manifest mismatch # errors, we need to copy those properties from the real manifest: # name, description and author. To help Firefox OS display useful info # to the user we also copy content_scripts and version. # We don't bother with locales at the moment, this probably breaks # extensions using https://developer.chrome.com/extensions/i18n but # we'll deal with that later. try: version = self.latest_public_version except ExtensionVersion.DoesNotExist: return {} manifest = version.manifest mini_manifest = { # 'id' here is the uuid, like in sign_file(). This is used by # platform to do blocklisting. 'id': self.uuid, 'name': manifest['name'], 'package_path': version.download_url, 'size': version.size, 'version': manifest['version'] } if 'author' in manifest: # author is copied as a different key to match app manifest format. mini_manifest['developer'] = {'name': manifest['author']} if 'content_scripts' in manifest: mini_manifest['content_scripts'] = manifest['content_scripts'] if 'description' in manifest: mini_manifest['description'] = manifest['description'] return mini_manifest @property def mini_manifest_url(self): return absolutify( reverse('extension.mini_manifest', kwargs={'uuid': self.uuid})) def save(self, *args, **kwargs): if not self.deleted: # Always clean slug before saving, to avoid clashes. self.clean_slug() return super(Extension, self).save(*args, **kwargs) def __unicode__(self): return u'%s: %s' % (self.pk, self.name) def unblock(self): """Unblock this Extension. The original status is restored.""" self.status = STATUS_NULL self.update_status_according_to_versions() def undelete(self): """Undelete this instance, making it available to all manager methods again and restoring its version number. Return False if it was not marked as deleted, True otherwise. Will re-generate a slug, that might differ from the original one if it was taken in the meantime.""" if not self.deleted: return False self.clean_slug() self.update(deleted=False, slug=self.slug) return True def update_manifest_fields_from_latest_public_version(self): """Update all fields for which the manifest is the source of truth with the manifest from the latest public add-on.""" if self.is_blocked(): raise BlockedExtensionError try: version = self.latest_public_version except ExtensionVersion.DoesNotExist: return if not version.manifest: return # Trigger icon fetch task asynchronously if necessary now that we have # an extension and a version. if 'icons' in version.manifest: fetch_icon.delay(self.pk, version.pk) # We need to re-extract the fields from manifest contents because some # fields like default_language are transformed before being stored. data = self.extract_manifest_fields(version.manifest) return self.update(**data) def update_status_according_to_versions(self): """Update `status`, `latest_version` and `latest_public_version` properties depending on the `status` on the ExtensionVersion instances attached to this Extension.""" if self.is_blocked(): raise BlockedExtensionError # If there is a public version available, the extension should be # public. If not, and if there is a pending version available, it # should be pending. If not, and if there is a rejected version # available, it should be rejected. Otherwise it should just be # incomplete. versions = self.versions.without_deleted() if versions.public().exists(): self.update(status=STATUS_PUBLIC) elif versions.pending().exists(): self.update(status=STATUS_PENDING) elif versions.rejected().exists(): self.update(status=STATUS_REJECTED) else: self.update(status=STATUS_NULL) # Delete latest_version and latest_public_version properties, since # they are writable cached_properties they will be reset the next time # they are accessed. try: if self.latest_version: del self.latest_version except ExtensionVersion.DoesNotExist: pass try: if self.latest_public_version: del self.latest_public_version except ExtensionVersion.DoesNotExist: pass
class LangPack(ModelBase): # Primary key is a uuid in order to be able to set it in advance (we need # something unique for the filename, and we don't have a slug). uuid = UUIDField(primary_key=True, auto=True) # Fields for which the manifest is the source of truth - can't be # overridden by the API. language = models.CharField(choices=LANGUAGE_CHOICES, default=settings.LANGUAGE_CODE, max_length=10) fxos_version = models.CharField(max_length=255, default='') version = models.CharField(max_length=255, default='') manifest = models.TextField() # Fields automatically set when uploading files. file_version = models.PositiveIntegerField(default=0) # Fields that can be modified using the API. active = models.BooleanField(default=False) # Note: we don't need to link a LangPack to an user right now, but in the # future, if we want to do that, call it user (single owner) or authors # (multiple authors) to be compatible with the API permission classes. class Meta: ordering = (('language'), ) index_together = (('fxos_version', 'active', 'language'), ) @property def filename(self): return '%s-%s.zip' % (self.uuid, self.version) @property def path_prefix(self): return os.path.join(settings.ADDONS_PATH, 'langpacks', str(self.pk)) @property def file_path(self): return os.path.join(self.path_prefix, nfd_str(self.filename)) @property def download_url(self): url = ('%s/langpack.zip' % reverse('downloads.langpack', args=[unicode(self.pk)])) return absolutify(url) @property def manifest_url(self): """Return URL to the minifest for the langpack""" if self.active: return absolutify( reverse('langpack.manifest', args=[unicode(UUID(self.pk))])) return '' def __unicode__(self): return u'%s (%s)' % (self.get_language_display(), self.fxos_version) def is_public(self): return self.active def get_package_path(self): return self.download_url def get_minifest_contents(self, force=False): """Return the "mini" manifest + etag for this langpack, caching it in the process. Call this with `force=True` whenever we need to update the cached version of this manifest, e.g., when a new version of the langpack has been pushed.""" return get_cached_minifest(self, force=force) def get_manifest_json(self): """Return the json representation of the (full) manifest for this langpack, as stored when it was uploaded.""" return json.loads(self.manifest) def reset_uuid(self): self.uuid = self._meta.get_field('uuid')._create_uuid() def handle_file_operations(self, upload): """Handle file operations on an instance by using the FileUpload object passed to set filename, file_version on the LangPack instance, and moving the temporary file to its final destination.""" upload.path = smart_path(nfd_str(upload.path)) if not self.uuid: self.reset_uuid() if storage.exists(self.filename): # The filename should not exist. If it does, it means we are trying # to re-upload the same version. This should have been caught # before, so just raise an exception. raise RuntimeError( 'Trying to upload a file to a destination that already exists') self.file_version = self.file_version + 1 # Because we are only dealing with langpacks generated by Mozilla atm, # we can directly sign the file before copying it to its final # destination. The filename changes with the version, so when a new # file is uploaded we should still be able to serve the old one until # the new info is stored in the db. self.sign_and_move_file(upload) def sign_and_move_file(self, upload): ids = json.dumps({ # 'id' needs to be unique for a given langpack, but should not # change when there is an update. 'id': self.pk, # 'version' should be an integer and should be monotonically # increasing. 'version': self.file_version }) with statsd.timer('langpacks.sign'): try: # This will read the upload.path file, generate a signature # and write the signed file to self.file_path. sign_app(storage.open(upload.path), self.file_path, ids) except SigningError: log.info('[LangPack:%s] Signing failed' % self.pk) if storage.exists(self.file_path): storage.delete(self.file_path) raise @classmethod def from_upload(cls, upload, instance=None): """Handle creating/editing the LangPack instance and saving it to db, as well as file operations, from a FileUpload instance. Can throw a ValidationError or SigningError, so should always be called within a try/except.""" parser = LanguagePackParser(instance=instance) data = parser.parse(upload) allowed_fields = ('language', 'fxos_version', 'version') data = dict((k, v) for k, v in data.items() if k in allowed_fields) data['manifest'] = json.dumps(parser.get_json_data(upload)) if instance: # If we were passed an instance, override fields on it using the # data from the uploaded package. instance.__dict__.update(**data) else: # Build a new instance. instance = cls(**data) # Do last-minute validation that requires an instance. cls._meta.get_field('language').validate(instance.language, instance) # Fill in fields depending on the file contents, and move the file. instance.handle_file_operations(upload) # Save! instance.save() # Bust caching of manifest by passing force=True. instance.get_minifest_contents(force=True) return instance
def reset_uuid(self): # Generate a new UUID. self.uuid = UUIDField()._create_uuid().hex
class FileUpload(ModelBase): """Created when a file is uploaded for validation/submission.""" uuid = UUIDField(auto=True) path = models.CharField(max_length=255, default='') name = models.CharField(max_length=255, default='', help_text="The user's original filename") hash = models.CharField(max_length=255, default='') user = models.ForeignKey('users.UserProfile', null=True) valid = models.BooleanField(default=False) validation = models.TextField(null=True) automated_signing = models.BooleanField(default=False) compat_with_app = models.PositiveIntegerField( choices=amo.APPS_CHOICES, db_column="compat_with_app_id", null=True) compat_with_appver = models.ForeignKey( AppVersion, null=True, related_name='uploads_compat_for_appver') # Not all FileUploads will have a version and addon but it will be set # if the file was uploaded using the new API. version = models.CharField(max_length=255, null=True) addon = models.ForeignKey('addons.Addon', null=True) objects = UncachedManagerBase() class Meta(ModelBase.Meta): db_table = 'file_uploads' def __unicode__(self): return self.uuid def save(self, *args, **kw): if self.validation: if self.load_validation()['errors'] == 0: self.valid = True super(FileUpload, self).save(*args, **kw) def add_file(self, chunks, filename, size): if not self.uuid: self.uuid = self._meta.get_field('uuid')._create_uuid().hex filename = smart_str(u'{0}_{1}'.format(self.uuid, filename)) loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex) base, ext = os.path.splitext(smart_path(filename)) is_crx = False # Change a ZIP to an XPI, to maintain backward compatibility # with older versions of Firefox and to keep the rest of the XPI code # path as consistent as possible for ZIP uploads. # See: https://github.com/mozilla/addons-server/pull/2785 if ext == '.zip': ext = '.xpi' # If the extension is a CRX, we need to do some actual work to it # before we just convert it to an XPI. We strip the header from the # CRX, then it's good; see more about the CRX file format here: # https://developer.chrome.com/extensions/crx if ext == '.crx': ext = '.xpi' is_crx = True if ext in EXTENSIONS: loc += ext log.info('UPLOAD: %r (%s bytes) to %r' % (filename, size, loc)) if is_crx: hash = write_crx_as_xpi(chunks, storage, loc) else: hash = hashlib.sha256() with storage.open(loc, 'wb') as file_destination: for chunk in chunks: hash.update(chunk) file_destination.write(chunk) self.path = loc self.name = filename self.hash = 'sha256:%s' % hash.hexdigest() self.save() @classmethod def from_post(cls, chunks, filename, size, **params): upload = FileUpload(**params) upload.add_file(chunks, filename, size) return upload @property def processed(self): return bool(self.valid or self.validation) @property def validation_timeout(self): if self.processed: validation = self.load_validation() messages = validation['messages'] timeout_id = [ 'validator', 'unexpected_exception', 'validation_timeout' ] return any(msg['id'] == timeout_id for msg in messages) else: return False @property def processed_validation(self): """Return processed validation results as expected by the frontend.""" if self.validation: # Import loop. from olympia.devhub.utils import process_validation validation = self.load_validation() is_compatibility = self.compat_with_app is not None return process_validation(validation, is_compatibility, self.hash) @property def passed_all_validations(self): return self.processed and self.valid @property def passed_auto_validation(self): return self.load_validation()['passed_auto_validation'] def load_validation(self): return json.loads(self.validation)
class IARCRequest(ModelBase): app = models.OneToOneField(Webapp, related_name='iarc_request') uuid = UUIDField(auto=True, editable=False) class Meta: db_table = 'iarc_request'
class FileUpload(amo.models.ModelBase): """Created when a file is uploaded for validation/submission.""" uuid = UUIDField(primary_key=True, auto=True) path = models.CharField(max_length=255, default='') name = models.CharField(max_length=255, default='', help_text="The user's original filename") hash = models.CharField(max_length=255, default='') user = models.ForeignKey('users.UserProfile', null=True) valid = models.BooleanField(default=False) is_webapp = models.BooleanField(default=False) validation = models.TextField(null=True) _escaped_validation = models.TextField(null=True, db_column='escaped_validation') compat_with_app = models.PositiveIntegerField( choices=amo.APPS_CHOICES, db_column="compat_with_app_id", null=True) compat_with_appver = models.ForeignKey( AppVersion, null=True, related_name='uploads_compat_for_appver') task_error = models.TextField(null=True) objects = amo.models.UncachedManagerBase() class Meta(amo.models.ModelBase.Meta): db_table = 'file_uploads' def __unicode__(self): return self.uuid def save(self, *args, **kw): if self.validation: try: if json.loads(self.validation)['errors'] == 0: self.valid = True except Exception: log.error('Invalid validation json: %r' % self) self._escape_validation() super(FileUpload, self).save() def add_file(self, chunks, filename, size): filename = smart_str(filename) loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex) base, ext = os.path.splitext(amo.utils.smart_path(filename)) if ext in EXTENSIONS: loc += ext log.info('UPLOAD: %r (%s bytes) to %r' % (filename, size, loc)) hash = hashlib.sha256() with storage.open(loc, 'wb') as fd: for chunk in chunks: hash.update(chunk) fd.write(chunk) self.path = loc self.name = filename self.hash = 'sha256:%s' % hash.hexdigest() self.save() @classmethod def from_post(cls, chunks, filename, size): fu = FileUpload() fu.add_file(chunks, filename, size) return fu @property def processed(self): return bool(self.valid or self.validation) def escaped_validation(self, is_compatibility=False): """ The HTML-escaped validation results limited to a message count of `settings.VALIDATOR_MESSAGE_LIMIT` and optionally prepared for a compatibility report if `is_compatibility` is `True`. If `_escaped_validation` is set it will be used, otherwise `_escape_validation` will be called to escape the validation. """ if self.validation and not self._escaped_validation: self._escape_validation() if not self._escaped_validation: return '' return limit_validation_results(json.loads(self._escaped_validation), is_compatibility=is_compatibility) def _escape_validation(self): """ HTML-escape `validation` to `_escaped_validation`. This will raise a ValueError if `validation` is not valid JSON. """ try: validation = json.loads(self.validation) except ValueError: tb = traceback.format_exception(*sys.exc_info()) self.update(task_error=''.join(tb)) else: escaped_validation = escape_validation(validation) self._escaped_validation = json.dumps(escaped_validation)
class Asset(TranslatedModel, LicensedModel, PublishedModel, TimestampedModel, AssetPermission): """A piece of content included in a story An asset could be an image, a block of text, an embedded resource represented by an HTML snippet or a media file. This is a base class that provides common metadata for the asset. However, it does not provide the fields that specify the content itself. Also, to reduce the number of database tables and queries this model class does not provide translated metadata fields. When creating an asset, one shouldn't instantiate this class, but instead use one of the model classes that inherits form Asset. """ asset_id = UUIDField(auto=True) type = models.CharField(max_length=10, choices=ASSET_TYPES) attribution = models.TextField(blank=True) source_url = models.URLField(blank=True) """The URL where an asset originated. It could be used to store the canonical URL for a resource that is not yet oEmbedable or the canonical URL of an article or tweet where text is quoted from. """ owner = models.ForeignKey(User, related_name="assets", blank=True, null=True) section_specific = models.BooleanField(default=False) datasets = models.ManyToManyField('DataSet', related_name='assets', blank=True) asset_created = models.DateTimeField(blank=True, null=True) """Date/time the non-digital version of an asset was created For example, the data a photo was taken """ translated_fields = ['title', 'caption'] translation_class = AssetTranslation # Use InheritanceManager from django-model-utils to make # fetching of subclassed objects easier objects = InheritanceManager() def __unicode__(self): subclass_obj = Asset.objects.get_subclass(pk=self.pk) return subclass_obj.__unicode__() @models.permalink def get_absolute_url(self): return ('asset_detail', [str(self.asset_id)]) def display_title(self): """ Wrapper to handle displaying some kind of title when the the title field is blank """ # For now just call the __unicode__() method return unicode(self) def render(self, format='html'): """Render a viewable representation of an asset Arguments: format -- the format to render the asset. defaults to 'html' which is presently the only available option. """ try: return getattr(self, "render_" + format).__call__() except AttributeError: return self.__unicode__() def render_thumbnail(self, width=None, height=None, format='html', **kwargs): """Render a thumbnail-sized viewable representation of an asset Arguments: height -- Height of the thumbnail in pixels width -- Width of the thumbnail in pixels format -- the format to render the asset. defaults to 'html' which is presently the only available option. """ return getattr(self, "render_thumbnail_" + format).__call__( width, height, **kwargs) def render_thumbnail_html(self, width=150, height=100, **kwargs): """ Render HTML for a thumbnail-sized viewable representation of an asset This just provides a dummy placeholder and should be implemented classes that inherit from Asset. Arguments: height -- Height of the thumbnail in pixels width -- Width of the thumbnail in pixels """ html_class = kwargs.get('html_class', "") return mark_safe("<div class='asset-thumbnail %s' " "style='height: %dpx; width: %dpx'>Asset Thumbnail</div>" % (html_class, height, width)) def get_thumbnail_url(self, width=150, height=100, **kwargs): """Return the URL of the Asset's thumbnail""" return None def dataset_html(self, label=_("Associated Datasets")): """Return an HTML list of associated datasets""" output = [] if self.datasets.count(): download_label = _("Download the data") output.append("<p class=\"datasets-label\">%s:</p>" % label) output.append("<ul class=\"datasets\">") for dataset in self.datasets.select_subclasses(): download_label = (_("Download the data") if dataset.links_to_file else _("View the data")) output.append("<li>%s <a href=\"%s\">%s</a></li>" % (dataset.title, dataset.download_url(), download_label)) output.append("</ul>") return mark_safe(u'\n'.join(output)) def full_caption_html(self, wrapper='figcaption'): """Return the caption and attribution text together""" output = "" if self.caption: output += "<div class='caption'>%s</div>" % (self.caption) if self.attribution: attribution = self.attribution if self.source_url: attribution = "<a href='%s'>%s</a>" % (self.source_url, attribution) output += "<div class='attribution'>%s</div>" % (attribution) dataset_html = self.dataset_html() if dataset_html: output += dataset_html if output: output = '<%s>%s</%s>' % (wrapper, output, wrapper) return output
class Asset(ImageRenderingMixin, TranslatedModel, LicensedModel, PublishedModel, TimestampedModel, AssetPermission): """A piece of content included in a story An asset could be an image, a block of text, an embedded resource represented by an HTML snippet or a media file. This is a base class that provides common metadata for the asset. However, it does not provide the fields that specify the content itself. Also, to reduce the number of database tables and queries this model class does not provide translated metadata fields. When creating an asset, one shouldn't instantiate this class, but instead use one of the model classes that inherits form Asset. """ asset_id = UUIDField(auto=True, db_index=True) type = models.CharField(max_length=10, choices=ASSET_TYPES) attribution = models.TextField(blank=True) source_url = models.URLField(blank=True) """The URL where an asset originated. It could be used to store the canonical URL for a resource that is not yet oEmbedable or the canonical URL of an article or tweet where text is quoted from. """ owner = models.ForeignKey(User, related_name="assets", blank=True, null=True) section_specific = models.BooleanField(default=False) datasets = models.ManyToManyField('DataSet', related_name='assets', blank=True) asset_created = models.DateTimeField(blank=True, null=True) """Date/time the non-digital version of an asset was created For example, the data a photo was taken """ translated_fields = ['title', 'caption'] translation_class = AssetTranslation # Use InheritanceManager from django-model-utils to make # fetching of subclassed objects easier objects = InheritanceManager() def __unicode__(self): subclass_obj = Asset.objects.get_subclass(pk=self.pk) return subclass_obj.__unicode__() @models.permalink def get_absolute_url(self): return ('asset_detail', [str(self.asset_id)]) def display_title(self): """ Wrapper to handle displaying some kind of title when the the title field is blank """ # For now just call the __unicode__() method return unicode(self) def css_classes(self): """ Returns string of CSS classes for the asset's HTML container element """ return "asset-%s asset-type-%s" % (self.asset_id, self.type) def render(self, format='html', **kwargs): """Render a viewable representation of an asset Arguments: format -- the format to render the asset. defaults to 'html' which is presently the only available option. """ try: return getattr(self, "render_" + format).__call__(**kwargs) except AttributeError: return self.__unicode__() def dataset_html(self, label=_("Get the Data")): """Return an HTML list of associated datasets""" if not self.datasets.count(): return u"" return render_to_string("storybase_asset/asset_datasets.html", { 'label': label, 'datasets': self.datasets.select_subclasses() }) def full_caption_html(self, wrapper='figcaption'): """Return the caption and attribution text together""" output = "" if self.caption: output += "<div class='caption'>%s</div>" % (self.caption) if self.attribution: attribution = self.attribution if self.source_url: attribution = "<a href='%s'>%s</a>" % (self.source_url, attribution) output += "<div class='attribution'>%s</div>" % (attribution) dataset_html = self.dataset_html() if dataset_html: output += dataset_html if output: output = '<%s>%s</%s>' % (wrapper, output, wrapper) return output def strings(self): """Print all the strings in all languages for this asset This is meant to be used to help generate a document for full-text search using Haystack. """ strings = [] translations = getattr(self, self.translation_set) for translation in translations.all(): trans_strings = translation.strings() if trans_strings: strings.append(trans_strings) return " ".join(strings)