def test_unicode_japanese(self): "Unicode with Japanese characters above extended ascii" raw = "男の子" self.assertIsInstance(raw, str) safe = tag_utils.unicode_to_ascii(raw) self.assertUnicodeAscii(safe) self.assertEqual(safe, "Nan noZi ")
def test_unicode_thai(self): "Unicode with Thai characters above extended ascii" raw = "เด็กผู้ชาย" self.assertIsInstance(raw, str) safe = tag_utils.unicode_to_ascii(raw) self.assertUnicodeAscii(safe) self.assertEqual(safe, "edkphuuchaay")
def test_unicode_ascii(self): "Unicode with ASCII characters" raw = "cake" self.assertIsInstance(raw, str) safe = tag_utils.unicode_to_ascii(raw) self.assertUnicodeAscii(safe) self.assertEqual(safe, raw)
def test_unicode_extended_ascii(self): "Unicode with extended ascii characters" raw = "niño, garçon" self.assertIsInstance(raw, str) safe = tag_utils.unicode_to_ascii(raw) self.assertUnicodeAscii(safe) self.assertEqual(safe, "nino, garcon")
def test_unicode_mix(self): "Unicode string with mix of characters" raw = "niño, 男の子, เด็กผู้ชาย, garçon" self.assertIsInstance(raw, str) safe = tag_utils.unicode_to_ascii(raw) self.assertUnicodeAscii(safe) self.assertEqual(safe, "nino, ___, _______, garcon")
def save(self, *args, **kwargs): """ Automatically generate a unique slug, if one does not exist """ # Based on django-taggit: don't worry about race conditions when # setting names and slugs, just avoid potential slugify clashes. # We could improve this if race conditions are ever a problem in the # real world, but until Django provides a reliable way to determine # the cause of an IntegrityError, we can never make this perfect. # If already in the database and has a slug set, just save as normal # Set slug to None to rebuild it if self.pk and self.slug: self._update_extra() return super(BaseTagModel, self).save(*args, **kwargs) # Set the slug using the label if possible (for TagTreeModel), else # the tag name. Run it through unicode_to_ascii because slugify will # cry if the label contains unicode characters label = getattr(self, "label", self.name) slug_max_length = self.__class__._meta.get_field("slug").max_length if settings.SLUG_ALLOW_UNICODE: slug_base = slugify(label, allow_unicode=True) else: slug_base = slugify(six.text_type(utils.unicode_to_ascii(label))) self.slug = slug_base[:slug_max_length] self._update_extra() # Make sure we're using the same db at all times cls = self.__class__ kwargs["using"] = kwargs.get("using") or router.db_for_write( cls, instance=self) # Try saving the slug - it'll probably be fine try: # If transaction supports atomic, we need to wrap the save call - # otherwise if save throws an exception it'll cause any current # queries to roll back with transaction_atomic(): return super(BaseTagModel, self).save(*args, **kwargs) except IntegrityError: pass # Integrity error - something is probably not unique. # Assume it was the slug - make it unique by appending a number. # See which numbers have been used slug_base = slug_base[:slug_max_length - settings.SLUG_TRUNCATE_UNIQUE] try: last = cls.objects.filter(slug__regex="^%s_[0-9]+$" % slug_base).latest("slug") except cls.DoesNotExist: # No numbered version of the slug exists number = 1 else: slug_base, number = last.slug.rsplit("_", 1) number = int(number) + 1 self.slug = "%s_%d" % (slug_base, number) self._update_extra() return super(BaseTagModel, self).save(*args, **kwargs)
def save(self, *args, **kwargs): """ Automatically generate a unique slug, if one does not exist """ # Based on django-taggit: don't worry about race conditions when # setting names and slugs, just avoid potential slugify clashes. # We could improve this if race conditions are ever a problem in the # real world, but until Django provides a reliable way to determine # the cause of an IntegrityError, we can never make this perfect. # If already in the database and has a slug set, just save as normal # Set slug to None to rebuild it if self.pk and self.slug: self._update_extra() return super(BaseTagModel, self).save(*args, **kwargs) # Set the slug using the label if possible (for TagTreeModel), else # the tag name. Run it through unicode_to_ascii because slugify will # cry if the label contains unicode characters label = getattr(self, 'label', self.name) slug_max_length = self.__class__._meta.get_field('slug').max_length slug_base = slugify(six.text_type(utils.unicode_to_ascii(label))) self.slug = slug_base[:slug_max_length] self._update_extra() # Make sure we're using the same db at all times cls = self.__class__ kwargs['using'] = kwargs.get('using') or router.db_for_write( cls, instance=self ) # Try saving the slug - it'll probably be fine try: # If transaction supports atomic, we need to wrap the save call - # otherwise if save throws an exception it'll cause any current # queries to roll back with transaction_atomic(): return super(BaseTagModel, self).save(*args, **kwargs) except IntegrityError: pass # Integrity error - something is probably not unique. # Assume it was the slug - make it unique by appending a number. # See which numbers have been used slug_base = slug_base[:slug_max_length - settings.SLUG_TRUNCATE_UNIQUE] try: last = cls.objects.filter( slug__regex="^%s_[0-9]+$" % slug_base ).latest('slug') except cls.DoesNotExist: # No numbered version of the slug exists number = 1 else: slug_base, number = last.slug.rsplit('_', 1) number = int(number) + 1 self.slug = '%s_%d' % (slug_base, number) self._update_extra() return super(BaseTagModel, self).save(*args, **kwargs)
def test_unicode_thai(self): "Unicode with Thai characters above extended ascii" raw = "เด็กผู้ชาย" self.assertIsInstance(raw, str) safe = tag_utils.unicode_to_ascii(raw) self.assertUnicodeAscii(safe) # Discrepancy in length of string because some chars were Mn category self.assertEqual(safe, "_______")
def test_unicode_japanese(self): "Unicode with Japanese characters above extended ascii" raw = "男の子" self.assertIsInstance(raw, str) safe = tag_utils.unicode_to_ascii(raw) self.assertUnicodeAscii(safe) # Discrepancy in length of thai string is because there were Mn chars self.assertEqual(safe, "___")
def test_unicode_mix(self): "Unicode string with mix of characters" raw = "niño, 男の子, เด็กผู้ชาย, garçon" self.assertIsInstance(raw, str) safe = tag_utils.unicode_to_ascii(raw) self.assertUnicodeAscii(safe) # Note trailing space after Japanese because unidecode can sometimes # return trailing spaces. This is fine, will be handled by slugify. self.assertEqual(safe, "nino, Nan noZi , edkphuuchaay, garcon")