def test_unicode_japanese(self):
"Unicode with Japanese characters above extended ascii"
raw = "男の子"
self.assertIsInstance(raw, str)
safe = tag_utils.unicode_to_ascii(raw)
self.assertUnicodeAscii(safe)
self.assertEqual(safe, "Nan noZi ")
def test_unicode_thai(self):
"Unicode with Thai characters above extended ascii"
raw = "เด็กผู้ชาย"
self.assertIsInstance(raw, str)
safe = tag_utils.unicode_to_ascii(raw)
self.assertUnicodeAscii(safe)
self.assertEqual(safe, "edkphuuchaay")
def test_unicode_ascii(self):
"Unicode with ASCII characters"
raw = "cake"
self.assertIsInstance(raw, str)
safe = tag_utils.unicode_to_ascii(raw)
self.assertUnicodeAscii(safe)
self.assertEqual(safe, raw)
def test_unicode_extended_ascii(self):
"Unicode with extended ascii characters"
raw = "niño, garçon"
self.assertIsInstance(raw, str)
safe = tag_utils.unicode_to_ascii(raw)
self.assertUnicodeAscii(safe)
self.assertEqual(safe, "nino, garcon")
def test_unicode_mix(self):
"Unicode string with mix of characters"
raw = "niño, 男の子, เด็กผู้ชาย, garçon"
self.assertIsInstance(raw, str)
safe = tag_utils.unicode_to_ascii(raw)
self.assertUnicodeAscii(safe)
self.assertEqual(safe, "nino, ___, _______, garcon")
def save(self, *args, **kwargs):
"""
Automatically generate a unique slug, if one does not exist
"""
# Based on django-taggit: don't worry about race conditions when
# setting names and slugs, just avoid potential slugify clashes.
# We could improve this if race conditions are ever a problem in the
# real world, but until Django provides a reliable way to determine
# the cause of an IntegrityError, we can never make this perfect.
# If already in the database and has a slug set, just save as normal
# Set slug to None to rebuild it
if self.pk and self.slug:
self._update_extra()
return super(BaseTagModel, self).save(*args, **kwargs)
# Set the slug using the label if possible (for TagTreeModel), else
# the tag name. Run it through unicode_to_ascii because slugify will
# cry if the label contains unicode characters
label = getattr(self, "label", self.name)
slug_max_length = self.__class__._meta.get_field("slug").max_length
if settings.SLUG_ALLOW_UNICODE:
slug_base = slugify(label, allow_unicode=True)
else:
slug_base = slugify(six.text_type(utils.unicode_to_ascii(label)))
self.slug = slug_base[:slug_max_length]
self._update_extra()
# Make sure we're using the same db at all times
cls = self.__class__
kwargs["using"] = kwargs.get("using") or router.db_for_write(
cls, instance=self)
# Try saving the slug - it'll probably be fine
try:
# If transaction supports atomic, we need to wrap the save call -
# otherwise if save throws an exception it'll cause any current
# queries to roll back
with transaction_atomic():
return super(BaseTagModel, self).save(*args, **kwargs)
except IntegrityError:
pass
# Integrity error - something is probably not unique.
# Assume it was the slug - make it unique by appending a number.
# See which numbers have been used
slug_base = slug_base[:slug_max_length - settings.SLUG_TRUNCATE_UNIQUE]
try:
last = cls.objects.filter(slug__regex="^%s_[0-9]+$" %
slug_base).latest("slug")
except cls.DoesNotExist:
# No numbered version of the slug exists
number = 1
else:
slug_base, number = last.slug.rsplit("_", 1)
number = int(number) + 1
self.slug = "%s_%d" % (slug_base, number)
self._update_extra()
return super(BaseTagModel, self).save(*args, **kwargs)
def save(self, *args, **kwargs):
"""
Automatically generate a unique slug, if one does not exist
"""
# Based on django-taggit: don't worry about race conditions when
# setting names and slugs, just avoid potential slugify clashes.
# We could improve this if race conditions are ever a problem in the
# real world, but until Django provides a reliable way to determine
# the cause of an IntegrityError, we can never make this perfect.
# If already in the database and has a slug set, just save as normal
# Set slug to None to rebuild it
if self.pk and self.slug:
self._update_extra()
return super(BaseTagModel, self).save(*args, **kwargs)
# Set the slug using the label if possible (for TagTreeModel), else
# the tag name. Run it through unicode_to_ascii because slugify will
# cry if the label contains unicode characters
label = getattr(self, 'label', self.name)
slug_max_length = self.__class__._meta.get_field('slug').max_length
slug_base = slugify(six.text_type(utils.unicode_to_ascii(label)))
self.slug = slug_base[:slug_max_length]
self._update_extra()
# Make sure we're using the same db at all times
cls = self.__class__
kwargs['using'] = kwargs.get('using') or router.db_for_write(
cls, instance=self
)
# Try saving the slug - it'll probably be fine
try:
# If transaction supports atomic, we need to wrap the save call -
# otherwise if save throws an exception it'll cause any current
# queries to roll back
with transaction_atomic():
return super(BaseTagModel, self).save(*args, **kwargs)
except IntegrityError:
pass
# Integrity error - something is probably not unique.
# Assume it was the slug - make it unique by appending a number.
# See which numbers have been used
slug_base = slug_base[:slug_max_length - settings.SLUG_TRUNCATE_UNIQUE]
try:
last = cls.objects.filter(
slug__regex="^%s_[0-9]+$" % slug_base
).latest('slug')
except cls.DoesNotExist:
# No numbered version of the slug exists
number = 1
else:
slug_base, number = last.slug.rsplit('_', 1)
number = int(number) + 1
self.slug = '%s_%d' % (slug_base, number)
self._update_extra()
return super(BaseTagModel, self).save(*args, **kwargs)
def test_unicode_thai(self):
"Unicode with Thai characters above extended ascii"
raw = "เด็กผู้ชาย"
self.assertIsInstance(raw, str)
safe = tag_utils.unicode_to_ascii(raw)
self.assertUnicodeAscii(safe)
# Discrepancy in length of string because some chars were Mn category
self.assertEqual(safe, "_______")
def test_unicode_japanese(self):
"Unicode with Japanese characters above extended ascii"
raw = "男の子"
self.assertIsInstance(raw, str)
safe = tag_utils.unicode_to_ascii(raw)
self.assertUnicodeAscii(safe)
# Discrepancy in length of thai string is because there were Mn chars
self.assertEqual(safe, "___")
def test_unicode_mix(self):
"Unicode string with mix of characters"
raw = "niño, 男の子, เด็กผู้ชาย, garçon"
self.assertIsInstance(raw, str)
safe = tag_utils.unicode_to_ascii(raw)
self.assertUnicodeAscii(safe)
# Note trailing space after Japanese because unidecode can sometimes
# return trailing spaces. This is fine, will be handled by slugify.
self.assertEqual(safe, "nino, Nan noZi , edkphuuchaay, garcon")
