def test_mathematical_digits(self):
# 5 consecutive sequences of 0-9
for n in range(0x1d7ce, 0x1d800):
self.assertEqual(
unidecode(chr(n)),
chr(ord('0') + (n - 0x1d7ce) % 10),
)
def test_circled_latin(self):
# 1 sequence of a-z
for n in range(0, 26):
self.assertEqual(
unidecode(chr(0x24d0 + n)),
chr(ord('a') + n),
)
def to_python(self, *args, **kwargs):
value = super(EncryptedCharField, self).to_python(*args, **kwargs)
if value:
if self.force_ascii:
value = unidecode(value)
if self.force_lower:
value = value.lower()
elif self.force_upper:
value = value.upper()
if self.normalize_spaces:
value = ' '.join(value.split())
elif self.trim_spaces:
value = value.strip()
return value
def test_mathematical_latin(self):
# 13 consecutive sequences of A-Z, a-z with some codepoints
# undefined. We just count the undefined ones and don't check
# positions.
empty_count = 0
for n in range(0x1d400, 0x1d6a4):
a = unidecode(chr(n))
if n % 52 < 26:
b = chr(ord('A') + n % 26)
else:
b = chr(ord('a') + n % 26)
if not a:
empty_count += 1
else:
self.assertEqual(a, b)
self.assertEqual(empty_count, 24)
def slugify(string,
extra_characters='',
ascii=False,
lower=True,
spaces=False):
"""
Returns a copy of the given ``string`` without non-alphanumeric characters.
Separators, symbols and punctuation characters are replaced by a dash. The
other characters are simply removed.
This is a modified version of the eponymous function of the Mozilla
Foundation to integrate it with the function ``unidecode()``. This
function generates an ASCII representation of the Unicode characters.
"""
if ascii:
string = unidecode(force_text(string))
else:
string = unicodedata.normalize('NFKC', force_text(string))
slug_tokens = []
for char in string:
category = unicodedata.category(char)[0]
# L and N signify letter and number.
if category in 'LN' or char in extra_characters:
slug_tokens.append(char)
# P, S and Z signify punctuation, symbol and separator.
elif category in 'PSZ':
slug_tokens.append(' ')
slug = ''.join(slug_tokens).strip()
if spaces:
slug = SPACES_RE.sub(' ', slug)
else:
slug = SPACES_RE.sub('-', slug)
if lower:
slug = slug.lower()
return slug
def to_python(self, value):
if value is None:
return value
if not isinstance(value, six.string_types):
value = force_text(value)
if self.normalize_spaces:
value = ' '.join(value.split())
elif self.trim_spaces:
value = value.strip()
if not value:
return value
if self.force_ascii:
value = unidecode(value)
if self.force_lower:
value = value.lower()
elif self.force_upper:
value = value.upper()
return value
def test_bmp(self):
# Just check that it doesn't throw an exception
for n in range(0, 0x10000):
unidecode(chr(n))
def test_ascii(self):
for n in range(0, 128):
self.assertEqual(
unidecode(chr(n)),
chr(n),
)
def checkUnidecode(self, tests):
for unicode, ascii in tests:
self.assertEqual(unidecode(unicode), ascii)
def toascii(value):
return unidecode(htmlentities.decode(value))