def default_slugifier(value):
"""
Oscar's default slugifier function.
Uses Django's slugify function, but first applies unidecode() to convert
non-ASCII strings to ASCII equivalents where possible, if it was not
allowed to use unicode in slugs. To keep backwards compatibility with
Django<1.9, we pass `allow_unicode` only if case it was enabled in
settings.
"""
if settings.OSCAR_SLUG_ALLOW_UNICODE:
return django_slugify(value, allow_unicode=True)
return django_slugify(value)
def default_slugifier(value):
"""
Oscar's default slugifier function.
Uses Django's slugify function, but first applies unidecode() to convert
non-ASCII strings to ASCII equivalents where possible, if it was not
allowed to use unicode in slugs. To keep backwards compatibility with
Django<1.9, we pass `allow_unicode` only if case it was enabled in
settings.
"""
if settings.OSCAR_SLUG_ALLOW_UNICODE:
return django_slugify(value, allow_unicode=True)
return django_slugify(value)
def slugify(text):
"""Version of slugify that supports Japanese characters"""
if not text:
return ""
slug = django_slugify(text)
if not slug:
# Title may be in Japanese
slug = django_slugify(romkan.to_roma(text))
if not slug:
# Title may be in Chinese
pinyin = Pinyin()
slug = django_slugify(pinyin.get_pinyin(text))
return slug[:50]
def slugify(string):
slug = '/'.join([django_slugify(part) for part in string.split('/')])
while slug.endswith('/'):
slug = slug[:-1]
while '//' in slug:
slug = slug.replace('//', '/')
return slug
def slugify(s,
entities=True,
decimal=True,
hexadecimal=True,
model=None,
slug_field='slug',
pk=None):
# we don't want a string > 40 characters
if len(s) > 40:
s = s[:40]
s = django_slugify(s)
slug = s
if model:
# return unique slug for a model (appending integer counter)
def get_query():
query = model.objects.filter(**{slug_field: slug})
if pk:
query = query.exclude(pk=pk)
return query
counter = 2
while get_query():
slug = "%s-%s" % (s, counter)
counter += 1
return slug
def slugify(s):
"""
Slugify based on django's slugify except uses unidecode to work with
non-ascii characters.
"""
return django_slugify(unidecode(s))
def slugify(string):
slug = '/'.join([django_slugify(part) for part in string.split('/')])
while slug.endswith('/'):
slug = slug[:-1]
while '//' in slug:
slug = slug.replace('//', '/')
return slug
def slugify(text):
"""Version of slugify that supports Japanese and Chinese characters"""
if not text:
return ""
slug = django_slugify(text)
if not slug:
# Title may be in Japanese
slug = django_slugify(romkan.to_roma(text))
if not slug:
# Title may be in Chinese
pinyin = Pinyin()
slug = django_slugify(pinyin.get_pinyin(text))
if not slug:
# Try transliterate which supports Cyryllic, Greek and other alphabets
slug = django_slugify(translit(text, reversed=True))
return slug[:50]
def cautious_slugify(value):
"""
Convert a string to ASCII exactly as Django's slugify does, with the exception
that any non-ASCII alphanumeric characters (that cannot be ASCIIfied under Unicode
normalisation) are escaped into codes like 'u0421' instead of being deleted entirely.
This ensures that the result of slugifying e.g. Cyrillic text will not be an empty
string, and can thus be safely used as an identifier (albeit not a human-readable one).
cautious_slugify was copied from Wagtail:
<https://github.com/wagtail/wagtail/blob/8b420b9/wagtail/core/utils.py>
Copyright (c) 2014-present Torchbox Ltd and individual contributors.
Released under the BSD 3-clause "New" or "Revised" License
<https://github.com/wagtail/wagtail/blob/8b420b9/LICENSE>
Date: 2018-06-15
"""
# Normalize the string to decomposed unicode form. This causes accented Latin
# characters to be split into 'base character' + 'accent modifier'; the latter will
# be stripped out by the regexp, resulting in an ASCII-clean character that doesn't
# need to be escaped
value = unicodedata.normalize('NFKD', value)
# Strip out characters that aren't letterlike, underscores or hyphens,
# using the same regexp that slugify uses. This ensures that non-ASCII non-letters
# (e.g. accent modifiers, fancy punctuation) get stripped rather than escaped
value = SLUGIFY_RE.sub('', value)
# Encode as ASCII, escaping non-ASCII characters with backslashreplace, then convert
# back to a unicode string (which is what slugify expects)
value = value.encode('ascii', 'backslashreplace').decode('ascii')
# Pass to slugify to perform final conversion (whitespace stripping); this will
# also strip out the backslashes from the 'backslashreplace' conversion
return django_slugify(value)
def gen_slug(s):
"""
Overriding django slugify that allows to use russian words as well.
"""
new_slug = django_slugify(''.join(alphabet.get(w, w) for w in s.lower()),
allow_unicode=True)
return new_slug
def slugify(name, strip_words=False, strip_parens=True):
name = unicode(name).lower()
if strip_parens:
name = re.sub(r'\([^\)]*\)', '', name)
if strip_words:
name = re.sub(r'\b(%s)\b' % '|'.join(UNIMPORTANT_WORDS), '', name)
return django_slugify(name)
def cautious_slugify(value):
"""
Convert a string to ASCII exactly as Django's slugify does, with the exception
that any non-ASCII alphanumeric characters (that cannot be ASCIIfied under Unicode
normalisation) are escaped into codes like 'u0421' instead of being deleted entirely.
This ensures that the result of slugifying e.g. Cyrillic text will not be an empty
string, and can thus be safely used as an identifier (albeit not a human-readable one).
cautious_slugify was copied from Wagtail:
<https://github.com/wagtail/wagtail/blob/8b420b9/wagtail/core/utils.py>
Copyright (c) 2014-present Torchbox Ltd and individual contributors.
Released under the BSD 3-clause "New" or "Revised" License
<https://github.com/wagtail/wagtail/blob/8b420b9/LICENSE>
Date: 2018-06-15
"""
# Normalize the string to decomposed unicode form. This causes accented Latin
# characters to be split into 'base character' + 'accent modifier'; the latter will
# be stripped out by the regexp, resulting in an ASCII-clean character that doesn't
# need to be escaped
value = unicodedata.normalize('NFKD', value)
# Strip out characters that aren't letterlike, underscores or hyphens,
# using the same regexp that slugify uses. This ensures that non-ASCII non-letters
# (e.g. accent modifiers, fancy punctuation) get stripped rather than escaped
value = SLUGIFY_RE.sub('', value)
# Encode as ASCII, escaping non-ASCII characters with backslashreplace, then convert
# back to a unicode string (which is what slugify expects)
value = value.encode('ascii', 'backslashreplace').decode('ascii')
# Pass to slugify to perform final conversion (whitespace stripping); this will
# also strip out the backslashes from the 'backslashreplace' conversion
return django_slugify(value)
def slugify(text):
"""
Slugify function that supports unicode symbols
:param text: any unicode text
:return: slugified version of passed text
"""
from django.utils.text import slugify as django_slugify
return django_slugify(force_text(unidecode(text)))
def slugify(value):
if _settings.TRANSLITERATE_FILENAME:
try:
from unidecode import unidecode
value = unidecode(value)
except ImportError as e:
logger.exception(e)
return django_slugify(value)
def slugify(text):
"""
Slugify function that supports unicode symbols
:param text: any unicode text
:return: slugified version of passed text
"""
from django.utils.text import slugify as django_slugify
return django_slugify(force_str(unidecode(text)))
def default_slugifier(value, allow_unicode=False):
"""
Oscar's default slugifier function. When unicode is allowed
it uses Django's slugify function, otherwise it uses cautious_slugify.
"""
if allow_unicode:
return django_slugify(value, allow_unicode=True)
else:
return cautious_slugify(value)
def default_slugifier(value, allow_unicode=False):
"""
Oscar's default slugifier function. When unicode is allowed
it uses Django's slugify function, otherwise it uses cautious_slugify.
"""
if allow_unicode:
return django_slugify(value, allow_unicode=True)
else:
return cautious_slugify(value)
def cautious_slugify(value):
"""
Convert a string to ASCII exactly as Django's slugify does, with the exception
that any non-ASCII alphanumeric characters (that cannot be ASCIIfied under Unicode
normalisation) are escaped into codes like 'u0421' instead of being deleted entirely.
This ensures that the result of slugifying e.g. Cyrillic text will not be an empty
string, and can thus be safely used as an identifier (albeit not a human-readable one).
cautious_slugify was copied from Wagtail:
<https://github.com/wagtail/wagtail/blob/8b420b9/wagtail/core/utils.py>
Copyright (c) 2014-present Torchbox Ltd and individual contributors.
Released under the BSD 3-clause "New" or "Revised" License
<https://github.com/wagtail/wagtail/blob/8b420b9/LICENSE>
Date: 2018-06-15
正如Django的slugify所做的那样，将字符串转换为ASCII，但任何非ASCII字母数字字符（在Unicode规
范化下不能被ASCII化）都会转义为“u0421”之类的代码，而不是完全删除。
这确保了例如slugifying的结果。 西里尔文本不是空字符串，因此可以安全地用作标识符（尽管不是人类可读的标识符）。
从Wagtail复制了cautious_slugify：
<https://github.com/wagtail/wagtail/blob/8b420b9/wagtail/core/utils.py>
版权所有（c）2014-present Torchbox Ltd和个人贡献者。
根据BSD 3条款“New”或“Revised”许可发布
<https://github.com/wagtail/wagtail/blob/8b420b9/LICENSE>
Date: 2018-06-15
"""
# Normalize the string to decomposed unicode form. This causes accented Latin
# characters to be split into 'base character' + 'accent modifier'; the latter will
# be stripped out by the regexp, resulting in an ASCII-clean character that doesn't
# need to be escaped
# 将字符串规范化为分解的unicode形式。 这导致重音拉丁字符被分成'基本字符'+'重音修饰
# 符'; 后者将被正则表达式删除，从而产生一个不需要转义的ASCII-clean字符
value = unicodedata.normalize('NFKD', value)
# Strip out characters that aren't letterlike, underscores or hyphens,
# using the same regexp that slugify uses. This ensures that non-ASCII non-letters
# (e.g. accent modifiers, fancy punctuation) get stripped rather than escaped
# 使用与slugify使用相同的正则表达式删除不是字母，下划线或连字符的字符。 这可以确保
# 非ASCII非字母（例如重音修饰符，花式标点符号）被剥离而不是转义
value = SLUGIFY_RE.sub('', value)
# Encode as ASCII, escaping non-ASCII characters with backslashreplace, then convert
# back to a unicode string (which is what slugify expects)
# 编码为ASCII，使用backslashreplace转义非ASCII字符，然后转换回unicode字符串
# （这是slugify期望的）
value = value.encode('ascii', 'backslashreplace').decode('ascii')
# Pass to slugify to perform final conversion (whitespace stripping, applying
# mark_safe); this will also strip out the backslashes from the 'backslashreplace'
# conversion
# 通过slugify进行最终转换（空白剥离，应用mark_safe）; 这也将
# 从'backslashreplace'转换中去掉反斜杠
return django_slugify(value)
def slugify(value, allow_unicode=False):
"""Override default slugify in django to handle custom scenarios.
Run value through functions declared in SLUGIFY_PROCESSORS. The value is
then passed-through to django's slugify.
:param value: string to slugify
:type value: string
:param allow_unicode: whether or not to allow unicode (e.g. chinese)
:type allow_unicode: bool
Examples of slugify processors, assume *project/app/slugify_processors.py*:
.. code-block:: python
def slugify_programming_languages(value):
value = value.lower()
value = value.replace('c++', 'cpp')
value = value.replace('c#', 'c-sharp')
return value
def slugify_geo_acronyms(value):
value = value.lower()
value = value.replace('New York City', 'nyc')
value = value.replace('United States', 'usa')
return value
def slugify_us_currency(value):
value = value.lower()
value = value.replace('$', 'usd')
value = value.replace('US$', 'usd')
value = value.replace('US Dollar', 'usd')
value = value.replace('U.S. Dollar', 'usd')
return value
Settings:
.. code-block:: python
SLUGIFY_PROCESSORS = [
'project.app.slugify_programming_languages',
'project.app.slugify_geo_acronyms',
'project.app.slugify_us_currency',
]
"""
if hasattr(settings, "SLUGIFY_PROCESSORS"):
for slugify_fn_str in settings.SLUGIFY_PROCESSORS:
slugify_fn_ = import_string(slugify_fn_str)
value = slugify_fn_(value)
return django_slugify(value, allow_unicode)
def default_slugifier(value, allow_unicode=False):
"""
Oscar's default slugifier function. When unicode is allowed
it uses Django's slugify function, otherwise it uses cautious_slugify.
奥斯卡的默认推土机功能。 当允许unicode时，它使用Django的slugify函数，否则
使用cautious_slugify。
"""
if allow_unicode:
return django_slugify(value, allow_unicode=True)
else:
return cautious_slugify(value)
def slugify(string):
'''
Helper function for converting Norwegian characters to ASCII representation.
'''
string = string.lower()
for substitution in (
('æ', 'ae'),
('ø', 'oe'),
('å', 'aa'),
):
string = string.replace(*substitution)
return django_slugify(string)
def slugify(s, entities=True, decimal=True, hexadecimal=True, instance=None, slug_field='slug', filter_dict=None):
s = django_slugify(s)
slug = s
if instance:
def get_query():
query = instance.__class__.objects.filter(**{slug_field: slug})
if filter_dict:
query = query.filter(**filter_dict)
if instance.pk:
query = query.exclude(pk=instance.pk)
return query
counter = 1
while get_query():
slug = "%s-%s" % (s, counter)
counter += 1
if len(slug) > 50:
slug = slug[:50]
return slug
def slugify(s, entities=True, decimal=True, hexadecimal=True, model=None, slug_field='slug', pk=None):
# we don't want a string > 40 characters
if len(s) > 40:
s = s[:40]
s = django_slugify(s)
slug = s
if model:
# return unique slug for a model (appending integer counter)
def get_query():
query = model.objects.filter(**{ slug_field: slug })
if pk:
query = query.exclude(pk=pk)
return query
counter = 2
while get_query():
slug = "%s-%s" % (s, counter)
counter += 1
return slug
def default_slugifier(value):
"""
Oscar's default slugifier function. Uses Django's slugify function.
"""
return django_slugify(value,
allow_unicode=settings.OSCAR_SLUG_ALLOW_UNICODE)
def slugify_long(value):
return django_slugify(downgrade(value))
def slugify(s):
return django_slugify(''.join(alphabet.get(w, w) for w in s.lower()), allow_unicode=True)
# ----------------------------------------------------------
# -----------------------------------------------------------
def slugify(value, max_length=128, usable=None, max_retries=1000):
"""
Normalizes string, removing non-ascii characters, converts to lowercase,
converts underscores and spaces to hyphens, removes all other non-alphanumeric
characters.
Providing an exists function will result in conflict resolution. Conflicts
will have a suffix appended to indicate the index. e.g. samsung, samsung~1
... samsung~10
If the appending the suffix exceeds maxlen then the original slug will be
truncated to fit exactly maxlen. e.g. samsung, samsun~1, samsun~2 ... samsu~10
The usable function takes a single value `slug` and returns True or False.
The algorithm will continue to try new slugs, until the usable method
returns True.
To prevent an infinate loop condition, the max_retries variable limits how
many different slugs to try before raising a RuntimeError.
def valid_slug(self, slug):
parent = self.parent or self.parent_id
return not MyModel.objects.filter(parent=parent, slug=slug).exists()
def save(self, *args, **kwargs):
if not self.slug is None and self.name:
self.slug = slugify(self.name, exists=self.valid_slug)
:param value: string to normalize
:param maxlen: maximum length for string, default is 128. If evaluates
False then no max length will be enforced
:param exists: a function that returns True if the slug already exists,
False otherwise.
:param max_retries: limit the number of times to retry slug creation
before giving up.
:return: slugified value
"""
if usable and not callable(usable):
raise TypeError('usable argument must be a callable')
if isinstance(value, unicode):
# if the value is currently unicode, convert it back to a ascii
# traslating any special chars to similar ones in ascii.
# For example: unidecode(u'Očko') -> 'Ocko'
value = unidecode(value)
slug = django_slugify(unicode(value)).replace('_', '-')
if max_length:
slug = slug[:max_length]
# decide whether to resolve conflicts
if usable is None:
return slug
# conflict detection/resolution
copy = slug
count = 0
# TODO: need a better slug collision algorithm, as this can be expensive
while not usable(slug):
count += 1
if max_retries and count > max_retries:
raise RuntimeError('slugify surpassed its max_retries limit of %s'
% max_retries)
suffix = '~%d' % count
slug = copy[:max_length - len(suffix)] if max_length else copy
slug = '%s%s' % (slug, suffix)
return slug
def slugify(string):
return '/'.join([django_slugify(part) for part in string.split('/')])
def slugify(word_for_translate):
"""
Проводим транслитерацию кирилических обозначений.
"""
return django_slugify(''.join(
alphabet.get(w, w) for w in word_for_translate.lower()))
def slugify(base):
return django_slugify(base, allow_unicode=True)
def test_slugify_fallback_to_default():
assert slugify("c++") == django_slugify("c++")
def slugify(string, allow_unicode=False):
return django_slugify(string)
def downgrading_slugify(value):
# Slugfiy only allowing hyphens, numbers and ASCII characters
# FIXME django_slugify might return an empty string; take care that we always return something
return re.sub("[ _]+", "-", django_slugify(downgrade(value)))
def slugify(text):
"""Unicode safe slugify function, which also handles blank slugs"""
slug = django_slugify(unidecode(text))
return slug[:255] if slug else "untitled"
def test_function_slugify(self):
S = self.random_string(add_chars="¶@&*\"'~%")
S += "¶@&*\"'~%"
expected = django_slugify(S)
ret = functions.slugify(S)
self.assertEqual(expected, ret)
def slugify(base):
if django.VERSION >= (1, 9):
return django_slugify(base, allow_unicode=True)
else:
return django_slugify(base)
def slugify(s):
return django_slugify(''.join(alphabet.get(w, w) for w in s.lower()))
def slugify(value, max_length=128, usable=None, max_retries=1000):
"""
Normalizes string, removing non-ascii characters, converts to lowercase,
converts underscores and spaces to hyphens, removes all other non-alphanumeric
characters.
Providing an exists function will result in conflict resolution. Conflicts
will have a suffix appended to indicate the index. e.g. samsung, samsung~1
... samsung~10
If the appending the suffix exceeds maxlen then the original slug will be
truncated to fit exactly maxlen. e.g. samsung, samsun~1, samsun~2 ... samsu~10
The usable function takes a single value `slug` and returns True or False.
The algorithm will continue to try new slugs, until the usable method
returns True.
To prevent an infinate loop condition, the max_retries variable limits how
many different slugs to try before raising a RuntimeError.
def valid_slug(self, slug):
parent = self.parent or self.parent_id
return not MyModel.objects.filter(parent=parent, slug=slug).exists()
def save(self, *args, **kwargs):
if not self.slug is None and self.name:
self.slug = slugify(self.name, exists=self.valid_slug)
:param value: string to normalize
:param maxlen: maximum length for string, default is 128. If evaluates
False then no max length will be enforced
:param exists: a function that returns True if the slug already exists,
False otherwise.
:param max_retries: limit the number of times to retry slug creation
before giving up.
:return: slugified value
"""
if usable and not callable(usable):
raise TypeError('usable argument must be a callable')
if isinstance(value, unicode):
# if the value is currently unicode, convert it back to a ascii
# traslating any special chars to similar ones in ascii.
value = unidecode(value)
slug = django_slugify(unicode(value)).replace('_', '-')
if max_length:
slug = slug[:max_length]
# decide whether to resolve conflicts
if usable is None:
return slug
# conflict detection/resolution
copy = slug
count = 0
# TODO: need a better slug collision algorithm, as this can be expensive
while not usable(slug):
count += 1
if max_retries and count > max_retries:
raise RuntimeError(
'slugify surpassed its max_retries limit of %s' % max_retries)
suffix = '~%d' % count
slug = copy[:max_length - len(suffix)] if max_length else copy
slug = '%s%s' % (slug, suffix)
return slug
def slugifier(value, allow_unicode=False):
if not allow_unicode:
value = unidecode(text_type(value))
return django_slugify(value, allow_unicode=allow_unicode)
def default_slugifier(value):
"""
Oscar's default slugifier function. Uses Django's slugify function.
"""
return django_slugify(value, allow_unicode=settings.OSCAR_SLUG_ALLOW_UNICODE)