def sequence_weirdness(text):
"""
Determine how often a text has unexpected characters or sequences of
characters. This metric is used to disambiguate when text should be
re-decoded or left as is.
We start by normalizing text in NFC form, so that penalties for
diacritical marks don't apply to characters that know what to do with
them.
The following things are deemed weird:
- Lowercase letters followed by non-ASCII uppercase letters
- Non-Latin characters next to Latin characters
- Un-combined diacritical marks, unless they're stacking on non-alphabetic
characters (in languages that do that kind of thing a lot) or other
marks
- C1 control characters
- Adjacent symbols from any different pair of these categories:
- Modifier marks
- Letter modifiers
- Non-digit numbers
- Symbols (including math and currency)
The return value is the number of instances of weirdness.
"""
text2 = unicodedata.normalize('NFC', text)
weirdness = len(WEIRDNESS_RE.findall(chars_to_classes(text2)))
adjustment = len(MOJIBAKE_SYMBOL_RE.findall(text2)) * 2 - len(
COMMON_SYMBOL_RE.findall(text2)
)
return weirdness * 2 + adjustment
def sequence_weirdness(text):
"""
Determine how often a text has unexpected characters or sequences of
characters. This metric is used to disambiguate when text should be
re-decoded or left as is.
We start by normalizing text in NFC form, so that penalties for
diacritical marks don't apply to characters that know what to do with
them.
The following things are deemed weird:
- Lowercase letters followed by non-ASCII uppercase letters
- Non-Latin characters next to Latin characters
- Un-combined diacritical marks, unless they're stacking on non-alphabetic
characters (in languages that do that kind of thing a lot) or other
marks
- C1 control characters
- Adjacent symbols from any different pair of these categories:
- Modifier marks
- Letter modifiers
- Non-digit numbers
- Symbols (including math and currency)
The return value is the number of instances of weirdness.
"""
text2 = unicodedata.normalize('NFC', text)
weirdness = len(WEIRDNESS_RE.findall(chars_to_classes(text2)))
punct_discount = len(COMMON_SYMBOL_RE.findall(text2))
return weirdness * 2 - punct_discount
def sequence_weirdness(text):
"""
Determine how often a text has unexpected characters or sequences of
characters. This metric is used to disambiguate when text should be
re-decoded or left as is.
We start by normalizing text in NFC form, so that penalties for
diacritical marks don't apply to characters that know what to do with
them.
"""
text2 = unicodedata.normalize('NFC', text)
return len(WEIRDNESS_RE.findall(chars_to_classes(text2)))
def sequence_weirdness(text):
"""
Determine how often a text has unexpected characters or sequences of
characters. This metric is used to disambiguate when text should be
re-decoded or left as is.
We start by normalizing text in NFC form, so that penalties for
diacritical marks don't apply to characters that know what to do with
them.
"""
text2 = unicodedata.normalize('NFC', text)
return len(WEIRDNESS_RE.findall(chars_to_classes(text2)))