def language(self) -> str:
"""
Most probable language found in decoded sequence. If none were detected or inferred, the property will return
"Unknown".
"""
if not self._languages:
# Trying to infer the language based on the given encoding
# Its either English or we should not pronounce ourselves in certain cases.
if "ascii" in self.could_be_from_charset:
return "English"
# doing it there to avoid circular import
from charset_normalizer.cd import encoding_languages, mb_encoding_languages
languages = (
mb_encoding_languages(self.encoding)
if is_multi_byte_encoding(self.encoding)
else encoding_languages(self.encoding)
)
if len(languages) == 0 or "Latin Based" in languages:
return "Unknown"
return languages[0]
return self._languages[0][0]
def from_bytes(sequences: bytes,
steps: int = 5,
chunk_size: int = 512,
threshold: float = 0.2,
cp_isolation: List[str] = None,
cp_exclusion: List[str] = None,
preemptive_behaviour: bool = True,
explain: bool = False) -> CharsetMatches:
"""
Given a raw bytes sequence, return the best possibles charset usable to render str objects.
If there is no results, it is a strong indicator that the source is binary/not text.
"""
if not explain:
logger.setLevel(logging.CRITICAL)
else:
logger.setLevel(logging.INFO)
length = len(sequences) # type: int
if length == 0:
logger.warning(
"Given content is empty, stopping the process very early, returning empty utf_8 str match"
)
return CharsetMatches(
[CharsetMatch(sequences, "utf_8", 0., False, [], "")])
if cp_isolation is not None:
logger.warning(
'cp_isolation is set. use this flag for debugging purpose. '
'limited list of encoding allowed : %s.', ', '.join(cp_isolation))
cp_isolation = [iana_name(cp, False) for cp in cp_isolation]
else:
cp_isolation = []
if cp_exclusion is not None:
logger.warning(
'cp_exclusion is set. use this flag for debugging purpose. '
'limited list of encoding excluded : %s.', ', '.join(cp_exclusion))
cp_exclusion = [iana_name(cp, False) for cp in cp_exclusion]
else:
cp_exclusion = []
if length <= (chunk_size * steps):
logger.warning(
'override steps (%i) and chunk_size (%i) as content does not fit (%i byte(s) given) parameters.',
steps, chunk_size, length)
steps = 1
chunk_size = length
if steps > 1 and length / steps < chunk_size:
chunk_size = int(length / steps)
is_too_small_sequence = len(sequences) < TOO_SMALL_SEQUENCE # type: bool
is_too_large_sequence = len(sequences) >= TOO_BIG_SEQUENCE # type: bool
if is_too_small_sequence:
warn('Trying to detect encoding from a tiny portion of ({}) byte(s).'.
format(length))
prioritized_encodings = [] # type: List[str]
specified_encoding = any_specified_encoding(
sequences
) if preemptive_behaviour is True else None # type: Optional[str]
if specified_encoding is not None:
prioritized_encodings.append(specified_encoding)
logger.info(
'Detected declarative mark in sequence. Priority +1 given for %s.',
specified_encoding)
tested = set() # type: Set[str]
tested_but_hard_failure = [] # type: List[str]
tested_but_soft_failure = [] # type: List[str]
single_byte_hard_failure_count = 0 # type: int
single_byte_soft_failure_count = 0 # type: int
results = CharsetMatches() # type: CharsetMatches
sig_encoding, sig_payload = identify_sig_or_bom(sequences)
if sig_encoding is not None:
prioritized_encodings.append(sig_encoding)
logger.info(
'Detected a SIG or BOM mark on first %i byte(s). Priority +1 given for %s.',
len(sig_payload), sig_encoding)
prioritized_encodings.append("ascii")
if "utf_8" not in prioritized_encodings:
prioritized_encodings.append("utf_8")
for encoding_iana in prioritized_encodings + IANA_SUPPORTED:
if cp_isolation and encoding_iana not in cp_isolation:
continue
if cp_exclusion and encoding_iana in cp_exclusion:
continue
if encoding_iana in tested:
continue
tested.add(encoding_iana)
decoded_payload = None # type: Optional[str]
bom_or_sig_available = sig_encoding == encoding_iana # type: bool
strip_sig_or_bom = bom_or_sig_available and should_strip_sig_or_bom(
encoding_iana) # type: bool
if encoding_iana in {"utf_16", "utf_32"
} and bom_or_sig_available is False:
logger.info(
"Encoding %s wont be tested as-is because it require a BOM. Will try some sub-encoder LE/BE.",
encoding_iana)
continue
try:
is_multi_byte_decoder = is_multi_byte_encoding(
encoding_iana) # type: bool
except (ModuleNotFoundError, ImportError):
logger.debug("Encoding %s does not provide an IncrementalDecoder",
encoding_iana)
continue
try:
if is_too_large_sequence and is_multi_byte_decoder is False:
str(sequences[:int(50e4)], encoding=encoding_iana)
else:
decoded_payload = str(sequences if strip_sig_or_bom is False
else sequences[len(sig_payload):],
encoding=encoding_iana)
except UnicodeDecodeError as e:
logger.warning(
'Code page %s does not fit given bytes sequence at ALL. %s',
encoding_iana, str(e))
tested_but_hard_failure.append(encoding_iana)
if not is_multi_byte_decoder:
single_byte_hard_failure_count += 1
continue
except LookupError:
tested_but_hard_failure.append(encoding_iana)
if not is_multi_byte_decoder:
single_byte_hard_failure_count += 1
continue
similar_soft_failure_test = False # type: bool
for encoding_soft_failed in tested_but_soft_failure:
if is_cp_similar(encoding_iana, encoding_soft_failed):
similar_soft_failure_test = True
break
if similar_soft_failure_test:
logger.warning(
"%s is deemed too similar to code page %s and was consider unsuited already. Continuing!",
encoding_iana, encoding_soft_failed)
continue
r_ = range(0 if bom_or_sig_available is False else len(sig_payload),
length, int(length / steps))
multi_byte_bonus = is_multi_byte_decoder and decoded_payload is not None and len(
decoded_payload) < length # type: bool
if multi_byte_bonus:
logger.info(
'Code page %s is a multi byte encoding table and it appear that at least one character was encoded using n-bytes. Should not be a coincidence. Priority +1 given.',
encoding_iana)
max_chunk_gave_up = int(len(r_) / 4) # type: int
if max_chunk_gave_up < 2:
max_chunk_gave_up = 2
early_stop_count = 0 # type: int
md_chunks = [] # type: List[str]
md_ratios = []
for i in r_:
cut_sequence = sequences[i:i + chunk_size]
if bom_or_sig_available and strip_sig_or_bom is False:
cut_sequence = sig_payload + cut_sequence
chunk = cut_sequence.decode(encoding_iana,
errors="ignore") # type: str
md_chunks.append(chunk)
md_ratios.append(mess_ratio(chunk, threshold))
if md_ratios[-1] >= threshold:
early_stop_count += 1
if (early_stop_count >= max_chunk_gave_up) or (
bom_or_sig_available and strip_sig_or_bom is False):
break
if md_ratios:
mean_mess_ratio = sum(md_ratios) / len(md_ratios) # type: float
else:
mean_mess_ratio = 0.
if mean_mess_ratio >= threshold or early_stop_count >= max_chunk_gave_up:
tested_but_soft_failure.append(encoding_iana)
if not is_multi_byte_decoder:
single_byte_soft_failure_count += 1
logger.warning(
'%s was excluded because of initial chaos probing. Gave up %i time(s). '
'Computed mean chaos is %f %%.', encoding_iana,
early_stop_count, round(mean_mess_ratio * 100, ndigits=3))
continue
logger.info(
'%s passed initial chaos probing. Mean measured chaos is %f %%',
encoding_iana, round(mean_mess_ratio * 100, ndigits=3))
if not is_multi_byte_decoder:
target_languages = encoding_languages(
encoding_iana) # type: List[str]
else:
target_languages = mb_encoding_languages(encoding_iana)
if target_languages:
logger.info("{} should target any language(s) of {}".format(
encoding_iana, str(target_languages)))
cd_ratios = []
for chunk in md_chunks:
chunk_languages = coherence_ratio(
chunk, 0.1,
",".join(target_languages) if target_languages else None)
cd_ratios.append(chunk_languages)
cd_ratios_merged = merge_coherence_ratios(cd_ratios)
if cd_ratios_merged:
logger.info("We detected language {} using {}".format(
cd_ratios_merged, encoding_iana))
results.append(
CharsetMatch(sequences, encoding_iana, mean_mess_ratio,
bom_or_sig_available, cd_ratios_merged,
decoded_payload))
if encoding_iana in [specified_encoding, "ascii", "utf_8"
] and mean_mess_ratio < 0.1:
logger.info("%s is most likely the one. Stopping the process.",
encoding_iana)
return CharsetMatches([results[encoding_iana]])
if encoding_iana == sig_encoding:
logger.info(
"%s is most likely the one as we detected a BOM or SIG within the beginning of the sequence.",
encoding_iana)
return CharsetMatches([results[encoding_iana]])
if results[-1].languages:
logger.info(
"Using %s code page we detected the following languages: %s",
encoding_iana, results[-1]._languages)
return results
def test_infer_language_from_cp(iana_encoding, expected_languages):
languages = mb_encoding_languages(iana_encoding) if is_multi_byte_encoding(iana_encoding) else encoding_languages(iana_encoding)
for expected_language in expected_languages:
assert expected_language in languages, "Wrongly detected language for given code page"