def test_detect_encoding_windows_1252():
sample = b'A\n' * 300 + b'\xff\xff'
try:
import cchardet
assert helpers.detect_encoding(sample) == 'cp1252'
except ImportError:
assert helpers.detect_encoding(sample) == 'iso8859-1'
def test_detect_encoding_windows_1252():
sample = b'A\n' * 300 + b'\xff\xff'
assert helpers.detect_encoding(sample) == 'cp1252'
def test_detect_encoding():
with io.open('Makefile', 'rb') as fp:
sample = fp.read(config.DEFAULT_BYTES_SAMPLE_SIZE)
assert helpers.detect_encoding(sample) == 'utf-8'
def test_detect_encoding_unknown():
sample = b'\xff\x81'
assert helpers.detect_encoding(sample) == 'utf-8'
def test_detect_encoding_windows_1252():
sample = b'A\n' * 300 + b'\xff\xff'
assert helpers.detect_encoding(sample) == 'windows-1252'
def test_detect_encoding():
sample = io.open('README.md', 'rb').read(config.BYTES_SAMPLE_SIZE)
assert helpers.detect_encoding(sample) == 'utf-8'
def test_detect_encoding_not_so_long():
bytes = io.BytesIO(b'A\n' * 999 + b'\xff\xff')
assert helpers.detect_encoding(bytes) == 'windows-1252'
def test_detect_encoding():
bytes = io.open('README.md', 'rb')
assert helpers.detect_encoding(bytes) == 'utf-8'
def test_detect_encoding_unknown():
bytes = io.BytesIO(b'\xff\x81')
assert helpers.detect_encoding(bytes) == 'utf-8'
def test_detect_encoding():
bytes = io.open('README.md', 'rb')
assert helpers.detect_encoding(bytes) == 'utf-8'
def test_detect_encoding():
sample = io.open('README.md', 'rb').read(config.BYTES_SAMPLE_SIZE)
assert helpers.detect_encoding(sample) == 'utf-8'
def test_detect_encoding_not_so_long():
bytes = io.BytesIO(b'A\n' * 999 + b'\xff\xff')
assert helpers.detect_encoding(bytes) == 'windows-1252'
def test_detect_encoding_long():
bytes = io.BytesIO(b'A\n' * 1000 + b'\xff\xff')
assert helpers.detect_encoding(bytes) == 'utf-8'
def test_detect_encoding_unknown():
bytes = io.BytesIO(b'\xff\x81')
assert helpers.detect_encoding(bytes) == 'utf-8'
def test_detect_encoding_utf_16_le():
sample = u'\uFEFFthen some text'.encode('utf-16-le')
assert helpers.detect_encoding(sample) == 'utf-16'
def test_detect_encoding_unknown():
sample = b'\xff\x81'
assert helpers.detect_encoding(sample) == 'utf-8'
def test_detect_encoding_long():
bytes = io.BytesIO(b'A\n' * 1000 + b'\xff\xff')
assert helpers.detect_encoding(bytes) == 'utf-8'
