def test_encode_decode(self):
with self.subTest('Encode & Detect UTF-8 WITHOUT SIG SMALL CONTENT'):
self.assertEqual(
CnM.from_bytes('h\xe9llo world!\n'.encode(
'utf_8')).best().first().encoding, 'utf_8')
with self.subTest('Encode & Detect GB18030 WITHOUT SIG'):
self.assertEqual(
CnM.from_bytes('我没有埋怨，磋砣的只是一些时间。'.encode(
'gb18030')).best().first().encoding, 'gb18030')
with self.subTest('Encode & Detect GB18030 WITH SIG (CJK)'):
self.assertEqual(
CnM.from_bytes((u'\uFEFF' + '我没有埋怨，磋砣的只是一些时间。'
).encode('gb18030')).best().first().encoding,
'gb18030')
with self.subTest('Encode & Detect UTF-8 WITHOUT SIG (CJK)'):
self.assertEqual(
CnM.from_bytes('我没有埋怨，磋砣的只是一些时间。'.encode(
'utf_8')).best().first().encoding, 'utf_8')
with self.subTest('Encode & Detect UTF-7 WITHOUT BOM (CJK)'):
self.assertEqual(
CnM.from_bytes('我没有埋怨,蹉跎的只是一些时间。'.encode(
'utf_7')).best().first().encoding, 'utf_7')
with self.subTest('Encode & Detect UTF-7 WITH BOM (CJK)'):
self.assertEqual(
CnM.from_bytes(b'\x2b\x2f\x76\x38' +
'我没有埋怨，磋砣的只是一些时间。'.encode('utf_7')).best().
first().encoding, 'utf_7')
with self.subTest('Encode & Detect UTF-7 WITHOUT BOM (CYRILLIC)'):
self.assertEqual(
CnM.from_bytes(
'Bсеки човек има право на образование. Oбразованието трябва да бъде безплатно,'
.encode('utf_7')).best().first().encoding, 'utf_7')
with self.subTest('Encode & Detect UTF-8 WITH SIG (CJK)'):
self.assertEqual(
CnM.from_bytes(b'\xef\xbb\xbf' +
'我没有埋怨，磋砣的只是一些时间。'.encode('utf_8')).best().
first().encoding, 'utf_8')
with self.subTest('Encode & Detect UTF-8 WITHOUT SIG (CYRILLIC)'):
self.assertEqual(
CnM.from_bytes(
'Bсеки човек има право на образование. Oбразованието трябва да бъде безплатно, '
'поне що се отнася до началното и основното образование.'.
encode('utf_8')).best().first().encoding, 'utf_8')
with self.subTest('Encode & Detect UTF-8 WITHOUT SIG (CYRILLIC)'):
self.assertEqual(
CnM.from_bytes('Bсеки човек има право на образование.'.encode(
'utf_8')).best().first().encoding, 'utf_8')
def normalise_to_utf8(bytes_or_filepath):
"""Convert any text input with unknown encoding to utf-8.
Parameters
----------
bytes_or_filepath : bytes or str
A binary string or path to any text file in any encoding.
Returns
-------
str
A string with correct utf-8 encoding.
Raises
------
TypeError
Input is not of type bytes or a valid path to an existing file.
"""
if type(bytes_or_filepath) == bytes:
utf8_str = str(cnm.from_bytes(bytes_or_filepath).best().first())
elif os.path.isfile(bytes_or_filepath):
utf8_str = str(cnm.from_path(bytes_or_filepath).best().first())
else:
raise TypeError('Input must be bytes or a valid file path')
return utf8_str
def read_file(path: str, filename: str = '') -> str:
if filename:
path = join_path(path, filename=filename)
file_bytes = Path(path).read_bytes()
encodings = CharsetNormalizerMatches.from_bytes(file_bytes).best()
if len(encodings) == 0:
encoding = None
else:
encoding = encodings.first().encoding
return Path(path).read_text(encoding=encoding)
def test_bom_detection(self):
with self.subTest('GB18030 UNAVAILABLE SIG'):
self.assertFalse(
CnM.from_bytes('我没有埋怨，磋砣的只是一些时间。'.encode(
'gb18030')).best().first().byte_order_mark)
with self.subTest('GB18030 AVAILABLE SIG'):
self.assertTrue(
CnM.from_bytes(
(u'\uFEFF' + '我没有埋怨，磋砣的只是一些时间。'
).encode('gb18030')).best().first().byte_order_mark)
with self.subTest('UTF-7 AVAILABLE BOM'):
self.assertTrue(
CnM.from_bytes(b'\x2b\x2f\x76\x38' +
'我没有埋怨，磋砣的只是一些时间。'.encode('utf_7')).best().
first().byte_order_mark)
with self.subTest('UTF-8 AVAILABLE BOM'):
self.assertTrue(
CnM.from_bytes(b'\xef\xbb\xbf' +
'我没有埋怨，磋砣的只是一些时间。'.encode('utf_8')).best().
first().byte_order_mark)
def get_data(cls, self, report_type, option_type):
"""Construct and make get request."""
rpt_date = self.report_date.strftime('%Y%m%d')
p1_url = f"{self.occ_burl}/flex-reports?reportType={report_type}"
p2_url = f"&optionType={option_type}&reportDate={rpt_date}"
# Make get request with passed url
flex_bytes = requests.get(f"{p1_url}{p2_url}")
# If a short error message assume wrong date
if len(flex_bytes.content) < 500:
self.report_date = self.report_date - timedelta(days=1)
rpt_date = self.report_date.strftime('%Y%m%d')
p2_url = f"&optionType={option_type}&reportDate={rpt_date}"
# Make get request with passed url
flex_bytes = requests.get(f"{p1_url}{p2_url}")
self.byte_length = len(flex_bytes.content)
self.rpt_to_print = CnM.from_bytes(flex_bytes.content).best().first()
return flex_bytes
def detect():
if 'file' not in request.files:
return jsonify({'message': 'No file has been sent'}), 400
my_file = request.files['file'] # type: FileStorage
byte_str = my_file.stream.read()
r_ = CnM.from_bytes(byte_str).best()
k_ = chardet_detect(byte_str)
k_['confidence'] = str(round(
k_['confidence'] *
100., ndigits=3)) + ' %' if k_['confidence'] is not None else None
z_ = cchardet_detect(byte_str)
z_['confidence'] = str(round(
z_['confidence'] *
100., ndigits=3)) + ' %' if z_['confidence'] is not None else None
return jsonify({
'charset-normalizer': {
'encoding': r_.encoding,
'aliases': r_.encoding_aliases,
'alphabets': r_.alphabets,
'language': r_.language,
'chaos': str(r_.percent_chaos) + ' %',
'coherence': str(r_.percent_coherence) + ' %',
'could_be': r_.could_be_from_charset
} if r_ is not None else None,
'chardet':
k_,
'cchardet':
z_,
'filename':
my_file.filename,
'b64_content':
b64encode(r_.output()).decode('ascii') if r_ is not None else ''
})
for i in range(N_REQUIRED_LOOP):
st_t = perf_counter_ns()
z_ = n_detect(seq_)
l_.append(perf_counter_ns() - st_t)
st_ar[srt_file]['cchardet'] = locale.format_string('%d',
mean(l_),
grouping=True)
st_re[srt_file]['cchardet'] = z_['encoding']
l_.clear()
for i in range(N_REQUIRED_LOOP):
st_t = perf_counter_ns()
y_ = CharsetNormalizerMatches.from_bytes(seq_)
l_.append(perf_counter_ns() - st_t)
st_ar[srt_file]['charset_normalizer'] = locale.format_string(
'%d', mean(l_), grouping=True)
st_re[srt_file]['charset_normalizer'] = y_.best().first().encoding
x_ = prettytable.PrettyTable(
['File', 'Chardet', 'cChardet', 'Charset Normalizer'])
for k, v in st_ar.items():
x_.add_row([k, v['chardet'], v['cchardet'], v['charset_normalizer']])
print(x_)
x_ = prettytable.PrettyTable(
pd.set_option('display.max_columns', 100)
# Display maximum rows
pd.set_option('display.max_rows', 50)
# %% codecell
########################################
url = "https://www.sec.gov/Archives/edgar/daily-index/2021/QTR2/sitemap.20210426.xml"
get = requests.get(url)
fpath = '/Users/unknown1/Algo/data/sec/raw/daily_index/2021/20210426'
f = open(fpath, 'wb')
f.write(get.content)
f.close()
print(CnM.from_bytes(get.content).best().first())
root = ET.fromstring(get.content.decode('UTF-8'))
# %% codecell
#####################################################################
data = []
for i, child in enumerate(root):
data.append([subchild.text for subchild in child])
df = pd.DataFrame(data) # Write in DF
df.columns = ['url', 'lastmod', 'changefreq', 'priority']
def detect_file_encoding(file_path, buffer_size=1024, max_lines=20):
"""
Determines the encoding of files within the initial `max_lines` of length
`buffer_size`.
:param file_path: path to the file
:type file_path: str
:param buffer_size: buffer length for each line being read
:type buffer_size: int
:param max_lines: number of lines to read from file of length buffer_size
:type max_lines: int
:return: encoding type
:rtype: str
"""
detector = UniversalDetector()
line_count = 0
with FileOrBufferHandler(file_path, 'rb') as input_file:
chunk = input_file.read(buffer_size)
while chunk and line_count < max_lines:
detector.feed(chunk)
chunk = input_file.read(buffer_size)
line_count += 1
detector.close()
encoding = detector.result["encoding"]
# Typical file representation is utf-8 instead of ascii, treat as such.
if not encoding or encoding.lower() in ['ascii', 'windows-1254']:
encoding = 'utf-8'
# Check if encoding can be used to decode without throwing an error
def _decode_is_valid(encoding):
try:
with FileOrBufferHandler(file_path,
encoding=encoding) as input_file:
input_file.read(1024 * 1024)
return True
except:
return False
if not _decode_is_valid(encoding):
try:
from charset_normalizer import CharsetNormalizerMatches as CnM
# Try with small sample
with FileOrBufferHandler(file_path, 'rb') as input_file:
raw_data = input_file.read(10000)
result = CnM.from_bytes(raw_data,
steps=5,
chunk_size=512,
threshold=0.2,
cp_isolation=None,
cp_exclusion=None,
preemptive_behaviour=True,
explain=False)
result = result.best()
if result:
if result.first():
encoding = result.first().encoding
# Try again with full sample
if not _decode_is_valid(encoding):
with FileOrBufferHandler(file_path, 'rb') as input_file:
raw_data = input_file.read(max_lines * buffer_size)
result = CnM.from_bytes(raw_data,
steps=max_lines,
chunk_size=buffer_size,
threshold=0.2,
cp_isolation=None,
cp_exclusion=None,
preemptive_behaviour=True,
explain=False)
result = result.best()
if result:
if result.first():
encoding = result.first().encoding
except:
logger.info("Install charset_normalizer for improved file "
"encoding detection")
# If no encoding is still found, default to utf-8
if not encoding:
encoding = 'utf-8'
return encoding.lower()
import xml.etree.ElementTree as ET
# Display max 50 columns
pd.set_option('display.max_columns', None)
# Display maximum rows
pd.set_option('display.max_rows', None)
# %% codecell
##############################################################
url = 'https://marketdata.theocc.com/delo-download?prodType=ALL&downloadFields=OS;US;SN&format=txt'
get = requests.get(url)
dlp_df = pd.DataFrame(pd.read_csv(BytesIO(get.content)), escapechar='\n', delimiter='\t')
print(CnM.from_bytes(get.content).best().first())
get_sample = get.content[0:1000]
get_sample
"""
sym = 'IBM'
occ = requests.get(f"https://marketdata.theocc.com/series-search?symbolType=U&symbol={sym}")
occ_df = pd.read_csv(BytesIO(occ.content), skiprows=6, escapechar='\n', delimiter='\t')
cols = occ_df.columns[:-1]
occ_df.drop('year', inplace=True, axis=1)
occ_df.columns = cols