def detect_encoding(f):
detector = UniversalDetector()
detector.reset()
for line in f:
detector.feed(line)
if detector.done:
break
detector.close()
return detector.result["encoding"]
def detect_encoding(path):
detector = UniversalDetector()
detector.reset()
with open(path, "rb") as f:
for line in f:
detector.feed(line)
if detector.done:
break
detector.close()
return detector.result["encoding"]
def detect_encoding(self, path, config):
"""
Detect the encoding of the file specified by path, unless encoding is already set in config.
If the autodetection can't determine an
encoding with confidence, it defaults to UTF-8.
param path: The path to the input file which encoding should be detected_encoding
param config: Config object where results are recorded
"""
config.detected_encoding = None
config.detected_encoding_confidence = 0
if config.encoding is not None:
logger.debug(
'CmdLineConfigReader: detect_encoding, encoding pre-set'
' (encoding={encoding})'.format(encoding=config.encoding))
return
default = 'UTF-8'
detector = UniversalDetector()
with open(path, 'rb') as file, detector:
for line in file:
detector.feed(line)
if detector.done:
break
confidence = config.detected_encoding_confidence = detector.result[
'confidence']
encoding = config.detected_encoding = detector.result['encoding']
logger.debug(
'CmdLineConfigReader: detect_encoding '
'(detected-encoding={encoding}, confidence={confidence})'.format(
encoding=encoding, confidence=confidence))
if confidence is not None and float(
confidence) >= 0.25 and encoding is not None:
config.encoding = encoding
else:
config.encoding = default
logger.debug(
'CmdLineConfigReader: detect_encoding (encoding={encoding})'.
format(encoding=config.encoding))
def _get_dict_reader(self, resource_data_fp):
'''
Returns a dict reader for the given source file pointer. If the encoding of the source
hasn't been guessed yet then this function will read the from the file pointer until EOF and
guess the encoding. The file pointer is reset to the start after this occurs.
:param resource_data_fp: the file pointer to the source
:return: a DictReader object
'''
if self.encoding is None:
with ensure_reset(resource_data_fp):
detector = UniversalDetector()
while True:
chunk = resource_data_fp.read(8192)
if chunk:
detector.feed(chunk)
else:
detector.close()
break
self.encoding = detector.result[u'encoding']
# if the detector failed to work out the encoding (unlikely) or if the encoding it
# comes up with is ASCII, just default to UTF-8 (UTF-8 is a superset of ASCII)
if self.encoding is None or self.encoding == u'ASCII':
self.encoding = u'utf-8'
# create and return the dict reader
line_iterator = iter_universal_lines(resource_data_fp, self.encoding)
return unicodecsv.DictReader(line_iterator, dialect=self.dialect, encoding=self.encoding)
def guessWithChardet(content):
u = UniversalDetector()
for line in content.split(b"\n"):
u.feed(line)
u.close()
result = u.result
return result
def detect_encoding(f: BinaryIO, limit: int = 2500) -> str:
"""
Return encoding of provided input stream.
Most of the time it's unicode, but if we are unable to decode the input
natively, use `chardet` to determine the encoding heuristically.
"""
unicode_decodable = True
for line_no, line in enumerate(f):
try:
line.decode("utf-8")
except UnicodeDecodeError:
unicode_decodable = False
break
if line_no > limit:
break
if unicode_decodable:
return "utf-8"
f.seek(0)
u = UniversalDetector()
for line_no, line in enumerate(f):
u.feed(line)
if u.done or line_no > limit:
break
u.close()
return u.result["encoding"]
def get_file_encoding(filename):
"""
input: filename(str) absolute path of a file
output: str
"""
detector = UniversalDetector()
with open(filename, "rb") as f:
for line, content in enumerate(f):
detector.feed(content)
if line>100:
break
if detector.done:
break
detector.close()
return detector.result
def detect_convert(filename):
detector = UniversalDetector()
detector.reset()
cache = b''
with open(filename, 'rb') as f:
for line in f:
detector.feed(line)
cache += line
if detector.done:
break
detector.close()
cache = cache.decode(detector.result['encoding'] or args.fallback_enc,
errors='ignore')
cache += f.read().decode(detector.result['encoding']
or args.fallback_enc,
errors='ignore')
cf = convertfunc(cache, args.locale, args.locale_only)
return cf(cache)
def detect_encoding(the_file):
'''Detects file encoding using chardet based on N first lines
'''
detector = UniversalDetector()
for line in the_file.readlines():
detector.feed(line)
if detector.done:
break
detector.close()
return detector.result
def SetEncoding(self):
if os.path.isfile(self.filename):
bigdata = open(filename, "rb")
from cchardet import UniversalDetector
detector = UniversalDetector()
for line in bigdata.readlines():
detector.feed(line)
if detector.done:
break
detector.close()
bigdata.close()
result = detector.result
self.encoding = result.get('encoding', self.encoding)
def _file_encoding(self):
detector = UniversalDetector()
for chunk in self.chunks:
for line in chunk.splitlines():
detector.feed(line)
if detector.done:
break
detector.close()
encoding = detector.result['encoding']
return encoding
def detect_encoding(f: BinaryIO, limit: int = 2500) -> str:
u = UniversalDetector()
for line in f:
u.feed(line)
limit -= 1
if u.done or limit < 1:
break
u.close()
if u.result["encoding"].lower() == "ascii":
return "utf-8"
else:
return u.result["encoding"]
def detect_convert(filename):
detector = UniversalDetector()
detector.reset()
cache = b''
with open(filename, 'rb') as f:
for line in f:
detector.feed(line)
cache += line
if detector.done:
break
detector.close()
cache = cache.decode(
detector.result['encoding'] or args.fallback_enc,
errors='ignore')
cache += f.read().decode(
detector.result['encoding'] or args.fallback_enc,
errors='ignore')
cf = convertfunc(cache, args.locale, args.locale_only)
return cf(cache)
def detect_encoding_with_chardet(iostream,
min_lines=10,
max_lines=anycsvconfig.NO_SNIFF_LINES):
detector = UniversalDetector()
if max_lines != -1:
max_lines = max(min_lines, max_lines)
c = 0
for line in iostream:
c += 1
detector.feed(line)
if c > min_lines and (detector.done or min(c, max_lines) == max_lines):
break
detector.close()
return detector.result #{'encoding':, 'confidence':}
def read_file(usr_fil, log=None):
"""Reads a file and returns each line as a list of strings.
:param usr_fil: Required - Input file
:param log: Optional - Log file
:return: str: The file contents as a string
"""
out = ""
try:
print(usr_fil)
# Detect file encoding
file = open(usr_fil, 'rb')
det = UniversalDetector()
for line in file.readlines():
det.feed(line)
if det.done:
break
det.close()
usr_enc = det.result.get("encoding")
out += f"File encoding: {usr_enc}\n"
# Open file using detected encoding
text_read = open(usr_fil, 'r', encoding='UTF-8', errors='ignore')
text = text_read.read()
text_read.close()
return text
except IOError:
out += "\nFile reading failed\n"
return False
finally:
# Log info
if log is not None:
log.write(out)
else:
print(out, end='')
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import sys
import shutil
try:
from cchardet import UniversalDetector
except ImportError:
from chardet.universaldetector import UniversalDetector
detector = UniversalDetector()
arg = sys.argv[1:]
if arg:
fns = list(filter(os.path.isfile, arg))
else:
fns = ['-']
for fn in fns:
if fn == '-':
stream = sys.stdin.buffer
else:
stream = open(fn, 'rb')
detector.reset()
cache = []
for line in stream:
detector.feed(line)
cache.append(line)
if detector.done:
break
def convert(asmfile, opt, verbose, stdout) -> None:
encoding = "utf-8"
requires_manual_conversion = False
try:
with open(asmfile, "r") as f:
text = f.readlines()
except Exception as e:
detector = UniversalDetector()
for line in open(asmfile, "rb"):
detector.feed(line)
if detector.done:
break
detector.close()
encod = detector.result
if encod["confidence"] is not None:
if encod["confidence"] >= 0.5:
encoding = encod["encoding"]
else:
encoding = "SHIFT_JIS"
else:
encoding = "SHIFT_JIS" # if confidence is low, try japanese
try:
if verbose:
print(f"Guessed encoding {encoding}, will try to parse now...")
with open(asmfile, "r", encoding=encoding) as f:
text = f.readlines()
except Exception as e:
raise e # propagate
bw_defs = []
outputfile = asmfile.replace(".asm", "_sa1.asm")
outfile = open(outputfile, "w", encoding=encoding)
stdout.write(bytes(f"Processing file {asmfile}:\n", encoding=encoding))
outlines = []
if opt:
outfile.write("incsrc conv_defines.asm\n")
tot_conversions = 0
for index, line in enumerate(text, start=1):
outlines.append("")
data_types = ["db", "dw", "dl", "dd"]
in_comment = False
in_data = False
define_found = re.match(
r"![A-Za-z\d_]+\s+=\s+((\$)?[\dA-Fa-f]{2,6})\S*", line.strip())
words = re.split(r"([ \t;])", line.rstrip())
if line.strip() == "" or line.lstrip().startswith(";") or define_found:
# shortcuts for comments and blank lines and defines
if define_found:
requires_manual_conversion = True
is_hex = define_found.group(2) is not None
if (int(
define_found.group(1).replace("$", "0x")
if is_hex else define_found.group(1),
16 if is_hex else 10,
) == 12):
stdout.write(
bytes(
f"There is define {define_found.group(0)} at line {index} which is equal to 12,"
f" this might be a define related to how many sprites can be loaded by the game"
f" if so, change it to 22 or $16, or (even better) use the following\n"
f"\tif read1($00FFD5) == $23\n\t\t{define_found.group(0)}\n\telse\n\t\t"
f'{define_found.group(0).split("=")[0]}= {"$16" if is_hex else "22"}\n\tendif\n',
encoding=encoding,
))
elif (int(define_found.group(1).replace("$", "0x"), 16)
in sprite_addr_list and is_hex):
stdout.write(
bytes(
f"There is define {define_found.group(0)} at line {index} which is a sprite "
f"address, usually replacing the $ with ! works in most tools, it didn't get "
f"converted automatically because it might not be necessary to do so, make sure "
f"to convert manually it ONLY if needed.\n",
encoding=encoding,
))
elif (0x0100 <= int(
define_found.group(1).replace("$", "0x")
if is_hex else define_found.group(1),
16 if is_hex else 10,
) <= 0x1FFF):
stdout.write(
bytes(
f"There is define {define_found.group(0)} at line {index} which might be a ram"
f" address, if it is, convert it by adding |!addr at the end of it, if it's not"
f" a ram address leave it alone\n",
encoding=encoding,
))
outlines[index - 1] = line.rstrip()
continue
ignore_next_address = False
for og_word in words:
stripped_word = og_word.strip()
to_insert = ""
if in_comment or in_data:
pass
elif stripped_word.startswith(";"):
in_comment = True
elif any([stripped_word.startswith(a) for a in data_types]):
in_data = True
elif stripped_word.startswith("PEA") or stripped_word.startswith(
"PER"):
ignore_next_address = True
elif addr := re.findall(r"\$[\da-fA-F]{1,6}\|![a-zA-Z\d_]+\b",
og_word):
stdout.write(
bytes(
f"Possibly address {addr[0]} at line {index} was already hybrid.\n",
encoding=encoding,
))
elif re.findall(r"\$[^, \n()\[\]]{1,6}", og_word):
if ignore_next_address:
ignore_next_address = False
outlines[index - 1] += og_word
continue
splitted = re.split(r"([\[\](), ])", og_word)
word_tuples = []
for i, word in enumerate(splitted):
if word.startswith("$"):
try:
proc_word = eval(word.replace("$", "0x"))
expr = re.split(
r"[+\\\-^*~<>|]", word.replace(
"$", "")) # +\-^*~<> some asar math ops
word = "${:0{}X}".format(
proc_word, max([len(e) for e in expr]))
word_tuples.append((WordType.ADDR, word, i))
except SyntaxError:
bunch = re.split(r"([+\-^*~<>| ])", word)
for w in bunch:
if w.startswith("$"):
word_tuples.append((WordType.ADDR, w, i))
else:
word_tuples.append((WordType.OTHER, w, i))
elif word.startswith(","):
word_tuples.append((WordType.COMMA, word, i))
else:
word_tuples.append((WordType.OTHER, word, i))
for wordtype, word, i in word_tuples:
if wordtype == WordType.ADDR:
try:
try:
comma_index = (i + 1 if word_tuples[i + 1][0]
== WordType.COMMA else -1)
except IndexError:
comma_index = -1
(
ww,
bwram_define_needed,
converted,
manual_conversion,
) = process_word(
word.replace("$", ""),
stdout,
encoding,
index,
splitted,
comma_index,
)
if manual_conversion:
requires_manual_conversion = True
if converted:
tot_conversions += 1
stdout.write(
bytes(
f"Conversion: {word} -> {ww}\n",
encoding=encoding,
))
bw_defs.append(bwram_define_needed)
to_insert += ww
except ValueError:
to_insert += word
else:
to_insert += word
outlines[index - 1] += to_insert if to_insert != "" else og_word
