def cli_detect(argv=None):
parser = argparse.ArgumentParser(
description="The Real First Universal Charset Detector. "
"Discover originating encoding used on text file. "
"Normalize text to unicode.")
parser.add_argument('file',
type=argparse.FileType('rb'),
nargs='+',
help='Filename')
parser.add_argument(
'--verbose',
action="store_true",
default=False,
dest='verbose',
help='Display complementary information about file if any.')
parser.add_argument(
'--normalize',
action="store_true",
default=False,
dest='normalize',
help=
'Permit to normalize input file. If not set, program does not write anything.'
)
parser.add_argument(
'--replace',
action="store_true",
default=False,
dest='replace',
help=
'Replace file when trying to normalize it instead of creating a new one.'
)
parser.add_argument(
'--force',
action="store_true",
default=False,
dest='force',
help=
'Replace file without asking if you are sure, use this flag with caution.'
)
args = parser.parse_args(argv)
if len(args.file) == 0:
print(
'This command purpose is to analyse text file. Please specify any filename.',
file=sys.stderr)
parser.print_help(file=sys.stderr)
return 1
if args.replace is True and args.normalize is False:
print('Use --replace in addition of --normalize only.',
file=sys.stderr)
return 1
if args.force is True and args.replace is False:
print('Use --force in addition of --replace only.', file=sys.stderr)
return 1
for my_file in args.file:
matches = CharsetNormalizerMatches.from_fp(my_file)
if len(matches) == 0:
print('Unable to identify originating encoding for "{}".'.format(
my_file.name),
file=sys.stderr)
continue
x_ = PrettyTable([
'Filename', 'Encoding', 'Language', 'Alphabets', 'Chaos',
'Coherence'
])
r_ = matches.best()
p_ = r_.first()
x_.add_row([
my_file.name, p_.encoding, p_.language,
(' and ' if len(p_.alphabets) < 4 else '\n').join([
el if 'and' not in el else '"{}"'.format(el)
for el in p_.alphabets
]), '{} %'.format(round(p_.chaos * 100., ndigits=3)),
'{} %'.format(round(100. - p_.coherence * 100., ndigits=3))
])
if len(matches) > 1 and args.verbose:
for el in matches:
if el != p_:
x_.add_row([
'** ALTERNATIVE ' + my_file.name + '**', el.encoding,
el.language,
(' and ' if len(el.alphabets) < 4 else '\n').join([
el if 'and' not in el else '"{}"'.format(el)
for el in el.alphabets
]), '{} %'.format(round(el.chaos * 100., ndigits=3)),
'{} %'.format(
round(100. - el.coherence * 100., ndigits=3))
])
print(x_)
if args.verbose is True:
print('"{}" could be also originating from {}.'.format(
my_file.name, ','.join(r_.could_be_from_charset)))
print('"{}" could be also be written in {}.'.format(
my_file.name, ' or '.join(p_.languages)))
if args.normalize is True:
if p_.encoding.startswith('utf') is True:
print(
'"{}" file does not need to be normalized, as it already came from unicode.'
)
continue
o_ = my_file.name.split('.') # type: list[str]
if args.replace is False:
o_.insert(-1, p_.encoding)
else:
if args.force is False and query_yes_no(
'Are you sure to normalize "{}" by replacing it ?'.
format(my_file.name), 'no') is False:
continue
try:
with open('./{}'.format('.'.join(o_)), 'w',
encoding='utf-8') as fp:
fp.write(str(p_))
except IOError as e:
print(str(e), file=sys.stderr)
return 2
return 0
def cli_detect(argv=None):
"""
CLI assistant using ARGV and ArgumentParser
:param argv:
:return: 0 if everything is fine, anything else equal trouble
"""
parser = argparse.ArgumentParser(
description="The Real First Universal Charset Detector. "
"Discover originating encoding used on text file. "
"Normalize text to unicode."
)
parser.add_argument('file', type=argparse.FileType('rb'), nargs='+', help='Filename')
parser.add_argument('--verbose', action="store_true", default=False, dest='verbose',
help='Display complementary information about file if any.')
parser.add_argument('--normalize', action="store_true", default=False, dest='normalize',
help='Permit to normalize input file. If not set, program does not write anything.')
parser.add_argument('--replace', action="store_true", default=False, dest='replace',
help='Replace file when trying to normalize it instead of creating a new one.')
parser.add_argument('--force', action="store_true", default=False, dest='force',
help='Replace file without asking if you are sure, use this flag with caution.')
parser.add_argument('--threshold', action="store", default=0.2, type=float, dest='threshold',
help="Define a custom maximum amount of chaos allowed in decoded content. 0. <= chaos <= 1.")
args = parser.parse_args(argv)
if len(args.file) == 0:
print('This command purpose is to analyse text file. Please specify any filename.', file=sys.stderr)
parser.print_help(file=sys.stderr)
return 1
if args.replace is True and args.normalize is False:
print('Use --replace in addition of --normalize only.', file=sys.stderr)
return 1
if args.force is True and args.replace is False:
print('Use --force in addition of --replace only.', file=sys.stderr)
return 1
if args.threshold < 0. or args.threshold > 1.:
print('--threshold VALUE should be between 0. AND 1.')
return 1
for my_file in args.file:
matches = CharsetNormalizerMatches.from_fp(
my_file,
threshold=args.threshold
)
if len(matches) == 0:
print('Unable to identify originating encoding for "{}". {}'.format(my_file.name, 'Maybe try increasing maximum amount of chaos.' if args.threshold < 1. else ''), file=sys.stderr)
if my_file.closed is False:
my_file.close()
continue
x_ = PrettyTable(['Filename', 'Encoding', 'Language', 'Alphabets', 'Chaos', 'Coherence'])
r_ = matches.best()
p_ = r_.first()
x_.add_row(
[
my_file.name,
p_.encoding,
p_.language,
(' and ' if len(p_.alphabets) < 4 else '\n').join([el if 'and' not in el else '"{}"'.format(el) for el in p_.alphabets]),
'{} %'.format(round(p_.chaos * 100., ndigits=3)),
'{} %'.format(round(100. - p_.coherence * 100., ndigits=3))
]
)
if len(matches) > 1 and args.verbose:
for el in matches:
if el != p_:
x_.add_row(
[
'** ALTERNATIVE '+my_file.name+'**',
el.encoding,
el.language,
(' and ' if len(el.alphabets) < 4 else '\n').join([el if 'and' not in el else '"{}"'.format(el) for el in el.alphabets]),
'{} %'.format(round(el.chaos * 100., ndigits=3)),
'{} %'.format(round(100. - el.coherence * 100., ndigits=3))
]
)
print(x_)
if args.verbose is True:
if len(r_.could_be_from_charset) > 1:
print('"{}" could be also originating from {}.'.format(my_file.name, ','.join(r_.could_be_from_charset)))
if len(p_.could_be_from_charset) > 1:
print('"{}" produce the EXACT same output with those encoding : {}.'.format(my_file.name, ' OR '.join(p_.could_be_from_charset)))
if len(p_.languages) > 1:
print('"{}" could be also be written in {}.'.format(my_file.name, ' or '.join(p_.languages)))
if p_.byte_order_mark is True:
print('"{}" has a signature or byte order mark (BOM) in it.'.format(my_file.name))
if args.normalize is True:
if p_.encoding.startswith('utf') is True:
print('"{}" file does not need to be normalized, as it already came from unicode.'.format(my_file.name))
if my_file.closed is False:
my_file.close()
continue
o_ = my_file.name.split('.') # type: list[str]
if args.replace is False:
o_.insert(-1, p_.encoding)
if my_file.closed is False:
my_file.close()
else:
if args.force is False and query_yes_no(
'Are you sure to normalize "{}" by replacing it ?'.format(my_file.name), 'no') is False:
if my_file.closed is False:
my_file.close()
continue
try:
with open('./{}'.format('.'.join(o_)), 'w', encoding='utf-8') as fp:
fp.write(
str(p_)
)
print('"{}" has been successfully written to disk.'.format('.'.join(o_)))
except IOError as e:
print(str(e), file=sys.stderr)
if my_file.closed is False:
my_file.close()
return 2
if my_file.closed is False:
my_file.close()
return 0