def __init__(self, baseurl, cachefile, login, password=None):
""" Create a nashi client
Parameters
----------
baseurl : web address of nashi instance
cachefile : filename of hdf5-cache
login : user for nashi
password : asks for user input if empty
"""
self.baseurl = baseurl
self.session = None
self.traindata = None
self.recogdata = None
self.valdata = None
self.bookcache = {}
self.cachefile = cachefile
self.login(login, password)
params = DataPreprocessorParams()
params.line_height = 48
params.pad = 16
params.pad_value = 1
params.no_invert = False
params.no_transpose = False
self.data_proc = MultiDataProcessor([
DataRangeNormalizer(),
CenterNormalizer(params),
FinalPreparation(params, as_uint8=True),
])
# Text pre processing (reading)
preproc = TextProcessorParams()
preproc.type = TextProcessorParams.MULTI_NORMALIZER
default_text_normalizer_params(preproc.children.add(), default="NFC")
default_text_regularizer_params(preproc.children.add(), groups=["extended"])
strip_processor_params = preproc.children.add()
strip_processor_params.type = TextProcessorParams.STRIP_NORMALIZER
self.txt_preproc = text_processor_from_proto(preproc, "pre")
# Text post processing (prediction)
postproc = TextProcessorParams()
postproc.type = TextProcessorParams.MULTI_NORMALIZER
default_text_normalizer_params(postproc.children.add(), default="NFC")
default_text_regularizer_params(postproc.children.add(), groups=["extended"])
strip_processor_params = postproc.children.add()
strip_processor_params.type = TextProcessorParams.STRIP_NORMALIZER
self.text_postproc = text_processor_from_proto(postproc, "post")
# BIDI text preprocessing
bidi_processor_params = preproc.children.add()
bidi_processor_params.type = TextProcessorParams.BIDI_NORMALIZER
bidi_processor_params.bidi_direction = TextProcessorParams.BIDI_RTL
self.bidi_preproc = text_processor_from_proto(preproc, "pre")
bidi_processor_params = postproc.children.add()
bidi_processor_params.type = TextProcessorParams.BIDI_NORMALIZER
bidi_processor_params.bidi_direction = TextProcessorParams.BIDI_AUTO
self.bidi_postproc = text_processor_from_proto(postproc, "post")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--files",
type=str,
nargs="+",
required=True,
help="Text files to apply text processing")
parser.add_argument("--line_height",
type=int,
default=48,
help="The line height")
parser.add_argument("--pad",
type=int,
default=16,
help="Padding (left right) of the line")
parser.add_argument("--pad_value",
type=int,
default=1,
help="Padding (left right) of the line")
parser.add_argument("--processes", type=int, default=1)
parser.add_argument("--verbose", action="store_true")
parser.add_argument("--invert", action="store_true")
parser.add_argument("--transpose", action="store_true")
parser.add_argument("--dry_run",
action="store_true",
help="No not overwrite files, just run")
args = parser.parse_args()
params = DataPreprocessorParams()
params.line_height = args.line_height
params.pad = args.pad
params.pad_value = args.pad_value
params.no_invert = not args.invert
params.no_transpos = not args.transpose
data_proc = MultiDataProcessor([
DataRangeNormalizer(),
CenterNormalizer(params),
FinalPreparation(params, as_uint8=True),
])
print("Resolving files")
img_files = sorted(glob_all(args.files))
handler = Handler(data_proc, args.dry_run)
with multiprocessing.Pool(processes=args.processes,
maxtasksperchild=100) as pool:
list(
tqdm(pool.imap(handler.handle_single, img_files),
desc="Processing",
total=len(img_files)))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--files", type=str, nargs="+", required=True,
help="Text files to apply text processing")
parser.add_argument("--line_height", type=int, default=48,
help="The line height")
parser.add_argument("--pad", type=int, default=16,
help="Padding (left right) of the line")
parser.add_argument("--pad_value", type=int, default=1,
help="Padding (left right) of the line")
parser.add_argument("--processes", type=int, default=1)
parser.add_argument("--verbose", action="store_true")
parser.add_argument("--invert", action="store_true")
parser.add_argument("--transpose", action="store_true")
parser.add_argument("--dry_run", action="store_true",
help="No not overwrite files, just run")
args = parser.parse_args()
params = DataPreprocessorParams()
params.line_height = args.line_height
params.pad = args.pad
params.pad_value = args.pad_value
params.no_invert = not args.invert
params.no_transpos = not args.transpose
data_proc = MultiDataProcessor([
DataRangeNormalizer(),
CenterNormalizer(params),
FinalPreparation(params, as_uint8=True),
])
print("Resolving files")
img_files = sorted(glob_all(args.files))
handler = Handler(data_proc, args.dry_run)
with multiprocessing.Pool(processes=args.processes, maxtasksperchild=100) as pool:
list(tqdm(pool.imap(handler.handle_single, img_files), desc="Processing", total=len(img_files)))