def main(count=None, start=0): # Collect inputs with open(input_file) as fp: inputs = fp.readlines() num_inputs = len(inputs) stop = start + count if count else None inputs = itertools.islice(inputs, start, stop) max_value = count or num_inputs - start # Write the reference outputs with open(ref_file) as fp, open(ref_output_file, "w") as out: refs = fp.readlines() refs = itertools.islice(refs, stop) out.write("".join(refs)) # Create the objects for scoring inria = LexicalSplitMetrics("inria", 10) combined = LexicalSplitMetrics("combined", 10) bar = progressbar.ProgressBar(max_value=max_value) with outputctx(strict_io=False): for line in bar(inputs): s = SanskritObject(line.strip(), encoding=SLP1, replace_ending_visarga=None) logger.debug("Input in SLP1 = %s", s.canonical()) # Compute splits inria.update(s) combined.update(s) print("{:20s} | {:30s} | {:5s}".format("Name", "BLEU", "CHRF")) print("-" * 70) inria.print_metrics() combined.print_metrics()
def test_file_splits(lexan, splittext_refs): f = splittext_refs[0] s = splittext_refs[1] with outputctx(False): i = SanskritObject(f, encoding=SLP1, strict_io=True, replace_ending_visarga=None) graph = lexan.getSandhiSplits(i) assert graph is not None splits = graph.find_all_paths(max_paths=300, sort=False) assert s in [list(map(str, ss)) for ss in splits]
def main(): args = getArgs() if args.input_encoding is None: ie = None else: ie = SCHEMES[args.input_encoding] # Setup logging if args.loglevel: numeric_level = getattr(logging, args.loglevel.upper(), None) if not isinstance(numeric_level, int): raise ValueError('Invalid log level: %s' % args.loglevel) logging.basicConfig(filename="sandhi.log", filemode="wb", level=numeric_level) logging.info("---------------------------------------------------") logging.info("Started processing at %s", datetime.datetime.now()) sandhi = Sandhi() # if neither split nor join is chosen, just demo both if not args.split and not args.join: print( "Neither split nor join option chosen. Here's a demo of joining" ) args.join = True with outputctx(args.strict_io): if args.split: word_in = SanskritNormalizedString(args.word, encoding=ie, strict_io=args.strict_io) if args.all: print("All possible splits for {}".format(args.word)) splits = sandhi.split_all(word_in) else: pos = int(args.word_or_pos) print("Splitting {0} at {1}".format(args.word, pos)) splits = sandhi.split_at(word_in, pos) print(splits) if args.join: print("Joining {0} {1}".format(args.word, args.word_or_pos)) first_in = SanskritNormalizedString(args.word, encoding=ie, strict_io=args.strict_io) second_in = SanskritNormalizedString(args.word_or_pos, encoding=ie, strict_io=args.strict_io) joins = sandhi.join(first_in, second_in) print(joins) logging.info("Finished processing at %s", datetime.datetime.now()) logging.info("---------------------------------------------------") logging.shutdown()
def main(): global need_lakara args = getArgs() print("Input String:", args.data) need_lakara = args.need_lakara if args.debug: logging.basicConfig(filename='SanskritMorphologicalAnalyzer.log', filemode='w', level=logging.DEBUG) s = SanskritMorphologicalAnalyzer(args.lexical_lookup) if args.input_encoding is None: ie = None else: ie = SanskritBase.SCHEMES[args.input_encoding] i = SanskritBase.SanskritObject(args.data, encoding=ie, strict_io=args.strict_io, replace_ending_visarga=None) print("Input String in SLP1:", i.canonical()) import time print("Start Split") start_split = time.time() graph = s.getSandhiSplits(i, tag=True) end_split = time.time() print("End DAG generation") with SanskritBase.outputctx(args.strict_io): if graph: start_path = time.time() splits = graph.findAllPaths(max_paths=args.max_paths) end_path = time.time() print("End pathfinding") print("Splits:") for sp in splits: print("Lexical Split:", sp) p = s.constrainPath(sp) if p: print("Valid Morphologies") for pp in p: print([(spp, pp[str(spp)]) for spp in sp]) else: print("No valid morphologies for this split") print("End Morphological Analysis") print("-----------") print("Performance") print("Time taken for split: {0:0.6f}s".format(end_split-start_split)) print("Time taken for path: {0:0.6f}s".format(end_path-start_path)) else: print("No Valid Splits Found") return
def test_file_splits(lexan, kosh_entry): clean_input = False f = kosh_entry['Word'] s = kosh_entry['Split'] clean_input = True with outputctx(False): i = SanskritObject(f, encoding=DEVANAGARI, strict_io=True, replace_ending_visarga=None) if clean_input: sl = [SanskritObject(x, encoding=DEVANAGARI, strict_io=True, replace_ending_visarga=None).canonical() for x in s.strip().replace(" ", "+").split('+')] else: sl = [SanskritObject(x, encoding=DEVANAGARI, strict_io=True, replace_ending_visarga=None).canonical() for x in s.split('+')] graph = lexan.getSandhiSplits(i) assert graph is not None splits = graph.find_all_paths(max_paths=300, sort=False) assert sl in [list(map(str, ss)) for ss in splits]
def tags(argv=None): args = getTagsArgs(argv) if args.strict_io: print("Interpreting input strictly") else: print("Interpreting input loosely (strict_io set to false)") logger.info(f"Input String: {args.data}") if args.input_encoding is None: ie = None else: ie = SCHEMES[args.input_encoding] s = LexicalSandhiAnalyzer(args.lexical_lookup) with outputctx(args.strict_io): i = SanskritNormalizedString(args.data, encoding=ie, strict_io=args.strict_io, replace_ending_visarga='s') print("Input String in SLP1:", i.canonical()) ts = s.getMorphologicalTags(i, tmap=args.map_tags) print("Morphological tags:") if ts is not None: for t in ts: print(t) # Possible rakaranta # Try by replacing end visarga with 'r' instead elif not args.strict_io: i = SanskritNormalizedString(args.data, encoding=ie, strict_io=args.strict_io, replace_ending_visarga='r') ts = s.getMorphologicalTags(i) if ts is not None: print("Input String in SLP1:", i.canonical()) for t in ts: print(t) if args.tag_set or args.base: if args.tag_set is not None: g = set(args.tag_set) else: g = None if args.base is not None: b = SanskritNormalizedString(args.base) else: b = None print(s.hasTag(i, b, g))
def sort_file_splits(kosh_entry): global le clean_input = False f = kosh_entry['Word'] s = kosh_entry['Split'] clean_input = True with outputctx(False): i = SanskritObject(f, encoding=DEVANAGARI, strict_io=True, replace_ending_visarga=None) if not (isinstance(s, str)): # Bad input return "Bad_Input" if clean_input: sl = [ SanskritObject(x, encoding=DEVANAGARI, strict_io=True, replace_ending_visarga=None).canonical() for x in s.strip().replace(" ", "+").split('+') ] else: sl = [ SanskritObject(x, encoding=DEVANAGARI, strict_io=True, replace_ending_visarga=None).canonical() for x in s.split('+') ] graph = le.getSandhiSplits(i) if graph is not None: splits = graph.find_all_paths(max_paths=300, sort=False) if sl in [list(map(str, ss)) for ss in splits]: # Pass print("P", end='', flush=True) return "Pass" else: print("F", end='', flush=True) return "Fail" else: print("S", end='', flush=True) return "Split_Fail"
def main(argv=None): graph = None args = getArgs(argv) if args.strict_io: print("Interpreting input strictly") else: print("Interpreting input loosely (strict_io set to false)") print("Input String:", args.data) s = LexicalSandhiAnalyzer(args.lexical_lookup) if args.input_encoding is None: ie = None else: ie = SanskritBase.SCHEMES[args.input_encoding] with SanskritBase.outputctx(args.strict_io): if not args.split: i = SanskritBase.SanskritNormalizedString( args.data, encoding=ie, strict_io=args.strict_io, replace_ending_visarga='s') print("Input String in SLP1:", i.canonical()) ts = s.getMorphologicalTags(i, tmap=args.map_tags) print("Morphological tags:") if ts is not None: for t in ts: print(t) # Possible rakaranta # Try by replacing end visarga with 'r' instead elif not args.strict_io: i = SanskritBase.SanskritNormalizedString( args.data, encoding=ie, strict_io=args.strict_io, replace_ending_visarga='r') ts = s.getMorphologicalTags(i) if ts is not None: print("Input String in SLP1:", i.canonical()) for t in ts: print(t) if args.tag_set or args.base: if args.tag_set is not None: g = set(args.tag_set) else: g = None if args.base is not None: b = SanskritBase.SanskritNormalizedString(args.base) else: b = None print(s.hasTag(i, b, g)) else: import time i = SanskritBase.SanskritNormalizedString( args.data, encoding=ie, strict_io=args.strict_io, replace_ending_visarga=None) print("Input String in SLP1:", i.canonical()) print("Start Split") start_split = time.time() graph = s.getSandhiSplits(i) end_graph = time.time() print("End DAG generation") if graph: logger.debug("Graph has %d nodes and %d edges" % (len(graph.G.nodes()), len(graph.G.edges()))) splits = graph.find_all_paths(max_paths=args.max_paths, score=args.score) print("End pathfinding", time.time()) print("Splits:") if splits: for split in splits: print(split) else: print("None") else: print("No Valid Splits Found") end_split = time.time() print("-----------") print("Performance") print("Time for graph generation = {0:0.6f}s".format(end_graph - start_split)) print("Total time for graph generation + find paths = {0:0.6f}s". format(end_split - start_split)) return graph
def main(): args = getArgs() if args.strict_io: print("Interpreting input strictly") else: print("Interpreting input loosely (strict_io set to false)") print("Input String:", args.data) if args.debug: logging.basicConfig(filename='SanskritLexicalAnalyzer.log', filemode='w', level=logging.DEBUG) else: logging.basicConfig(filename='SanskritLexicalAnalyzer.log', filemode='w', level=logging.INFO) s = SanskritLexicalAnalyzer(args.lexical_lookup) if args.input_encoding is None: ie = None else: ie = SanskritBase.SCHEMES[args.input_encoding] with SanskritBase.outputctx(args.strict_io): if not args.split: i = SanskritBase.SanskritObject(args.data, encoding=ie, strict_io=args.strict_io, replace_ending_visarga='s') print("Input String in SLP1:", i.canonical()) ts = s.getLexicalTags(i) print(ts) # Possible rakaranta # Try by replacing end visarga with 'r' instead if not args.strict_io: i = SanskritBase.SanskritObject(args.data, encoding=ie, strict_io=args.strict_io, replace_ending_visarga='r') ts = s.getLexicalTags(i) if ts is not None: print("Input String in SLP1:", i.canonical()) print(ts) if args.tag_set or args.base: if args.tag_set: g = set(args.tag_set) print( s.hasTag(i, SanskritBase.SanskritObject(args.base), g)) else: import time i = SanskritBase.SanskritObject(args.data, encoding=ie, strict_io=args.strict_io, replace_ending_visarga=None) print("Input String in SLP1:", i.canonical()) print("Start Split") start_split = time.time() graph = s.getSandhiSplits(i) end_graph = time.time() print("End DAG generation") if graph: logger.debug("Graph has %d nodes and %d edges" % (len(graph.G.nodes()), len(graph.G.edges()))) splits = graph.findAllPaths(max_paths=args.max_paths, score=args.score) print("End pathfinding", time.time()) print("Splits:") if splits: for split in splits: print(split) else: print("None") else: print("No Valid Splits Found") end_split = time.time() print("-----------") print("Performance") print("Time for graph generation = {0:0.6f}s".format( end_graph - start_split)) print( "Total time for graph generation + find paths = {0:0.6f}s". format(end_split - start_split))