def main_swap(args): print('Swapping inhibiting layers between', args.project, 'and', args.project2) project = Project.load(args.project) project2 = Project.load(args.project2) if len(project.patternset) != len(project2.patternset): raise ValueError( 'You can only swap layers between projects with same number of layers!' ) for i in range(1, len(project.patternset), 2): project.patternset[i], project2.patternset[i] = project2.patternset[ i], project.patternset[i] print('Performance of', args.project) project.missed, project.false = do_test(project, project.dictionary) print('Performance of', args.project2) project2.missed, project2.false = do_test(project2, project2.dictionary) if args.commit: project.save(args.project) project2.save(args.project2) print('...Committed') else: print('...Projects NOT changed (use --commit flag to save changes)') print() return 0
def main_swap(args): print('Swapping inhibiting layers between', args.project, 'and', args.project2) project = Project.load(args.project) project2 = Project.load(args.project2) if len(project.patternset) != len(project2.patternset): raise ValueError('You can only swap layers between projects with same number of layers!') for i in range(1, len(project.patternset), 2): project.patternset[i], project2.patternset[i] = project2.patternset[i], project.patternset[i] print('Performance of', args.project) project.missed, project.false = do_test(project, project.dictionary) print('Performance of', args.project2) project2.missed, project2.false = do_test(project2, project2.dictionary) if args.commit: project.save(args.project) project2.save(args.project2) print('...Committed') else: print('...Projects NOT changed (use --commit flag to save changes)') print() return 0
def main_new(args): print('Creating new project', args.project, 'from dictionary', args.dictionary) if os.path.exists(args.project): print( 'Project file already exists! Use different name or delete old project first. File %s' % args.project) return -1 if not os.path.exists(args.dictionary): print('Dictionary file not found', args.dictionary) return -1 dictionary = Dictionary.load(args.dictionary) if args.margins is None: print('Automatically computing hyphenation margins from dictionary') margins = dictionary.compute_margins() else: margins = Margins.parse(args.margins) project = Project(dictionary, margins) project.save(args.project) return main_show(args)
def main_show(args): if not os.path.exists(args.project): print('Project file not found:', args.project) return -1 project = Project.load(args.project) print('Project file', args.project) print('\tcreated:', project.created) print('\tlast modified:', project.modified) print('\tmargins:', project.margins) print('\tdictionary size:', len(project.dictionary.keys())) #if project.ignore_weights: # print('\tdictionary weights were ignored (-i flag active)') print('\ttotal hyphens: (weighted)', project.total_hyphens) print('\ttotal missed : (weighted)', project.missed, percent(project.missed, project.total_hyphens)) print('\ttotal false : (weighted)', project.false, percent(project.false, project.total_hyphens)) print('\tnumber of pattern levels:', len(project.patternset)) for i, layer in enumerate(project.patternset): if i & 1 == 0: print((i+1), 'HYPHENATING patternset, num patterns:', len(layer)) else: print((i+1), 'INHIBITING patternset, num patterns:', len(layer)) print('\tTrained with: range %r, selector %r' % (layer.patlen_range, layer.selector)) print() return 0
def main_train(args): print('Training project', args.project, 'using range', args.range, 'and selector', args.selector) project = Project.load(args.project) if len(project.patternset) & 1: print('Training INHIBINTING pattern layer (level=%s)' % (len(project.patternset)+1)) else: print('Training HYPHENATION pattern layer (level=%s)' % (len(project.patternset)+1)) patlen_rng = Range.parse(args.range) selector = Selector.parse(args.selector) print('\tpattern lengths:', patlen_rng) print('\tselector:', args.selector) total_hyphens = project.total_hyphens project.train_new_layer(patlen_rng, selector) missed, false = project.missed, project.false print('Missed (weighted):', missed, percent(missed, total_hyphens)) print('False (weighted):', false, percent(false, total_hyphens)) if args.commit: project.save(args.project) print('...Committed!') else: print('...Projects NOT changed (use --commit flag to save changes)') print() return 0
def main_show(args): if not os.path.exists(args.project): print('Project file not found:', args.project) return -1 project = Project.load(args.project) print('Project file', args.project) print('\tcreated:', project.created) print('\tlast modified:', project.modified) print('\tmargins:', project.margins) print('\tdictionary size:', len(project.dictionary.keys())) #if project.ignore_weights: # print('\tdictionary weights were ignored (-i flag active)') print('\ttotal hyphens: (weighted)', project.total_hyphens) print('\ttotal non-hyphens: ', project.total_nonhyphens) print('\ttotal missed : (weighted)', project.missed, percent(project.missed, project.total_hyphens)) print('\ttotal false : (weighted)', project.false, percent(project.false, project.total_nonhyphens)) print('\tnumber of pattern levels:', len(project.patternset)) for i, layer in enumerate(project.patternset): if i & 1 == 0: print((i + 1), 'HYPHENATING patternset, num patterns:', len(layer)) else: print((i + 1), 'INHIBITING patternset, num patterns:', len(layer)) print('\tTrained with: range %r, selector %r' % (layer.patlen_range, layer.selector)) print() return 0
def main_compact(args): print('Compacting hyphenation patterns for', args.project) project = Project.load(args.project) before_compact = [ layer.compute_num_patterns() for layer in project.patternset ] project.patternset.compact() after_compact = [ layer.compute_num_patterns() for layer in project.patternset ] print('Result:') for level0, (before, after) in enumerate(zip(before_compact, after_compact)): print('\tLevel %s: %6d => %6d' % (level0 + 1, before, after)) if args.commit: project.save(args.project) print('...Committed') else: print('...Project NOT changed (use --commit flag to save changes)') print() return 0
def main_test(args): print('Testing', args.project, 'on dictionary', args.dictionary) project = Project.load(args.project) dictionary = Dictionary.load(args.dictionary) print('Performance of', args.project, 'on', args.dictionary) do_test(project, dictionary) if args.errors: with codecs.open(args.errors, 'w', 'utf-8') as f: for word, hyphens, missed, false in project.patternset.errors( dictionary, project.margins): f.write( format_dictionary_word(word, hyphens, missed, false) + '\n') print('Saved errors to', args.errors) if args.patterns: with codecs.open(args.patterns, 'w', 'utf-8') as f: for word, hyphens, missed, false in project.patternset.errors( dictionary, project.margins): f.write(format_word_as_pattern(word, missed, false) + '\n') print('Saved errors to', args.patterns) print() return 0
def main_explain(args): print('Explaining hyphenation of', args.input, 'into', args.output, 'using project', args.project) project = Project.load(args.project) with codecs.open(args.input or sys.stdin.fileno(), 'r', 'utf-8') as f: with codecs.open(args.output or sys.stdout.fileno(), 'w', 'utf-8') as out: for word in f: word = word.strip() if not word: continue explain = Explain() prediction = project.patternset.hyphenate_explain( word, margins=project.margins, explain=explain) s = format_dictionary_word(word, prediction) out.write(s + '\n') out.write(s.encode('unicode-escape').decode('ascii') + '\n') out.write(explain.format() + '\n\n') print() return 0
def main_import(args): print('Loading patterns from', args.input, 'into project', args.project) project = Project.load(args.project) if len(project.patternset) > 0: print( 'ERROR: project already has some patterns. Can only load into empty project!' ) return -1 patterns = {} entered = False with codecs.open(args.input, 'r', 'utf-8') as f: for line in f: line = line.strip() line = line.split('%')[0] if not line: continue if line == '\\patterns{': entered = True elif entered and line == '}': break elif entered: text, control = PatternSet.parse_pattern(line) patterns[text] = control if patterns: maxlevel = 0 for control in patterns.values(): for level in control.values(): maxlevel = max(maxlevel, level) patlen = max(len(text) for text in patterns.keys()) for i in range(maxlevel): project.patternset.append( Layer(Range(1, patlen + 2), None, i & 2 == 1)) for text, control in patterns.items(): project.patternset.set_pattern_control(text, control) else: print('WARNING: patterns file is empty!') project.missed, project.false = do_test(project, project.dictionary) if args.commit: project.save(args.project) print('...Committed') else: print('...Project NOT changed (use --commit flag to save changes)') print() return 0
def main_import(args): print('Loading patterns from', args.input, 'into project', args.project) project = Project.load(args.project) if len(project.patternset) > 0: print('ERROR: project already has some patterns. Can only load into empty project!') return -1 patterns = {} entered = False with codecs.open(args.input, 'r', 'utf-8') as f: for line in f: line = line.strip() line = line.split('%')[0] if not line: continue if line == '\\patterns{': entered = True elif entered and line == '}': break elif entered: text, control = PatternSet.parse_pattern(line) patterns[text] = control if patterns: maxlevel = 0 for control in patterns.values(): for level in control.values(): maxlevel = max(maxlevel, level) patlen = max(len(text) for text in patterns.keys()) for i in range(maxlevel): project.patternset.append(Layer(Range(1, patlen+2), None, i & 2 == 1)) for text, control in patterns.items(): project.patternset.set_pattern_control(text, control) else: print('WARNING: patterns file is empty!') project.missed, project.false = do_test(project, project.dictionary) if args.commit: project.save(args.project) print('...Committed') else: print('...Project NOT changed (use --commit flag to save changes)') print() return 0
def main_new(args): print('Creating new project', args.project, 'from dictionary', args.dictionary) if os.path.exists(args.project): print('Project file already exists! Use different name or delete old project first. File %s' % args.project) return -1 if not os.path.exists(args.dictionary): print('Dictionary file not found', args.dictionary) return -1 dictionary = Dictionary.load(args.dictionary) if args.margins is None: print('Automatically computing hyphenation margins from dictionary') margins = dictionary.compute_margins() else: margins = Margins.parse(args.margins) project = Project(dictionary, margins) project.save(args.project) return main_show(args)
def main_export(args): print('Exporting patterns from', args.project, 'and saving them in TeX format to', args.output) if os.path.exists(args.output): print( 'Pattern file already exists! Delete it first, or change the name. Pattern file: %s' % args.output) return -1 project = Project.load(args.project) pattern_strings = list(project.patternset.pattern_strings()) exceptions = list( project.patternset.errors(project.dictionary, project.margins)) with codecs.open(args.output, 'w', 'utf-8') as f: f.write('\\patterns{\n') for patt in pattern_strings: f.write(patt + '\n') f.write('}\n') f.write('\\hyphenation{\n') for word, hyphens, _, _ in exceptions: text = format_dictionary_word(word, hyphens) f.write(text + '\n') f.write('}\n') print('Created TeX patterns file', args.output) print('Number of patterns:', len(pattern_strings)) print('Number of exceptions:', len(exceptions)) if args.patterns: print() with codecs.open(args.patterns, 'w', 'utf-8') as f: for patt in pattern_strings: f.write(patt + '\n') print('Written raw patterns to', args.patterns) if args.exceptions: print() with codecs.open(args.exceptions, 'w', 'utf-8') as f: for word, hyphens, _, _ in exceptions: text = format_dictionary_word(word, hyphens) f.write(text + '\n') print('Written raw exceptions to', args.exceptions) print() return 0
def get_heuristic(b, g, r, t, hfunc): # generating a set of patterns for the given parameters # initialising p = Project.load('bds') # input the training set d = p.dictionary.clone() s = Selector(g, b, t) range = Range(1, r) # generating patterns p.train_new_layer(range, s) # trains the pattern false = p.false # number of false positives by the patterns missed = p.missed # number of false negatives by the patterns p.dictionary = d # do we need this? return hfunc(false, missed) # evaluated by given heuristic function
def main_export(args): print('Exporting patterns from', args.project, 'and saving them in TeX format to', args.output) if os.path.exists(args.output): print('Pattern file already exists! Delete it first, or change the name. Pattern file: %s' % args.output) return -1 project = Project.load(args.project) pattern_strings = list(project.patternset.pattern_strings()) exceptions = list(project.patternset.errors(project.dictionary, project.margins)) with codecs.open(args.output, 'w', 'utf-8') as f: f.write('\\patterns{\n') for patt in pattern_strings: f.write(patt + '\n') f.write('}\n') f.write('\\hyphenation{\n') for word, hyphens, _, _ in exceptions: text = format_dictionary_word(word, hyphens) f.write(text + '\n') f.write('}\n') print('Created TeX patterns file', args.output) print('Number of patterns:', len(pattern_strings)) print('Number of exceptions:', len(exceptions)) if args.patterns: print() with codecs.open(args.patterns, 'w', 'utf-8') as f: for patt in pattern_strings: f.write(patt + '\n') print('Written raw patterns to', args.patterns) if args.exceptions: print() with codecs.open(args.exceptions, 'w', 'utf-8') as f: for word, hyphens, _, _ in exceptions: text = format_dictionary_word(word, hyphens) f.write(text + '\n') print('Written raw exceptions to', args.exceptions) print() return 0
def optimise_level_driver(b, g, r, t, hfunc_o, hfunc_e): level_bgr_map = dict() for i in xrange( 1, 10): # number of levels to perform. Start with odd level (1) if i % 2 == 0: # even level b, g, r = optimise_level.optimise_level(b, g, r, t, hfunc_e) else: # odd level b, g, r = optimise_level.optimise_level(b, g, r, t, hfunc_o) print(b, g, r) level_bgr_map[i] = (b, g, r) p = Project.load('bds') s = Selector(g, b, t) range = Range(1, r) l = p.train_new_layer(range, s) p.commit(l) print "Commiting Layer" return level_bgr_map
def main_hyphenate(args): print('Hyphenating', args.input, 'into', args.output, 'using project', args.project) project = Project.load(args.project) with codecs.open(args.input or sys.stdin.fileno(), 'r', 'utf-8') as f: with codecs.open(args.output or sys.stdout.fileno(), 'w', 'utf-8') as out: for word in f: word = word.strip() if not word: continue prediction = project.patternset.hyphenate(word, margins=project.margins) s = format_dictionary_word(word, prediction) out.write(s + '\n') print() return 0
def test(self): dictionary = Dictionary.from_string(''' lo-rem ip-sum do-l-or sit a-met con-sec-te-tur adi-pis-cing elit ves-ti-bu-l-um eu-is-mod di-am eg-et bi-b-en-d-um ''') project = Project(dictionary) rng = Range.parse('1-2') selector = Selector.parse('1:1:1') project.train_new_layer(rng, selector) self.assertEqual(1, len(project.patternset)) self.assertEqual(project.missed, 0) self.assertEqual(project.false, 3) project.train_new_layer(rng, selector) self.assertEqual(2, len(project.patternset)) self.assertEqual(project.missed, 0) self.assertEqual(project.false, 0) patterns = list(project.patternset.pattern_strings()) self.assertEqual(patterns, [ '.e2', '1a', '1a1m', '1b', '1b1e', 'bi1', '1bu1', '1ci', 'co2', 'c1t', 'di1', 'do1', '1d1u', 'ec1', 'eg1', 'e2l', '1en1', 'es1', 'e1t', 'eu1', 'g1', 'g1e', 'i1a', 'i1b', 'is1', '1l', '2li', '1lo1', '1l1u', '1m', '1me', '1mo', '2n1', 'n1d1', '2n1s', 'o1', 'o1l1', 'o2n1', 'o1r', '1pi', 'p1s2', '1r', '1re', 's1c', '1se', 's1m', 's1t', '1s2u', '1t', '1te1', '1ti1', '1tu', 'u1', 'u1i', 'u1l1', '1um' ])
def main_compact(args): print('Compacting hyphenation patterns for', args.project) project = Project.load(args.project) before_compact = [layer.compute_num_patterns() for layer in project.patternset] project.patternset.compact() after_compact = [layer.compute_num_patterns() for layer in project.patternset] print('Result:') for level0, (before, after) in enumerate(zip(before_compact, after_compact)): print('\tLevel %s: %6d => %6d' % (level0+1, before, after)) if args.commit: project.save(args.project) print('...Committed') else: print('...Project NOT changed (use --commit flag to save changes)') print() return 0
def main_hyphenate(args): print('Hyphenating', args.input, 'into', args.output, 'using project', args.project) project = Project.load(args.project) with codecs.open(args.input or sys.stdin.fileno(), 'r', 'utf-8') as f: with codecs.open(args.output or sys.stdout.fileno(), 'w', 'utf-8') as out: for word in f: word = word.strip() if not word: continue prediction = project.patternset.hyphenate( word, margins=project.margins) s = format_dictionary_word(word, prediction) out.write(s + '\n') print() return 0
def main_test(args): print('Testing', args.project, 'on dictionary', args.dictionary) project = Project.load(args.project) dictionary = Dictionary.load(args.dictionary) print('Performance of', args.project, 'on', args.dictionary) do_test(project, dictionary) if args.errors: with codecs.open(args.errors, 'w', 'utf-8') as f: for word, hyphens, missed, false in project.patternset.errors(dictionary, project.margins): f.write(format_dictionary_word(word, hyphens, missed, false) + '\n') print('Saved errors to', args.errors) if args.patterns: with codecs.open(args.patterns, 'w', 'utf-8') as f: for word, hyphens, missed, false in project.patternset.errors(dictionary, project.margins): f.write(format_word_as_pattern(word, missed, false) + '\n') print('Saved errors to', args.patterns) print() return 0
def main_train(args): print('Training project', args.project, 'using range', args.range, 'and selector', args.selector) project = Project.load(args.project) if len(project.patternset) & 1: print('Training INHIBINTING pattern layer (level=%s)' % (len(project.patternset) + 1)) else: print('Training HYPHENATION pattern layer (level=%s)' % (len(project.patternset) + 1)) patlen_rng = Range.parse(args.range) selector = Selector.parse(args.selector) print('\tpattern lengths:', patlen_rng) print('\tselector:', args.selector) total_hyphens = project.total_hyphens layer = project.train_new_layer(patlen_rng, selector) missed, false = project.missed, project.false print('Missed (weighted):', missed, percent(missed, total_hyphens)) print('False (weighted):', false, percent(false, project.total_nonhyphens)) if args.commit: project.commit(layer) project.save(args.project) print('...Committed!') else: print('...Projects NOT changed (use --commit flag to save changes)') print() return 0
def main_explain(args): print('Explaining hyphenation of', args.input, 'into', args.output, 'using project', args.project) project = Project.load(args.project) with codecs.open(args.input or sys.stdin.fileno(), 'r', 'utf-8') as f: with codecs.open(args.output or sys.stdout.fileno(), 'w', 'utf-8') as out: for word in f: word = word.strip() if not word: continue explain = Explain() prediction = project.patternset.hyphenate_explain(word, margins=project.margins, explain=explain) s = format_dictionary_word(word, prediction) out.write(s + '\n') out.write(s.encode('unicode-escape').decode('ascii') + '\n') out.write(explain.format() + '\n\n') print() return 0
from patgen.project import Project from patgen.range import Range from patgen.selector import Selector import csv import pprint p = Project.load('bds') csvw = csv.writer(open('bds.csv', 'w')) for r1 in xrange(3, 4): for g1 in xrange(1, 5): for b1 in xrange(1, 5): rg1 = Range(1, r1) s1 = Selector(g1, b1, 10) d1 = p.dictionary.clone() p.train_new_layer(rg1, s1) for r2 in xrange(3, 4): for g2 in xrange(1, 5): for b2 in xrange(1, 5): rg2 = Range(1, r2) s2 = Selector(g2, b2, 10) d2 = p.dictionary.clone() p.train_new_layer(rg2, s2) num_patterns = sum(l.compute_num_patterns() for l in p.patternset) csvw.writerow((r1, g1, b1, r2, g2, b2, num_patterns, p.missed, p.false)) p.patternset.pop() p.dictionary = d2
def test(self): dictionary = Dictionary.from_string(''' lo-rem ip-sum do-l-or sit a-met con-sec-te-tur adi-pis-cing elit ves-ti-bu-l-um eu-is-mod di-am eg-et bi-b-en-d-um ''') project = Project(dictionary) rng = Range.parse('1-2') selector = Selector.parse('1:1:1') project.train_new_layer(rng, selector) self.assertEqual(1, len(project.patternset)) self.assertEqual(project.missed, 0) self.assertEqual(project.false, 3) project.train_new_layer(rng, selector) self.assertEqual(2, len(project.patternset)) self.assertEqual(project.missed, 0) self.assertEqual(project.false, 0) patterns = list(project.patternset.pattern_strings()) self.assertEqual(patterns, [ '.e2', '1a', '1a1m', '1b', '1b1e', 'bi1', '1bu1', '1ci', 'co2', 'c1t', 'di1', 'do1', '1d1u', 'ec1', 'eg1', 'e2l', '1en1', 'es1', 'e1t', 'eu1', 'g1', 'g1e', 'i1a', 'i1b', 'is1', '1l', '2li', '1lo1', '1l1u', '1m', '1me', '1mo', '2n1', 'n1d1', '2n1s', 'o1', 'o1l1', 'o2n1', 'o1r', '1pi', 'p1s2', '1r', '1re', 's1c', '1se', 's1m', 's1t', '1s2u', '1t', '1te1', '1ti1', '1tu', 'u1', 'u1i', 'u1l1', '1um' ])