def customize_matrix(path, arch_type, destination=None): if os.path.isdir(path): path = os.path.join(path, 'choices') # if no destination dir is specified, just use the choices file's dir destination = destination or os.path.dirname(path) global ch ch = ChoicesFile(path) language = ch['language'] grammar_path = get_grammar_path( ch.get('iso-code', language).lower(), language.lower(), destination) # delete any existing contents at grammar path if os.path.exists(grammar_path): shutil.rmtree(grammar_path) # the rsync command won't create the target dirs, so do it now os.makedirs(grammar_path) # Use the following command when python2.6 is available #shutil.copytree('matrix-core', grammar_path, # ignore=shutil.ignore_patterns('.svn')) IGNORE = open(os.devnull, 'w') try: call([ 'rsync', '-a', '--exclude=.svn', get_matrix_core_path() + os.path.sep, grammar_path ], stdout=IGNORE, stderr=IGNORE) except OSError, er: print "OS Error. Most likely rsync is not installed." print er.message sys.exit(1)
def test_get(self): c = ChoicesFile() # no file loaded c.load_choices(simple_choices_file) self.assertEqual(c.get('NO_SUCH'), '') self.assertEqual(c.get(''), c.choices) self.assertEqual(c.get('language'), 'Simple') self.assertEqual(c.get('iso-code'), 'smp') self.assertEqual(c.get('verb1_name'), 'testverb') self.assertEqual(c.get('verb2_name'), 'testverb2') self.assertEqual(c.get('verb1_stem1_orth'), 'test') self.assertEqual(c.get('verb1_stem1_pred'), 'test_v_1_rel') self.assertEqual(c.get('verb2_stem1_orth'), 'test') self.assertEqual(c.get('verb2_stem1_pred'), 'test_v_2_rel') self.assertEqual(c.get('verb2'), {'name':'testverb2', 'valence':'trans', 'stem':[{'orth':'test','pred':'test_v_2_rel'}]}) self.assertEqual(c.get('verb3'), {})
def add(choices_file, txt_suite): cust_root = os.environ.get("CUSTOMIZATIONROOT") # Check whether arguments correspond to existing files rt_root = os.path.join(cust_root, 'regression_tests') # CMC 2017-04-08: use expansion to handle relative paths (can't deal with slashes) choices_file = os.path.join(rt_root, 'scratch', *choices_file.split('/')) txt_suite = os.path.join(rt_root, 'scratch', *txt_suite.split('/')) if not os.path.exists(cust_root): raise ValueError, "Invalid path name for customization root." if not os.path.exists(choices_file): raise ValueError, "Invalid path name for choices file." if not os.path.exists(txt_suite): raise ValueError, "Invalid path name for txt-suite file." # CMC 2017-04-08: don't do path joins by hand if not os.path.exists(os.path.join(rt_root, "regression-test-index")): raise ValueError, "No file regression_tests/regression-test-index in customization root. Invalid customization root or missing file." # Get the language name ch = ChoicesFile(choices_file) lg_name = ch.get('language') # Make sure illegal characters (for lisp strings) are not in the lg_name. # For now this is just double-quotes " if any(c in ('"', ) for c in lg_name): raise ValueError, 'Double-quotes are not allowed in the language name.' # Load up the current regression test index index = RegressionTestIndex(os.path.join(rt_root, "regression-test-index")) # Check whether we already have a regression-test with that # name, and if not, if there are any files in the way anyway. if index.exists(lg_name): # TJT 2014-09-11: Updating this error to suggest using matrix.py to remove test #raise ValueError, "A regression test with that language name already exists. If you must remove it, be sure to edit regression-test-index." raise ValueError, "A regression test with that language name already exists. If you need to remove the existing test, use matrix.py rr TEST" if os.path.exists(os.path.join(rt_root, "choices", lg_name)): raise ValueError, "Move regression_tests/choices/" + lg_name + ", it is in the way." if os.path.exists(os.path.join(rt_root, "txt-suites", lg_name)): raise ValueError, "Move regression_tests/txt-suites/" + lg_name + ", it is in the way." if os.path.exists(os.path.join(rt_root, "home", "gold", lg_name)): raise ValueError, "Move regression_tests/home/gold/" + lg_name + ", it is in the way." # CMC 2018-04-09: this path was wrong (skeletons dir is rt_root, not rt_root/home) if os.path.exists(os.path.join(rt_root, "skeletons", lg_name)): raise ValueError, "Move regression_tests/skeletons/" + lg_name + ", it is in the way. You might also need to edit regression_tests/skeletons/Index.lisp." # Prompt user for comment on test. comment = raw_input( "Enter a short comment describing this regression test: ") # Make sure illegal characters (for lisp strings) are not in the comment. # For now this is just double-quotes " if any(c in ('"', ) for c in comment): raise ValueError, 'Double-quotes are not allowed in the comment.' # Make the profile cmd = os.path.join(os.environ['CUSTOMIZATIONROOT'], 'regression_tests', 'add_regression_test.sh') retval = subprocess.call([cmd, choices_file, txt_suite, lg_name], env=os.environ) #if retval != 0: # print "Error creating regression test... (possible missing gold results?)" # Add line to regression-test-index index_file = open(os.path.join(rt_root, "regression-test-index"), 'a') index_file.write(lg_name + "=" + comment + "\n") index_file.close() # Copy choices file, txt-suite and profile to the appropriate places. shutil.copy(choices_file, os.path.join(rt_root, "choices", lg_name)) shutil.copy(txt_suite, os.path.join(rt_root, "txt-suites", lg_name)) # Create skeleton os.mkdir(os.path.join(rt_root, "skeletons", lg_name)) shutil.copy(os.path.join(rt_root, "home", "gold", lg_name, "item"), os.path.join(rt_root, "skeletons", lg_name)) shutil.copy(os.path.join(rt_root, "skeletons", "Relations"), os.path.join(rt_root, "skeletons", lg_name, "relations")) # Update Index.lisp f = open(os.path.join(rt_root, "skeletons", "Index.lisp"), 'r') lines = f.readlines() f.close f = open(os.path.join(rt_root, "skeletons", "Index.lisp"), 'w') for l in lines: f.write(l) if (re.search("new-regression-test-here", l)): f.write("((:path . \"") f.write(lg_name) f.write("\") (:content . \"") f.write(lg_name + ": " + comment) f.write("\"))\n") f.close() return lg_name
def customize_matrix(path, arch_type, destination=None, force_dest=False): if os.path.isdir(path): path = os.path.join(path, 'choices') # if no destination dir is specified, just use the choices file's dir destination = destination or os.path.dirname(path) global ch ch = ChoicesFile(path) language = ch['language'] if force_dest: grammar_path = destination else: grammar_path = get_grammar_path( ch.get('iso-code', language).lower(), language.lower(), destination) # delete any existing contents at grammar path if os.path.exists(grammar_path): shutil.rmtree(grammar_path) # the rsync command won't create the target dirs, so do it now os.makedirs(grammar_path) # Use the following command when python2.6 is available # shutil.copytree('matrix-core', grammar_path, # ignore=shutil.ignore_patterns('.svn')) with open(os.devnull, 'w') as IGNORE: try: call([ 'rsync', '-a', '--exclude=.svn', get_matrix_core_path() + os.path.sep, grammar_path ], stdout=IGNORE, stderr=IGNORE) except OSError as er: print("OS Error. Most likely rsync is not installed.") print(er.message) sys.exit(1) # include a copy of choices (named 'choices' to avoid collisions) shutil.copy(path, os.path.join(grammar_path, 'choices')) # Create TDL object for each output file global mylang, rules, irules, lrules, lexicon, roots mylang = tdl.TDLfile(os.path.join(grammar_path, language.lower() + '.tdl')) mylang.define_sections([['addenda', 'Matrix Type Addenda', True, False], ['features', 'Features', True, False], ['dirinv', 'Direct-Inverse', True, False], ['lextypes', 'Lexical Types', True, True], ['nounlex', 'Nouns', False, False], ['verblex', 'Verbs', False, False], ['auxlex', 'Auxiliaries', False, False], ['coplex', 'Copulas', False, False], ['adjlex', 'Adjectives', False, False], ['subordlex', 'Subordinators', True, False], ['complex', 'Complementizers', False, True], ['otherlex', 'Others', False, False], ['lexrules', 'Lexical Rules', True, False], ['phrases', 'Phrasal Types', True, False], ['coord', 'Coordination', True, False]]) rules = tdl.TDLfile(os.path.join(grammar_path, 'rules.tdl')) irules = tdl.TDLfile(os.path.join(grammar_path, 'irules.tdl')) lrules = tdl.TDLfile(os.path.join(grammar_path, 'lrules.tdl')) lexicon = tdl.TDLfile(os.path.join(grammar_path, 'lexicon.tdl')) roots = tdl.TDLfile(os.path.join(grammar_path, 'roots.tdl')) trigger = tdl.TDLfile(os.path.join(grammar_path, 'trigger.mtr')) trigger.add_literal(';;; Semantically Empty Lexical Entries') vpm = tdl.TDLfile(os.path.join(grammar_path, 'semi.vpm')) # date/time try: with open('datestamp', 'r') as f: matrix_dt = f.readlines()[0].strip() except: matrix_dt = 'unknown time' current_dt = datetime.datetime.utcnow() tdl_dt = current_dt.strftime('%a %b %d %H:%M:%S UTC %Y') lisp_dt = current_dt.strftime('%Y-%m-%d_%H:%M:%S_UTC') # Put the current date/time in my_language.tdl... mylang.add_literal( ';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n' + ';;; Grammar of ' + ch.get('language') + '\n' + ';;; created at:\n' + ';;; ' + tdl_dt + '\n' + ';;; based on Matrix customization system version of:\n' + ';;; ' + matrix_dt + '\n' + ';;;\n' + format_comment_block(ch.get('comment')) + '\n' + ';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;') # Put the date/time of the Matrix version in Version.lsp (along # with the name of the language). global version_lsp version_lsp = tdl.TDLfile(os.path.join(grammar_path, 'Version.lsp')) version_lsp.add_literal('(in-package :common-lisp-user)\n\n' + '(defparameter *grammar-version* \"' + ch.get('language') + ' (' + lisp_dt + ')\")') # Initialize various type hierarchies case.init_case_hierarchy(ch, hierarchies) agreement_features.init_agreement_hierarchies(ch, mylang, hierarchies) verbal_features.init_verbal_hierarchies(ch, hierarchies) # Integrate choices related to lexical entries imported from # Toolbox lexicon file(s), if any. NOTE: This needs to be called # before anything else that looks at the lexicon-related choices, # so before lexical_items.insert_ids(). toolboximport.integrate_imported_entries(ch) # Create unique ids for each lexical entry; this allows # us to do the same merging on the lexicon TDL file as we # do on the other TDL files. NOTE: This needs to be called # before customize_lexicon() and customize_inflection() lexical_items.insert_ids(ch) # Currently, yes-no questions need to go before the lexicon # because the lexicon needs to know whether there are obligatory question particles, # to determine the head on the question-embedding verb. yes_no_questions.customize_yesno_questions(mylang, ch, rules, lrules, hierarchies, roots) # The following might modify hierarchies in some way, so it's best # to customize those components and only have them contribute their # information to lexical rules when we customize inflection. lexical_items.customize_lexicon(mylang, ch, lexicon, trigger, hierarchies, rules) information_structure.customize_information_structure( mylang, ch, rules, irules, lexicon, trigger, hierarchies) argument_optionality.customize_arg_op(mylang, ch, rules, hierarchies) direct_inverse.customize_direct_inverse(ch, mylang, hierarchies) case.customize_case(mylang, ch, hierarchies) # after all structures have been customized, customize inflection, # but provide the methods the components above have for their own # contributions to the lexical rules nominalized_clauses.customize_nmcs(mylang, ch, rules, roots) negation.customize_sentential_negation(mylang, ch, lexicon, rules, lrules, hierarchies) # save the hierarchies to the choices object ch['hierarchies'] = hierarchies add_lexrules_methods = [ case.add_lexrules, argument_optionality.add_lexrules, valence_change.add_lexrules, direct_inverse.add_lexrules, wh_ques.add_lexrules ] to_cfv = morphotactics.customize_inflection(ch, add_lexrules_methods, mylang, irules, lrules, lexicon) # customize_feature_values is called by process_cfv_list # negation.py needs to run first! features.process_cfv_list(mylang, ch, hierarchies, to_cfv) # Call the other customization functions agreement_features.customize_agreement_features(mylang, hierarchies) adnominal_possession.customize_adnominal_possession( mylang, ch, rules, irules, lexicon, hierarchies) # LTX 2022-04-22: It is possible to generate disjunctive case from # adnom_poss library (see issue issue #621). # Therefore, it should save case hierarchy to mylang.tdl after # adnominal possession customization. case.customize_case_type(mylang, hierarchies) verbal_features.customize_verbal_features(mylang, hierarchies) valence_change.customize_valence_change(mylang, ch, lexicon, rules, lrules, hierarchies) word_order.customize_word_order(mylang, ch, rules) coordination.customize_coordination(mylang, ch, lexicon, rules, irules) clausalmods.customize_clausalmods(mylang, ch, lexicon, rules, roots, trigger) clausalcomps.customize_clausalcomps(mylang, ch, lexicon, rules) adverbs_adpositions.customize_adv_adp(ch, mylang, rules) wh_ques.customize_wh_ques(mylang, ch, rules, roots) # Customization having to do with punctuation, [incr tsdb()], # parsers, roots, and vpm. customize_punctuation(grammar_path) customize_test_sentences(grammar_path) customize_itsdb(grammar_path) customize_script(grammar_path) customize_pettdl(grammar_path) customize_acetdl(grammar_path) customize_roots() customize_vpm(ch, vpm, hierarchies) # Save the output files mylang.save() rules.save() irules.save() lrules.save() lexicon.save() roots.save() trigger.save() vpm.save() version_lsp.save() # Setup version control, if any setup_vcs(ch, grammar_path) return grammar_path