def customize_matrix(path, arch_type, destination=None):
if os.path.isdir(path):
path = os.path.join(path, 'choices')
# if no destination dir is specified, just use the choices file's dir
destination = destination or os.path.dirname(path)
global ch
ch = ChoicesFile(path)
language = ch['language']
grammar_path = get_grammar_path(
ch.get('iso-code', language).lower(), language.lower(), destination)
# delete any existing contents at grammar path
if os.path.exists(grammar_path):
shutil.rmtree(grammar_path)
# the rsync command won't create the target dirs, so do it now
os.makedirs(grammar_path)
# Use the following command when python2.6 is available
#shutil.copytree('matrix-core', grammar_path,
# ignore=shutil.ignore_patterns('.svn'))
IGNORE = open(os.devnull, 'w')
try:
call([
'rsync', '-a', '--exclude=.svn',
get_matrix_core_path() + os.path.sep, grammar_path
],
stdout=IGNORE,
stderr=IGNORE)
except OSError, er:
print "OS Error. Most likely rsync is not installed."
print er.message
sys.exit(1)
def test_get(self):
c = ChoicesFile() # no file loaded
c.load_choices(simple_choices_file)
self.assertEqual(c.get('NO_SUCH'), '')
self.assertEqual(c.get(''), c.choices)
self.assertEqual(c.get('language'), 'Simple')
self.assertEqual(c.get('iso-code'), 'smp')
self.assertEqual(c.get('verb1_name'), 'testverb')
self.assertEqual(c.get('verb2_name'), 'testverb2')
self.assertEqual(c.get('verb1_stem1_orth'), 'test')
self.assertEqual(c.get('verb1_stem1_pred'), 'test_v_1_rel')
self.assertEqual(c.get('verb2_stem1_orth'), 'test')
self.assertEqual(c.get('verb2_stem1_pred'), 'test_v_2_rel')
self.assertEqual(c.get('verb2'),
{'name':'testverb2', 'valence':'trans',
'stem':[{'orth':'test','pred':'test_v_2_rel'}]})
self.assertEqual(c.get('verb3'), {})
def add(choices_file, txt_suite):
cust_root = os.environ.get("CUSTOMIZATIONROOT")
# Check whether arguments correspond to existing files
rt_root = os.path.join(cust_root, 'regression_tests')
# CMC 2017-04-08: use expansion to handle relative paths (can't deal with slashes)
choices_file = os.path.join(rt_root, 'scratch', *choices_file.split('/'))
txt_suite = os.path.join(rt_root, 'scratch', *txt_suite.split('/'))
if not os.path.exists(cust_root):
raise ValueError, "Invalid path name for customization root."
if not os.path.exists(choices_file):
raise ValueError, "Invalid path name for choices file."
if not os.path.exists(txt_suite):
raise ValueError, "Invalid path name for txt-suite file."
# CMC 2017-04-08: don't do path joins by hand
if not os.path.exists(os.path.join(rt_root, "regression-test-index")):
raise ValueError, "No file regression_tests/regression-test-index in customization root. Invalid customization root or missing file."
# Get the language name
ch = ChoicesFile(choices_file)
lg_name = ch.get('language')
# Make sure illegal characters (for lisp strings) are not in the lg_name.
# For now this is just double-quotes "
if any(c in ('"', ) for c in lg_name):
raise ValueError, 'Double-quotes are not allowed in the language name.'
# Load up the current regression test index
index = RegressionTestIndex(os.path.join(rt_root, "regression-test-index"))
# Check whether we already have a regression-test with that
# name, and if not, if there are any files in the way anyway.
if index.exists(lg_name):
# TJT 2014-09-11: Updating this error to suggest using matrix.py to remove test
#raise ValueError, "A regression test with that language name already exists. If you must remove it, be sure to edit regression-test-index."
raise ValueError, "A regression test with that language name already exists. If you need to remove the existing test, use matrix.py rr TEST"
if os.path.exists(os.path.join(rt_root, "choices", lg_name)):
raise ValueError, "Move regression_tests/choices/" + lg_name + ", it is in the way."
if os.path.exists(os.path.join(rt_root, "txt-suites", lg_name)):
raise ValueError, "Move regression_tests/txt-suites/" + lg_name + ", it is in the way."
if os.path.exists(os.path.join(rt_root, "home", "gold", lg_name)):
raise ValueError, "Move regression_tests/home/gold/" + lg_name + ", it is in the way."
# CMC 2018-04-09: this path was wrong (skeletons dir is rt_root, not rt_root/home)
if os.path.exists(os.path.join(rt_root, "skeletons", lg_name)):
raise ValueError, "Move regression_tests/skeletons/" + lg_name + ", it is in the way. You might also need to edit regression_tests/skeletons/Index.lisp."
# Prompt user for comment on test.
comment = raw_input(
"Enter a short comment describing this regression test: ")
# Make sure illegal characters (for lisp strings) are not in the comment.
# For now this is just double-quotes "
if any(c in ('"', ) for c in comment):
raise ValueError, 'Double-quotes are not allowed in the comment.'
# Make the profile
cmd = os.path.join(os.environ['CUSTOMIZATIONROOT'], 'regression_tests',
'add_regression_test.sh')
retval = subprocess.call([cmd, choices_file, txt_suite, lg_name],
env=os.environ)
#if retval != 0:
# print "Error creating regression test... (possible missing gold results?)"
# Add line to regression-test-index
index_file = open(os.path.join(rt_root, "regression-test-index"), 'a')
index_file.write(lg_name + "=" + comment + "\n")
index_file.close()
# Copy choices file, txt-suite and profile to the appropriate places.
shutil.copy(choices_file, os.path.join(rt_root, "choices", lg_name))
shutil.copy(txt_suite, os.path.join(rt_root, "txt-suites", lg_name))
# Create skeleton
os.mkdir(os.path.join(rt_root, "skeletons", lg_name))
shutil.copy(os.path.join(rt_root, "home", "gold", lg_name, "item"),
os.path.join(rt_root, "skeletons", lg_name))
shutil.copy(os.path.join(rt_root, "skeletons", "Relations"),
os.path.join(rt_root, "skeletons", lg_name, "relations"))
# Update Index.lisp
f = open(os.path.join(rt_root, "skeletons", "Index.lisp"), 'r')
lines = f.readlines()
f.close
f = open(os.path.join(rt_root, "skeletons", "Index.lisp"), 'w')
for l in lines:
f.write(l)
if (re.search("new-regression-test-here", l)):
f.write("((:path . \"")
f.write(lg_name)
f.write("\") (:content . \"")
f.write(lg_name + ": " + comment)
f.write("\"))\n")
f.close()
return lg_name
def customize_matrix(path, arch_type, destination=None, force_dest=False):
if os.path.isdir(path):
path = os.path.join(path, 'choices')
# if no destination dir is specified, just use the choices file's dir
destination = destination or os.path.dirname(path)
global ch
ch = ChoicesFile(path)
language = ch['language']
if force_dest:
grammar_path = destination
else:
grammar_path = get_grammar_path(
ch.get('iso-code', language).lower(), language.lower(),
destination)
# delete any existing contents at grammar path
if os.path.exists(grammar_path):
shutil.rmtree(grammar_path)
# the rsync command won't create the target dirs, so do it now
os.makedirs(grammar_path)
# Use the following command when python2.6 is available
# shutil.copytree('matrix-core', grammar_path,
# ignore=shutil.ignore_patterns('.svn'))
with open(os.devnull, 'w') as IGNORE:
try:
call([
'rsync', '-a', '--exclude=.svn',
get_matrix_core_path() + os.path.sep, grammar_path
],
stdout=IGNORE,
stderr=IGNORE)
except OSError as er:
print("OS Error. Most likely rsync is not installed.")
print(er.message)
sys.exit(1)
# include a copy of choices (named 'choices' to avoid collisions)
shutil.copy(path, os.path.join(grammar_path, 'choices'))
# Create TDL object for each output file
global mylang, rules, irules, lrules, lexicon, roots
mylang = tdl.TDLfile(os.path.join(grammar_path, language.lower() + '.tdl'))
mylang.define_sections([['addenda', 'Matrix Type Addenda', True, False],
['features', 'Features', True, False],
['dirinv', 'Direct-Inverse', True, False],
['lextypes', 'Lexical Types', True, True],
['nounlex', 'Nouns', False, False],
['verblex', 'Verbs', False, False],
['auxlex', 'Auxiliaries', False, False],
['coplex', 'Copulas', False, False],
['adjlex', 'Adjectives', False, False],
['subordlex', 'Subordinators', True, False],
['complex', 'Complementizers', False, True],
['otherlex', 'Others', False, False],
['lexrules', 'Lexical Rules', True, False],
['phrases', 'Phrasal Types', True, False],
['coord', 'Coordination', True, False]])
rules = tdl.TDLfile(os.path.join(grammar_path, 'rules.tdl'))
irules = tdl.TDLfile(os.path.join(grammar_path, 'irules.tdl'))
lrules = tdl.TDLfile(os.path.join(grammar_path, 'lrules.tdl'))
lexicon = tdl.TDLfile(os.path.join(grammar_path, 'lexicon.tdl'))
roots = tdl.TDLfile(os.path.join(grammar_path, 'roots.tdl'))
trigger = tdl.TDLfile(os.path.join(grammar_path, 'trigger.mtr'))
trigger.add_literal(';;; Semantically Empty Lexical Entries')
vpm = tdl.TDLfile(os.path.join(grammar_path, 'semi.vpm'))
# date/time
try:
with open('datestamp', 'r') as f:
matrix_dt = f.readlines()[0].strip()
except:
matrix_dt = 'unknown time'
current_dt = datetime.datetime.utcnow()
tdl_dt = current_dt.strftime('%a %b %d %H:%M:%S UTC %Y')
lisp_dt = current_dt.strftime('%Y-%m-%d_%H:%M:%S_UTC')
# Put the current date/time in my_language.tdl...
mylang.add_literal(
';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n' +
';;; Grammar of ' + ch.get('language') + '\n' + ';;; created at:\n' +
';;; ' + tdl_dt + '\n' +
';;; based on Matrix customization system version of:\n' + ';;; ' +
matrix_dt + '\n' + ';;;\n' + format_comment_block(ch.get('comment')) +
'\n' + ';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;')
# Put the date/time of the Matrix version in Version.lsp (along
# with the name of the language).
global version_lsp
version_lsp = tdl.TDLfile(os.path.join(grammar_path, 'Version.lsp'))
version_lsp.add_literal('(in-package :common-lisp-user)\n\n' +
'(defparameter *grammar-version* \"' +
ch.get('language') + ' (' + lisp_dt + ')\")')
# Initialize various type hierarchies
case.init_case_hierarchy(ch, hierarchies)
agreement_features.init_agreement_hierarchies(ch, mylang, hierarchies)
verbal_features.init_verbal_hierarchies(ch, hierarchies)
# Integrate choices related to lexical entries imported from
# Toolbox lexicon file(s), if any. NOTE: This needs to be called
# before anything else that looks at the lexicon-related choices,
# so before lexical_items.insert_ids().
toolboximport.integrate_imported_entries(ch)
# Create unique ids for each lexical entry; this allows
# us to do the same merging on the lexicon TDL file as we
# do on the other TDL files. NOTE: This needs to be called
# before customize_lexicon() and customize_inflection()
lexical_items.insert_ids(ch)
# Currently, yes-no questions need to go before the lexicon
# because the lexicon needs to know whether there are obligatory question particles,
# to determine the head on the question-embedding verb.
yes_no_questions.customize_yesno_questions(mylang, ch, rules, lrules,
hierarchies, roots)
# The following might modify hierarchies in some way, so it's best
# to customize those components and only have them contribute their
# information to lexical rules when we customize inflection.
lexical_items.customize_lexicon(mylang, ch, lexicon, trigger, hierarchies,
rules)
information_structure.customize_information_structure(
mylang, ch, rules, irules, lexicon, trigger, hierarchies)
argument_optionality.customize_arg_op(mylang, ch, rules, hierarchies)
direct_inverse.customize_direct_inverse(ch, mylang, hierarchies)
case.customize_case(mylang, ch, hierarchies)
# after all structures have been customized, customize inflection,
# but provide the methods the components above have for their own
# contributions to the lexical rules
nominalized_clauses.customize_nmcs(mylang, ch, rules, roots)
negation.customize_sentential_negation(mylang, ch, lexicon, rules, lrules,
hierarchies)
# save the hierarchies to the choices object
ch['hierarchies'] = hierarchies
add_lexrules_methods = [
case.add_lexrules, argument_optionality.add_lexrules,
valence_change.add_lexrules, direct_inverse.add_lexrules,
wh_ques.add_lexrules
]
to_cfv = morphotactics.customize_inflection(ch, add_lexrules_methods,
mylang, irules, lrules,
lexicon)
# customize_feature_values is called by process_cfv_list
# negation.py needs to run first!
features.process_cfv_list(mylang, ch, hierarchies, to_cfv)
# Call the other customization functions
agreement_features.customize_agreement_features(mylang, hierarchies)
adnominal_possession.customize_adnominal_possession(
mylang, ch, rules, irules, lexicon, hierarchies)
# LTX 2022-04-22: It is possible to generate disjunctive case from
# adnom_poss library (see issue issue #621).
# Therefore, it should save case hierarchy to mylang.tdl after
# adnominal possession customization.
case.customize_case_type(mylang, hierarchies)
verbal_features.customize_verbal_features(mylang, hierarchies)
valence_change.customize_valence_change(mylang, ch, lexicon, rules, lrules,
hierarchies)
word_order.customize_word_order(mylang, ch, rules)
coordination.customize_coordination(mylang, ch, lexicon, rules, irules)
clausalmods.customize_clausalmods(mylang, ch, lexicon, rules, roots,
trigger)
clausalcomps.customize_clausalcomps(mylang, ch, lexicon, rules)
adverbs_adpositions.customize_adv_adp(ch, mylang, rules)
wh_ques.customize_wh_ques(mylang, ch, rules, roots)
# Customization having to do with punctuation, [incr tsdb()],
# parsers, roots, and vpm.
customize_punctuation(grammar_path)
customize_test_sentences(grammar_path)
customize_itsdb(grammar_path)
customize_script(grammar_path)
customize_pettdl(grammar_path)
customize_acetdl(grammar_path)
customize_roots()
customize_vpm(ch, vpm, hierarchies)
# Save the output files
mylang.save()
rules.save()
irules.save()
lrules.save()
lexicon.save()
roots.save()
trigger.save()
vpm.save()
version_lsp.save()
# Setup version control, if any
setup_vcs(ch, grammar_path)
return grammar_path