def main():
args = _parse_arguments()
with open(args.source) as json_file:
grammar_data = json.load(json_file)
try:
grammar = Grammar.get_grammar(grammar_data)
if grammar.grammar_type != GrammarType.CONTEXT_FREE:
raise ValueError("Grammar should be context free.")
with FileLogger("pushdown_automaton_trace.txt") as logger:
pd_automaton = PushdownAutomaton.from_grammar(grammar)
pd_automaton.set_logger(logger)
pd_automaton.set_input_sequence("n*n*n+n")
print(pd_automaton.start())
pd_automaton.set_input_sequence("n*n")
print(pd_automaton.start())
pd_automaton.set_input_sequence("h+h+h+h*n")
print(pd_automaton.start())
with FileLogger("pushdown_automaton_ext_trace.txt") as logger:
epd_automaton = ExtendedPushdownAutomaton.from_grammar(grammar)
epd_automaton.set_logger(logger)
epd_automaton.set_input_sequence("n*n*n+n")
print(epd_automaton.start())
epd_automaton.set_input_sequence("n*n")
print(epd_automaton.start())
epd_automaton.set_input_sequence("h+h+h+h*n")
print(epd_automaton.start())
except ValueError as error:
print(error)
def main():
args = _parse_arguments()
with open(args.source) as json_file:
grammar_data = json.load(json_file)
try:
grammar = Grammar.get_grammar(grammar_data)
print(grammar.grammar_type.value)
except ValueError as error:
print(error)
def load(cls, path):
params = json.load(open(path, 'r'))
entries = dict()
for key, val in params.items():
if key in ('grammar', ):
entry = Grammar.load(val)
else:
entry = VocabEntry.load(params=val)
entries[key] = entry
return cls(**entries)
print('train subtoken model for obj names')
# train subtoken models
spm.SentencePieceTrainer.Train(
f'--add_dummy_prefix=false --pad_id={PAD_ID} --bos_id=1 --eos_id=2 --unk_id=3 '
f'--control_symbols=<IDENTITY> --vocab_size={vocab_size} '
f'--model_prefix={vocab_file}.obj_name --model_type=bpe '
f'--input={id_names_file}')
obj_name_vocab_entry = VocabEntry(vocab_file + '.obj_name.model')
type_vocab = Counter(type_tokens)
num_types = 100
var_types = []
for type_token, freq in type_vocab.items():
if freq > 100:
print(type_token, freq)
var_types.append(type_token)
print('init node types and variable types')
grammar = Grammar(node_types, var_types)
print('Node types:', node_types)
print('Variable types:', var_types)
vocab = Vocab(source=src_var_vocab_entry,
source_tokens=src_code_tokens_vocab_entry,
target=tgt_var_vocab_entry,
obj_name=obj_name_vocab_entry,
grammar=grammar)
vocab.save(args['VOCAB_FILE'])
#!/usr/bin/env python3
import unittest
import utils.shiftreduce as sr
from utils.grammar import Grammar
GRAMMAR = Grammar("grammar/grammar_test.txt")
class TestShiftReduce(unittest.TestCase):
def test_shift_reduce_001(self):
entry = ''.split()
exepted_output = []
self.assertListEqual(sr.shift_reduce(entry, GRAMMAR), exepted_output)
def test_shift_reduce_002(self):
entry = 'A'.split()
exepted_output = ['A']
self.assertListEqual(sr.shift_reduce(entry, GRAMMAR), exepted_output)
def test_shift_reduce_003(self):
entry = 'A B'.split()
exepted_output = ['A']
self.assertListEqual(sr.shift_reduce(entry, GRAMMAR), exepted_output)
def test_shift_reduce_004(self):
entry = 'A C C C'.split()
exepted_output = ['A']
self.assertListEqual(sr.shift_reduce(entry, GRAMMAR), exepted_output)
def test_shift_reduce_005(self):
entry = 'A B C C C'.split()