def main(): literals = [ "Title:", "Time:", "Tempo:", "Composer:", "Arranged by:", "Key:", "Year:", "{", "}", "(", ")", "seq", "play", "chord", "=", "[", "|T|", "||", "]", ",", "\"", "//", "$(", ")", "(", "-" ] # Tokenizing print("Starting TOKENIZING") program = open_file() Tokenizer.makeTokenizer(program, literals) p = COMPOSITION() print("Completed TOKENIZING") # Parsing print("Starting PARSING") p.parse() print("Completed PARSING") print("Starting EVALUATION") input = Input() evaluator = Evaluator(input) p.accept(evaluator) print("Completed EVALUATION") print("======Build Input=====") print(repr(input)) print("======Build Output=====") createOutputs(input)
def main(): literals = [ "arena size", "make a", "called", "destroy", "set", "of", "to", "change", "by", "move", "wait", "do every", ":", "end loop", "ms", "block", "goal", "put player", "up", "down", "left", "right", "colour", "xpos", "ypos", "width", "height", ",", "at", "with size" ] Tokenizer.make_tokenizer("config.txt", literals) program = PROGRAM() program.parse() generate_game(program) """
def test_classify(self): db = {} Classifier.train(db, "Ruby", self.fixture("Ruby/foo.rb")) Classifier.train(db, "Objective-C", self.fixture("Objective-C/Foo.h")) Classifier.train(db, "Objective-C", self.fixture("Objective-C/Foo.m")) rs = Classifier.classify(db, self.fixture("Objective-C/hello.m")) assert "Objective-C" == rs[0][0] tokens = Tokenizer.tokenize(self.fixture("Objective-C/hello.m")) rs = Classifier.classify(db, tokens) assert "Objective-C" == rs[0][0]
def tokenize(self, data='', is_path=None): if is_path: data = open(join(join(ROOT_DIR, "samples"), str(data))).read() return Tokenizer.tokenize(data)
def __init__(self): self.tokenizer = Tokenizer.get_tokenizer()
def __init__(self): print("init node for " + str(type(self))) self.tokenizer = Tokenizer.getTokenizer()