def _expression(self): tree = ParseTree("Expr") node = self._log_expr() if node is not None: tree.add_child(node) return tree return None
def create_parse_trees(self, chart, start=None): if start is None: start = list(chart[len(chart) - 1][0])[0] if len(start.pointers) == 0: return [ParseTree(start.data)] parse_trees = [] for l, r in start.pointers: left_tree = self.create_parse_trees(chart, l) right_tree = self.create_parse_trees(chart, r) for x in left_tree: for y in right_tree: parse_trees.append(ParseTree(start.data, x, y)) return parse_trees
def parse(self, sentence): """Read a sentence (iterable of terminals), and: - if it's in the language, return its parse tree - otherwise, raise NotInLanguage [TRDB] Algorithm 4.3 p. 187""" # use a mixed stack with: # - symbols corresponding to the productions in progress # - tree nodes to go back to when their children are complete stack = [self.g.END, self.g.start_symbol] cur_node = None tok_stream = chain(iter(sentence), (self.g.END, )) token = next(tok_stream) while stack: state = stack.pop() if isinstance(state, ParseTree): cur_node = state elif state in self.g.non_terminals: if (state, token) not in self.table: msg = "In state '{}', got '{}'".format(state, token) raise self.NotInLanguage(msg) new_node = ParseTree(state) if cur_node: cur_node.children.append(new_node) stack.append(cur_node) cur_node = new_node prod_idx = self.table[state, token] rhs = self.g.productions[prod_idx][1] stack.extend(list(reversed(rhs))) if len(rhs) == 0: cur_node.children.append(ParseTree('')) else: if token != state: msg = "Expected '{}', got '{}'".format(state, token) raise self.NotInLanguage(msg) if state != self.g.END: cur_node.children.append(ParseTree(token)) token = next(tok_stream) return cur_node
def _identifier(self): tree = ParseTree("identifier") count = 0 while True: if self._out_of_range(): break if self.string[self.index] in self.identifiers: self.index += 1 count += 1 else: break if count: tree.add_child(ParseTree(self.string[self.index-count: self.index])) return tree return None
def _log_value(self): tree = ParseTree("log value") if self.string[self.index: self.index + 4] == "true": self.index += 4 tree.add_child(ParseTree("true")) return tree if self.string[self.index: self.index + 5] == "false": self.index += 5 tree.add_child(ParseTree("false")) return tree return None
def _log_expr_(self): if self._out_of_range(): return None tree = ParseTree("log expr'") if self.string[self.index] == "!": self.index += 1 tree.add_child(ParseTree("!")) log_mon_node = self._log_mon() if log_mon_node: tree.add_child(log_mon_node) log_expr_node = self._log_expr_() if log_expr_node: tree.add_child(log_expr_node) return tree return None
def parse(self, sentence): """Read a sentence (iterable of terminals), and: - if it's in the language, do nothing (for now), - otherwise, raise NotInLanguage [TRDB] Algorithm Fig 4.30 p. 219""" # store pairs on the stack instead of two values stack = [(0, None)] tok_stream = chain(iter(sentence), (self.g.END,)) token = next(tok_stream) while True: state = stack[-1][0] try: action, info = self.actions[state, token] except KeyError: msg = "In state '{}', got '{}'".format(state, token) raise self.NotInLanguage(msg) if action == self.SHIFT: node = ParseTree(token) stack.append((info, node)) token = next(tok_stream) elif action == self.REDUCE: lhs, rhs = self.g.productions[info] children = [stack.pop()[1] for _ in range(len(rhs))] children.reverse() node = ParseTree(lhs, children) prev_state = stack[-1][0] new_state = self.gotos[prev_state, lhs] stack.append((new_state, node)) else: # action == self.ACCEPT: return stack[-1][1]
def _sec_expr(self): tree = ParseTree("sec_expr") first_exp_node = self._first_expr() if first_exp_node: tree.add_child(first_exp_node) return tree if self._out_of_range(): return None if self.string[self.index] == "~": self.index += 1 tree.add_child(ParseTree("~")) first_exp_node = self._first_expr() if first_exp_node: tree.add_child(first_exp_node) return tree return None
def _log_mon(self): tree = ParseTree("log mon") sec_expr_node = self._sec_expr() if sec_expr_node: tree.add_child(sec_expr_node) log_mon_node = self._log_mon_() if log_mon_node is not None: tree.add_child(log_mon_node) if tree.childs: return tree return None
def _log_expr(self): tree = ParseTree("log expr") log_node = self._log_mon() if log_node is not None: tree.add_child(log_node) log_expr_node = self._log_expr_() if log_expr_node is not None: tree.add_child(log_expr_node) if tree.childs: return tree return None
def _first_expr(self): tree = ParseTree("first_expr") log_value_node = self._log_value() if log_value_node: tree.add_child(log_value_node) return tree identifier_node = self._identifier() if identifier_node: tree.add_child(identifier_node) return tree return None
def evaluate(self, string): max_length = 10 max_depth = 6 session_parse_tree = tf.Session() parse_tree = ParseTree(session_parse_tree, max_length, max_depth) session_parse_tree.run(tf.global_variables_initializer()) expression = string.split() parse_tree.initialize(expression) parse_tree.show() print("Input Expression: {0}".format(expression)) while len(expression) > 1: op_idx, _ = self.chain_model.apply(self.session, expression) op = expression[op_idx] try: self.op_to_apply()[op](self, op_idx, expression, parse_tree) expression = [t[1] for t in parse_tree.current()] except: break return [t[1] for t in parse_tree.current()]
def gen(self, module_roots, modules, extras, output_dir): """ Generate a set of Matlab mex source files by parsing exported symbols in a set of C++ headers. The headers can be input in one (or both) of two methods: 1. specify module_root and modules Given a path to the OpenCV module root and a list of module names, the headers to parse are implicitly constructed. 2. specifiy header locations explicitly in extras Each element in the list of extras must be of the form: 'namespace=/full/path/to/extra/header.hpp' where 'namespace' is the namespace in which the definitions should be added. The output_dir specifies the directory to write the generated sources to. """ # dynamically import the parsers from jinja2 import Environment, FileSystemLoader import hdr_parser import rst_parser # parse each of the files and store in a dictionary # as a separate "namespace" parser = hdr_parser.CppHeaderParser() rst = rst_parser.RstParser(parser) rst_parser.verbose = False rst_parser.show_warnings = False rst_parser.show_errors = False rst_parser.show_critical_errors = False ns = dict((key, []) for key in modules) doc = dict((key, []) for key in modules) path_template = Template('${module}/include/opencv2/${module}.hpp') for module in modules: for module_root in module_roots: # construct a header path from the module root and a path template header = os.path.join(module_root, path_template.substitute(module=module)) if os.path.isfile(header): break else: raise Exception('no header found for module %s!' % module) # parse the definitions ns[module] = parser.parse(header) # parse the documentation rst.parse(module, os.path.join(module_root, module)) doc[module] = rst.definitions rst.definitions = {} for extra in extras: module = extra.split("=")[0] header = extra.split("=")[1] ns[module] = ns[module] + parser.parse(header) if module in ns else parser.parse(header) # cleanify the parser output parse_tree = ParseTree() parse_tree.build(ns) # setup the template engine template_dir = os.path.join(os.path.dirname(__file__), 'templates') jtemplate = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True, lstrip_blocks=True) # add the custom filters jtemplate.filters['formatMatlabConstant'] = formatMatlabConstant jtemplate.filters['convertibleToInt'] = convertibleToInt jtemplate.filters['toUpperCamelCase'] = toUpperCamelCase jtemplate.filters['toLowerCamelCase'] = toLowerCamelCase jtemplate.filters['toUnderCase'] = toUnderCase jtemplate.filters['matlabURL'] = matlabURL jtemplate.filters['stripTags'] = stripTags jtemplate.filters['filename'] = filename jtemplate.filters['comment'] = comment jtemplate.filters['inputs'] = inputs jtemplate.filters['ninputs'] = ninputs jtemplate.filters['outputs'] = outputs jtemplate.filters['noutputs'] = noutputs jtemplate.filters['qualify'] = qualify jtemplate.filters['slugify'] = slugify jtemplate.filters['only'] = only jtemplate.filters['void'] = void jtemplate.filters['not'] = flip # load the templates tfunction = jtemplate.get_template('template_function_base.cpp') tclassm = jtemplate.get_template('template_class_base.m') tclassc = jtemplate.get_template('template_class_base.cpp') tdoc = jtemplate.get_template('template_doc_base.m') tconst = jtemplate.get_template('template_map_base.m') # create the build directory output_source_dir = output_dir+'/src' output_private_dir = output_source_dir+'/private' output_class_dir = output_dir+'/+cv' output_map_dir = output_dir+'/map' if not os.path.isdir(output_source_dir): os.makedirs(output_source_dir) if not os.path.isdir(output_private_dir): os.makedirs(output_private_dir) if not os.path.isdir(output_class_dir): os.makedirs(output_class_dir) if not os.path.isdir(output_map_dir): os.makedirs(output_map_dir) # populate templates for namespace in parse_tree.namespaces: # functions for method in namespace.methods: populated = tfunction.render(fun=method, time=time, includes=namespace.name) with open(output_source_dir+'/'+method.name+'.cpp', 'wb') as f: f.write(populated.encode('utf-8')) if namespace.name in doc and method.name in doc[namespace.name]: populated = tdoc.render(fun=method, doc=doc[namespace.name][method.name], time=time) with open(output_class_dir+'/'+method.name+'.m', 'wb') as f: f.write(populated.encode('utf-8')) # classes for clss in namespace.classes: # cpp converter populated = tclassc.render(clss=clss, time=time) with open(output_private_dir+'/'+clss.name+'Bridge.cpp', 'wb') as f: f.write(populated.encode('utf-8')) # matlab classdef populated = tclassm.render(clss=clss, time=time) with open(output_class_dir+'/'+clss.name+'.m', 'wb') as f: f.write(populated.encode('utf-8')) # create a global constants lookup table const = dict(constants(todict(parse_tree.namespaces))) populated = tconst.render(constants=const, time=time) with open(output_dir+'/cv.m', 'wb') as f: f.write(populated.encode('utf-8'))
class Parser: STATEMENT_TOKENS = [ TokenType.IF, TokenType.REPEAT, TokenType.READ, TokenType.WRITE, TokenType.ID ] COMPARISON_OP_TOKENS = [TokenType.EQUAL, TokenType.LESS_THAN] ADD_OP_TOKENS = [TokenType.PLUS, TokenType.MINUS] MUL_OP_TOKENS = [TokenType.MULT, TokenType.DIVIDE] def __init__(self, stringsAndTokens: list = []): self.stringsAndTokens = stringsAndTokens self.currentIndex = 0 self.parseTree = None self.parse() def set_strings_and_tokens(self, stringsAndTokens: list) -> None: self.stringsAndTokens = stringsAndTokens self.currentIndex = 0 self.parseTree: ParseTree = None self.parse() def get_parse_tree_iterator(self) -> ParseTreeIterator: if self.parseTree is None: return None else: return self.parseTree.iterator() def has_finished(self) -> bool: return self.currentIndex >= len(self.stringsAndTokens) def current_token(self) -> TokenType: return self.stringsAndTokens[self.currentIndex][1] def current_string(self) -> str: return self.stringsAndTokens[self.currentIndex][0] def parse(self): seqNode = self.get_stmt_seq_node() self.parseTree = ParseTree() self.parseTree.set_root(seqNode) def get_stmt_seq_node(self) -> Node: node: Node = None currentNode: Node = None while not self.has_finished(): if self.current_token() not in Parser.STATEMENT_TOKENS: break addedNode = self.get_stmt_node() if node is None: node = addedNode currentNode = addedNode else: currentNode.add_right(addedNode) currentNode = addedNode if not self.has_finished(): if self.current_token() == TokenType.SEMICOLON: self.currentIndex += 1 return node def get_stmt_node(self) -> Node: currentToken = self.current_token() node = None # Get the required node depending on the TokenType of the current token # In case of if statement if currentToken == TokenType.IF: node = self.get_if_stmt_node() # In case of repeat statement elif currentToken == TokenType.REPEAT: node = self.get_repeat_stmt_node() # In case of read statement elif currentToken == TokenType.READ: node = self.get_read_stmt_node() # In case of write statement elif currentToken == TokenType.WRITE: node = self.get_write_stmt_node() # In case of assign statement elif currentToken == TokenType.ID: node = self.get_assign_stmt_node() return node def get_if_stmt_node(self) -> Node: # Skip the "if" keyword self.currentIndex += 1 # Get the exp node expNode = self.get_exp_node() # Skip the "then" keyword self.currentIndex += 1 # Get the "then" statement sequence thenStmtSeqNode = self.get_stmt_seq_node() # Check if "else" exists elseStmtSeqNode = None if self.current_token() == TokenType.ELSE: # Skip the "else" keyword self.currentIndex += 1 # Get the "else" statement sequence elseStmtSeqNode = self.get_stmt_seq_node() # Skip the "end" keyword self.currentIndex += 1 # Create the "if" statement node node = Node("if", None, StructType.STATEMENT) node.add_child(expNode) node.add_child(thenStmtSeqNode) if elseStmtSeqNode is not None: node.add_child(elseStmtSeqNode) return node def get_repeat_stmt_node(self) -> Node: # Skip "repeat" keyword self.currentIndex += 1 # Get statement sequence stmtSeqNode = self.get_stmt_seq_node() # Skip "until" keyword self.currentIndex += 1 # Get exp node expNode = self.get_exp_node() # Create "repeat" statement node node = Node("repeat", None, StructType.STATEMENT) node.add_child(stmtSeqNode) node.add_child(expNode) return node def get_read_stmt_node(self) -> Node: # Skip "read" keyword self.currentIndex += 1 # Get identifier string identifier = self.current_string() # Skip the identifier self.currentIndex += 1 # Create "read" statement node node = Node("read", identifier, StructType.STATEMENT) return node def get_write_stmt_node(self) -> Node: # Skip "write" keyword self.currentIndex += 1 # Get exp node expNode = self.get_exp_node() # Create "write" statement node node = Node("write", None, StructType.STATEMENT) node.add_child(expNode) return node def get_assign_stmt_node(self) -> Node: # Get identifier string identifier = self.current_string() # Skip identifier self.currentIndex += 1 # Skip assign ":=" symbol self.currentIndex += 1 # Get exp node expNode = self.get_exp_node() # Create the "assign" statement node node = Node("assign", identifier, StructType.STATEMENT) node.add_child(expNode) return node def get_exp_node(self) -> Node: # Get a simple exp simpleExpNode1: Node = self.get_simple_exp_node() # Create other variables for in case there was an comparison op compOpNode: Node = None simpleExpNode2: Node = None # Variable used to know if the the expression is just a simple expression or it contains a comparison op simpleExpOnly = True if not self.has_finished(): # Check if there is a comparison op if self.current_token() in Parser.COMPARISON_OP_TOKENS: simpleExpOnly = False # Get the nodes of the comparison op and the other simple expression compOpNode = self.get_op_node() simpleExpNode2 = self.get_simple_exp_node() # Checks if it is a simple expression only if simpleExpOnly: node = simpleExpNode1 # In case there was a comparison op else: compOpNode.add_child(simpleExpNode1) compOpNode.add_child(simpleExpNode2) node = compOpNode return node def get_simple_exp_node(self) -> Node: # Initialize lists termsNodes = [] addOpsNodes = [] # Add a term because there must be at least 1 term termsNodes.append(self.get_term_node()) # Keep adding the repeated pattern which is "addop term" while not self.has_finished(): if self.current_token() in Parser.ADD_OP_TOKENS: addOpsNodes.append(self.get_op_node()) termsNodes.append(self.get_term_node()) else: break # Get and return the tree created from connecting the terms nodes and the add ops nodes in the right order return self.get_ops_tree(termsNodes, addOpsNodes) def get_term_node(self) -> Node: # Initialize lists factorsNodes = [] mulOpsNodes = [] # Add a factor because there must be at least 1 factor factorsNodes.append(self.get_factor_node()) # Keep adding the repeated pattern which is "multop factor" while not self.has_finished(): if self.current_token() in Parser.MUL_OP_TOKENS: mulOpsNodes.append(self.get_op_node()) factorsNodes.append(self.get_factor_node()) else: break # Get and return the tree created from connecting the factors nodes and the mul ops nodes in the right order return self.get_ops_tree(factorsNodes, mulOpsNodes) def get_factor_node(self) -> Node: node: Node = None currentToken = self.current_token() currentString = self.current_string() if currentToken == TokenType.ID: node = Node("id", currentString, StructType.EXPRESSION) # Skip the identifier self.currentIndex += 1 elif currentToken == TokenType.NUM: node = Node("const", currentString, StructType.EXPRESSION) # Skip the number self.currentIndex += 1 elif currentToken == TokenType.OPEN_PARAN: # Skip open paran "(" self.currentIndex += 1 # Get exp node node = self.get_exp_node() # Skip close paran ")" self.currentIndex += 1 return node def get_ops_tree(self, valuesNodes: list, opsNodes: list) -> Node: opsNodesLength = len(opsNodes) # Initialize the nodes variables leftNode: Node = valuesNodes[0] centerNode: Node = valuesNodes[0] # Create the tree for i in range(opsNodesLength): # Left node is the center node from the previous iteration # Right node is the next value rightNode = valuesNodes[i + 1] # The center node is the op node centerNode = opsNodes[i] # Add both sides of the op node as children to the op node centerNode.add_child(leftNode) centerNode.add_child(rightNode) # Update the value of the left node for the next iteration leftNode = centerNode # The result is the center node node = centerNode return node def get_op_node(self) -> Node: # Create the "op" node node = Node("op", self.current_string(), StructType.EXPRESSION) # Skip the op self.currentIndex += 1 return node
n_states = n_states_options[int(sys.argv[4])] n_games = 100 init_temp = 1 d = 1 max_game_rounds = 500 max_nodes = 100 n_MC_simulations = 1000 inner_SA = SimulatedAnnealing(n_SA_iterations, max_game_rounds, n_games, init_temp, d, False) outer_SA = SimulatedAnnealing(1000, max_game_rounds, n_games, init_temp, d, True) dsl = DSL() if search_type == 0: # IW tree = ParseTree(dsl=dsl, max_nodes=max_nodes, k=k, is_IW=True) search_algo = IteratedWidth(tree, n_states, k) suffix = 'IW' + '_LS' + str(LS_type) + '_SA' + str(n_SA_iterations) + '_ST' + str(n_states) + '_k' + str(k) + '_GA' + str(n_games) elif search_type == 1: # BFS tree = ParseTree(dsl=dsl, max_nodes=max_nodes, k=k, is_IW=False) search_algo = BFS(tree, n_states) suffix = 'BFS' + '_LS' + str(LS_type) + '_SA' + str(n_SA_iterations) + '_ST' + str(n_states) + '_k' + str(k) + '_GA' + str(n_games) SP = SelfPlay(n_games, n_MC_simulations, max_game_rounds, max_nodes, search_algo, inner_SA, outer_SA, dsl, LS_type, suffix) start = time.time() SP.run() elapsed = time.time() - start print('Elapsed = ', elapsed)
def gen(self, module_root, modules, extras, output_dir, out_h, out_c, out_xml): """ Generate a set of c source files by parsing exported symbols in a set of C++ headers. The headers can be input in one (or both) of two methods: 1. specify module_root and modules Given a path to the OpenCV module root and a list of module names, the headers to parse are implicitly constructed. 2. specifiy header locations explicitly in extras Each element in the list of extras must be of the form: 'namespace=/full/path/to/extra/header.hpp' where 'namespace' is the namespace in which the definitions should be added. The output_dir specifies the directory to write the generated sources to. """ # parse each of the files and store in a dictionary # as a separate "namespace" parser = CppHeaderParser() ns = dict((key, []) for key in modules) doc = dict((key, []) for key in modules) path_template = Template('${module}/include/opencv2/${module}.hpp') for module in modules: # construct a header path from the module root and a path template header = os.path.join(module_root, path_template.substitute(module=module)) # parse the definitions ns[module] = parser.parse(header) for extra in extras: module = extra.split("=")[0] header = extra.split("=")[1] ns[module] = ns[module] + parser.parse( header) if module in ns else parser.parse(header) # cleanify the parser output parse_tree = ParseTree() parse_tree.build(ns) # populate templates for namespace in parse_tree.namespaces: # functions print("// " + namespace.name) self.buf_xml.write("<category name=\"" + namespace.name + "\">\n") for method in namespace.methods: if method.name in self.IGNORE: continue longname = namespace.name + "_" + method.name if self.overload(longname): continue self.gen_method(method, None, namespace.name, "") self.buf_xml.write(" <block type=\"" + longname + "\"></block>\n") # classes for clss in namespace.classes: if clss.name in self.BLACK: print("// " + namespace.name + "." + clss.name + " skipped.") continue #~ print(namespace.name + "." + clss.name) #print(clss) for method in clss.methods: longname = namespace.name + "_" + clss.name + "_" + method.name if self.overload(longname): continue self.gen_method(method, clss.name, namespace.name, "") self.buf_xml.write(" <block type=\"" + longname + "\"></block>\n") self.buf_xml.write("</category>\n") # create a global constants lookup table # const = dict(constants(todict(parse_tree.namespaces))) # create a global constants lookup table const = dict(constants(todict(parse_tree.namespaces))) f = open("gen/const.js", "wb") f.write("[\n") for c in sorted(const): f.write(" [ " + c + ", cv2." + c + "],\n") f.write("]\n") self.save(output_dir, out_h, self.buf_decl) self.save(output_dir, out_c, self.buf_def) self.save(output_dir, out_xml, self.buf_xml)
def parse_tree(self): return ParseTree.from_parse_string(self.parse)
def parse(self): seqNode = self.get_stmt_seq_node() self.parseTree = ParseTree() self.parseTree.set_root(seqNode)