def _expression(self):
tree = ParseTree("Expr")
node = self._log_expr()
if node is not None:
tree.add_child(node)
return tree
return None
def create_parse_trees(self, chart, start=None):
if start is None:
start = list(chart[len(chart) - 1][0])[0]
if len(start.pointers) == 0:
return [ParseTree(start.data)]
parse_trees = []
for l, r in start.pointers:
left_tree = self.create_parse_trees(chart, l)
right_tree = self.create_parse_trees(chart, r)
for x in left_tree:
for y in right_tree:
parse_trees.append(ParseTree(start.data, x, y))
return parse_trees
def parse(self, sentence):
"""Read a sentence (iterable of terminals), and:
- if it's in the language, return its parse tree
- otherwise, raise NotInLanguage
[TRDB] Algorithm 4.3 p. 187"""
# use a mixed stack with:
# - symbols corresponding to the productions in progress
# - tree nodes to go back to when their children are complete
stack = [self.g.END, self.g.start_symbol]
cur_node = None
tok_stream = chain(iter(sentence), (self.g.END, ))
token = next(tok_stream)
while stack:
state = stack.pop()
if isinstance(state, ParseTree):
cur_node = state
elif state in self.g.non_terminals:
if (state, token) not in self.table:
msg = "In state '{}', got '{}'".format(state, token)
raise self.NotInLanguage(msg)
new_node = ParseTree(state)
if cur_node:
cur_node.children.append(new_node)
stack.append(cur_node)
cur_node = new_node
prod_idx = self.table[state, token]
rhs = self.g.productions[prod_idx][1]
stack.extend(list(reversed(rhs)))
if len(rhs) == 0:
cur_node.children.append(ParseTree(''))
else:
if token != state:
msg = "Expected '{}', got '{}'".format(state, token)
raise self.NotInLanguage(msg)
if state != self.g.END:
cur_node.children.append(ParseTree(token))
token = next(tok_stream)
return cur_node
def _identifier(self):
tree = ParseTree("identifier")
count = 0
while True:
if self._out_of_range():
break
if self.string[self.index] in self.identifiers:
self.index += 1
count += 1
else:
break
if count:
tree.add_child(ParseTree(self.string[self.index-count: self.index]))
return tree
return None
def _log_value(self):
tree = ParseTree("log value")
if self.string[self.index: self.index + 4] == "true":
self.index += 4
tree.add_child(ParseTree("true"))
return tree
if self.string[self.index: self.index + 5] == "false":
self.index += 5
tree.add_child(ParseTree("false"))
return tree
return None
def _log_expr_(self):
if self._out_of_range():
return None
tree = ParseTree("log expr'")
if self.string[self.index] == "!":
self.index += 1
tree.add_child(ParseTree("!"))
log_mon_node = self._log_mon()
if log_mon_node:
tree.add_child(log_mon_node)
log_expr_node = self._log_expr_()
if log_expr_node:
tree.add_child(log_expr_node)
return tree
return None
def parse(self, sentence):
"""Read a sentence (iterable of terminals), and:
- if it's in the language, do nothing (for now),
- otherwise, raise NotInLanguage
[TRDB] Algorithm Fig 4.30 p. 219"""
# store pairs on the stack instead of two values
stack = [(0, None)]
tok_stream = chain(iter(sentence), (self.g.END,))
token = next(tok_stream)
while True:
state = stack[-1][0]
try:
action, info = self.actions[state, token]
except KeyError:
msg = "In state '{}', got '{}'".format(state, token)
raise self.NotInLanguage(msg)
if action == self.SHIFT:
node = ParseTree(token)
stack.append((info, node))
token = next(tok_stream)
elif action == self.REDUCE:
lhs, rhs = self.g.productions[info]
children = [stack.pop()[1] for _ in range(len(rhs))]
children.reverse()
node = ParseTree(lhs, children)
prev_state = stack[-1][0]
new_state = self.gotos[prev_state, lhs]
stack.append((new_state, node))
else: # action == self.ACCEPT:
return stack[-1][1]
def _sec_expr(self):
tree = ParseTree("sec_expr")
first_exp_node = self._first_expr()
if first_exp_node:
tree.add_child(first_exp_node)
return tree
if self._out_of_range():
return None
if self.string[self.index] == "~":
self.index += 1
tree.add_child(ParseTree("~"))
first_exp_node = self._first_expr()
if first_exp_node:
tree.add_child(first_exp_node)
return tree
return None
def _log_mon(self):
tree = ParseTree("log mon")
sec_expr_node = self._sec_expr()
if sec_expr_node:
tree.add_child(sec_expr_node)
log_mon_node = self._log_mon_()
if log_mon_node is not None:
tree.add_child(log_mon_node)
if tree.childs:
return tree
return None
def _log_expr(self):
tree = ParseTree("log expr")
log_node = self._log_mon()
if log_node is not None:
tree.add_child(log_node)
log_expr_node = self._log_expr_()
if log_expr_node is not None:
tree.add_child(log_expr_node)
if tree.childs:
return tree
return None
def _first_expr(self):
tree = ParseTree("first_expr")
log_value_node = self._log_value()
if log_value_node:
tree.add_child(log_value_node)
return tree
identifier_node = self._identifier()
if identifier_node:
tree.add_child(identifier_node)
return tree
return None
def evaluate(self, string):
max_length = 10
max_depth = 6
session_parse_tree = tf.Session()
parse_tree = ParseTree(session_parse_tree, max_length, max_depth)
session_parse_tree.run(tf.global_variables_initializer())
expression = string.split()
parse_tree.initialize(expression)
parse_tree.show()
print("Input Expression: {0}".format(expression))
while len(expression) > 1:
op_idx, _ = self.chain_model.apply(self.session, expression)
op = expression[op_idx]
try:
self.op_to_apply()[op](self, op_idx, expression, parse_tree)
expression = [t[1] for t in parse_tree.current()]
except:
break
return [t[1] for t in parse_tree.current()]
def gen(self, module_roots, modules, extras, output_dir):
"""
Generate a set of Matlab mex source files by parsing exported symbols
in a set of C++ headers. The headers can be input in one (or both) of
two methods:
1. specify module_root and modules
Given a path to the OpenCV module root and a list of module names,
the headers to parse are implicitly constructed.
2. specifiy header locations explicitly in extras
Each element in the list of extras must be of the form:
'namespace=/full/path/to/extra/header.hpp' where 'namespace' is
the namespace in which the definitions should be added.
The output_dir specifies the directory to write the generated sources
to.
"""
# dynamically import the parsers
from jinja2 import Environment, FileSystemLoader
import hdr_parser
import rst_parser
# parse each of the files and store in a dictionary
# as a separate "namespace"
parser = hdr_parser.CppHeaderParser()
rst = rst_parser.RstParser(parser)
rst_parser.verbose = False
rst_parser.show_warnings = False
rst_parser.show_errors = False
rst_parser.show_critical_errors = False
ns = dict((key, []) for key in modules)
doc = dict((key, []) for key in modules)
path_template = Template('${module}/include/opencv2/${module}.hpp')
for module in modules:
for module_root in module_roots:
# construct a header path from the module root and a path template
header = os.path.join(module_root, path_template.substitute(module=module))
if os.path.isfile(header):
break
else:
raise Exception('no header found for module %s!' % module)
# parse the definitions
ns[module] = parser.parse(header)
# parse the documentation
rst.parse(module, os.path.join(module_root, module))
doc[module] = rst.definitions
rst.definitions = {}
for extra in extras:
module = extra.split("=")[0]
header = extra.split("=")[1]
ns[module] = ns[module] + parser.parse(header) if module in ns else parser.parse(header)
# cleanify the parser output
parse_tree = ParseTree()
parse_tree.build(ns)
# setup the template engine
template_dir = os.path.join(os.path.dirname(__file__), 'templates')
jtemplate = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True, lstrip_blocks=True)
# add the custom filters
jtemplate.filters['formatMatlabConstant'] = formatMatlabConstant
jtemplate.filters['convertibleToInt'] = convertibleToInt
jtemplate.filters['toUpperCamelCase'] = toUpperCamelCase
jtemplate.filters['toLowerCamelCase'] = toLowerCamelCase
jtemplate.filters['toUnderCase'] = toUnderCase
jtemplate.filters['matlabURL'] = matlabURL
jtemplate.filters['stripTags'] = stripTags
jtemplate.filters['filename'] = filename
jtemplate.filters['comment'] = comment
jtemplate.filters['inputs'] = inputs
jtemplate.filters['ninputs'] = ninputs
jtemplate.filters['outputs'] = outputs
jtemplate.filters['noutputs'] = noutputs
jtemplate.filters['qualify'] = qualify
jtemplate.filters['slugify'] = slugify
jtemplate.filters['only'] = only
jtemplate.filters['void'] = void
jtemplate.filters['not'] = flip
# load the templates
tfunction = jtemplate.get_template('template_function_base.cpp')
tclassm = jtemplate.get_template('template_class_base.m')
tclassc = jtemplate.get_template('template_class_base.cpp')
tdoc = jtemplate.get_template('template_doc_base.m')
tconst = jtemplate.get_template('template_map_base.m')
# create the build directory
output_source_dir = output_dir+'/src'
output_private_dir = output_source_dir+'/private'
output_class_dir = output_dir+'/+cv'
output_map_dir = output_dir+'/map'
if not os.path.isdir(output_source_dir):
os.makedirs(output_source_dir)
if not os.path.isdir(output_private_dir):
os.makedirs(output_private_dir)
if not os.path.isdir(output_class_dir):
os.makedirs(output_class_dir)
if not os.path.isdir(output_map_dir):
os.makedirs(output_map_dir)
# populate templates
for namespace in parse_tree.namespaces:
# functions
for method in namespace.methods:
populated = tfunction.render(fun=method, time=time, includes=namespace.name)
with open(output_source_dir+'/'+method.name+'.cpp', 'wb') as f:
f.write(populated.encode('utf-8'))
if namespace.name in doc and method.name in doc[namespace.name]:
populated = tdoc.render(fun=method, doc=doc[namespace.name][method.name], time=time)
with open(output_class_dir+'/'+method.name+'.m', 'wb') as f:
f.write(populated.encode('utf-8'))
# classes
for clss in namespace.classes:
# cpp converter
populated = tclassc.render(clss=clss, time=time)
with open(output_private_dir+'/'+clss.name+'Bridge.cpp', 'wb') as f:
f.write(populated.encode('utf-8'))
# matlab classdef
populated = tclassm.render(clss=clss, time=time)
with open(output_class_dir+'/'+clss.name+'.m', 'wb') as f:
f.write(populated.encode('utf-8'))
# create a global constants lookup table
const = dict(constants(todict(parse_tree.namespaces)))
populated = tconst.render(constants=const, time=time)
with open(output_dir+'/cv.m', 'wb') as f:
f.write(populated.encode('utf-8'))
class Parser:
STATEMENT_TOKENS = [
TokenType.IF, TokenType.REPEAT, TokenType.READ, TokenType.WRITE,
TokenType.ID
]
COMPARISON_OP_TOKENS = [TokenType.EQUAL, TokenType.LESS_THAN]
ADD_OP_TOKENS = [TokenType.PLUS, TokenType.MINUS]
MUL_OP_TOKENS = [TokenType.MULT, TokenType.DIVIDE]
def __init__(self, stringsAndTokens: list = []):
self.stringsAndTokens = stringsAndTokens
self.currentIndex = 0
self.parseTree = None
self.parse()
def set_strings_and_tokens(self, stringsAndTokens: list) -> None:
self.stringsAndTokens = stringsAndTokens
self.currentIndex = 0
self.parseTree: ParseTree = None
self.parse()
def get_parse_tree_iterator(self) -> ParseTreeIterator:
if self.parseTree is None:
return None
else:
return self.parseTree.iterator()
def has_finished(self) -> bool:
return self.currentIndex >= len(self.stringsAndTokens)
def current_token(self) -> TokenType:
return self.stringsAndTokens[self.currentIndex][1]
def current_string(self) -> str:
return self.stringsAndTokens[self.currentIndex][0]
def parse(self):
seqNode = self.get_stmt_seq_node()
self.parseTree = ParseTree()
self.parseTree.set_root(seqNode)
def get_stmt_seq_node(self) -> Node:
node: Node = None
currentNode: Node = None
while not self.has_finished():
if self.current_token() not in Parser.STATEMENT_TOKENS:
break
addedNode = self.get_stmt_node()
if node is None:
node = addedNode
currentNode = addedNode
else:
currentNode.add_right(addedNode)
currentNode = addedNode
if not self.has_finished():
if self.current_token() == TokenType.SEMICOLON:
self.currentIndex += 1
return node
def get_stmt_node(self) -> Node:
currentToken = self.current_token()
node = None
# Get the required node depending on the TokenType of the current token
# In case of if statement
if currentToken == TokenType.IF:
node = self.get_if_stmt_node()
# In case of repeat statement
elif currentToken == TokenType.REPEAT:
node = self.get_repeat_stmt_node()
# In case of read statement
elif currentToken == TokenType.READ:
node = self.get_read_stmt_node()
# In case of write statement
elif currentToken == TokenType.WRITE:
node = self.get_write_stmt_node()
# In case of assign statement
elif currentToken == TokenType.ID:
node = self.get_assign_stmt_node()
return node
def get_if_stmt_node(self) -> Node:
# Skip the "if" keyword
self.currentIndex += 1
# Get the exp node
expNode = self.get_exp_node()
# Skip the "then" keyword
self.currentIndex += 1
# Get the "then" statement sequence
thenStmtSeqNode = self.get_stmt_seq_node()
# Check if "else" exists
elseStmtSeqNode = None
if self.current_token() == TokenType.ELSE:
# Skip the "else" keyword
self.currentIndex += 1
# Get the "else" statement sequence
elseStmtSeqNode = self.get_stmt_seq_node()
# Skip the "end" keyword
self.currentIndex += 1
# Create the "if" statement node
node = Node("if", None, StructType.STATEMENT)
node.add_child(expNode)
node.add_child(thenStmtSeqNode)
if elseStmtSeqNode is not None:
node.add_child(elseStmtSeqNode)
return node
def get_repeat_stmt_node(self) -> Node:
# Skip "repeat" keyword
self.currentIndex += 1
# Get statement sequence
stmtSeqNode = self.get_stmt_seq_node()
# Skip "until" keyword
self.currentIndex += 1
# Get exp node
expNode = self.get_exp_node()
# Create "repeat" statement node
node = Node("repeat", None, StructType.STATEMENT)
node.add_child(stmtSeqNode)
node.add_child(expNode)
return node
def get_read_stmt_node(self) -> Node:
# Skip "read" keyword
self.currentIndex += 1
# Get identifier string
identifier = self.current_string()
# Skip the identifier
self.currentIndex += 1
# Create "read" statement node
node = Node("read", identifier, StructType.STATEMENT)
return node
def get_write_stmt_node(self) -> Node:
# Skip "write" keyword
self.currentIndex += 1
# Get exp node
expNode = self.get_exp_node()
# Create "write" statement node
node = Node("write", None, StructType.STATEMENT)
node.add_child(expNode)
return node
def get_assign_stmt_node(self) -> Node:
# Get identifier string
identifier = self.current_string()
# Skip identifier
self.currentIndex += 1
# Skip assign ":=" symbol
self.currentIndex += 1
# Get exp node
expNode = self.get_exp_node()
# Create the "assign" statement node
node = Node("assign", identifier, StructType.STATEMENT)
node.add_child(expNode)
return node
def get_exp_node(self) -> Node:
# Get a simple exp
simpleExpNode1: Node = self.get_simple_exp_node()
# Create other variables for in case there was an comparison op
compOpNode: Node = None
simpleExpNode2: Node = None
# Variable used to know if the the expression is just a simple expression or it contains a comparison op
simpleExpOnly = True
if not self.has_finished():
# Check if there is a comparison op
if self.current_token() in Parser.COMPARISON_OP_TOKENS:
simpleExpOnly = False
# Get the nodes of the comparison op and the other simple expression
compOpNode = self.get_op_node()
simpleExpNode2 = self.get_simple_exp_node()
# Checks if it is a simple expression only
if simpleExpOnly:
node = simpleExpNode1
# In case there was a comparison op
else:
compOpNode.add_child(simpleExpNode1)
compOpNode.add_child(simpleExpNode2)
node = compOpNode
return node
def get_simple_exp_node(self) -> Node:
# Initialize lists
termsNodes = []
addOpsNodes = []
# Add a term because there must be at least 1 term
termsNodes.append(self.get_term_node())
# Keep adding the repeated pattern which is "addop term"
while not self.has_finished():
if self.current_token() in Parser.ADD_OP_TOKENS:
addOpsNodes.append(self.get_op_node())
termsNodes.append(self.get_term_node())
else:
break
# Get and return the tree created from connecting the terms nodes and the add ops nodes in the right order
return self.get_ops_tree(termsNodes, addOpsNodes)
def get_term_node(self) -> Node:
# Initialize lists
factorsNodes = []
mulOpsNodes = []
# Add a factor because there must be at least 1 factor
factorsNodes.append(self.get_factor_node())
# Keep adding the repeated pattern which is "multop factor"
while not self.has_finished():
if self.current_token() in Parser.MUL_OP_TOKENS:
mulOpsNodes.append(self.get_op_node())
factorsNodes.append(self.get_factor_node())
else:
break
# Get and return the tree created from connecting the factors nodes and the mul ops nodes in the right order
return self.get_ops_tree(factorsNodes, mulOpsNodes)
def get_factor_node(self) -> Node:
node: Node = None
currentToken = self.current_token()
currentString = self.current_string()
if currentToken == TokenType.ID:
node = Node("id", currentString, StructType.EXPRESSION)
# Skip the identifier
self.currentIndex += 1
elif currentToken == TokenType.NUM:
node = Node("const", currentString, StructType.EXPRESSION)
# Skip the number
self.currentIndex += 1
elif currentToken == TokenType.OPEN_PARAN:
# Skip open paran "("
self.currentIndex += 1
# Get exp node
node = self.get_exp_node()
# Skip close paran ")"
self.currentIndex += 1
return node
def get_ops_tree(self, valuesNodes: list, opsNodes: list) -> Node:
opsNodesLength = len(opsNodes)
# Initialize the nodes variables
leftNode: Node = valuesNodes[0]
centerNode: Node = valuesNodes[0]
# Create the tree
for i in range(opsNodesLength):
# Left node is the center node from the previous iteration
# Right node is the next value
rightNode = valuesNodes[i + 1]
# The center node is the op node
centerNode = opsNodes[i]
# Add both sides of the op node as children to the op node
centerNode.add_child(leftNode)
centerNode.add_child(rightNode)
# Update the value of the left node for the next iteration
leftNode = centerNode
# The result is the center node
node = centerNode
return node
def get_op_node(self) -> Node:
# Create the "op" node
node = Node("op", self.current_string(), StructType.EXPRESSION)
# Skip the op
self.currentIndex += 1
return node
n_states = n_states_options[int(sys.argv[4])]
n_games = 100
init_temp = 1
d = 1
max_game_rounds = 500
max_nodes = 100
n_MC_simulations = 1000
inner_SA = SimulatedAnnealing(n_SA_iterations, max_game_rounds, n_games, init_temp, d, False)
outer_SA = SimulatedAnnealing(1000, max_game_rounds, n_games, init_temp, d, True)
dsl = DSL()
if search_type == 0:
# IW
tree = ParseTree(dsl=dsl, max_nodes=max_nodes, k=k, is_IW=True)
search_algo = IteratedWidth(tree, n_states, k)
suffix = 'IW' + '_LS' + str(LS_type) + '_SA' + str(n_SA_iterations) + '_ST' + str(n_states) + '_k' + str(k) + '_GA' + str(n_games)
elif search_type == 1:
# BFS
tree = ParseTree(dsl=dsl, max_nodes=max_nodes, k=k, is_IW=False)
search_algo = BFS(tree, n_states)
suffix = 'BFS' + '_LS' + str(LS_type) + '_SA' + str(n_SA_iterations) + '_ST' + str(n_states) + '_k' + str(k) + '_GA' + str(n_games)
SP = SelfPlay(n_games, n_MC_simulations, max_game_rounds, max_nodes, search_algo, inner_SA, outer_SA, dsl, LS_type, suffix)
start = time.time()
SP.run()
elapsed = time.time() - start
print('Elapsed = ', elapsed)
def gen(self, module_root, modules, extras, output_dir, out_h, out_c,
out_xml):
"""
Generate a set of c source files by parsing exported symbols
in a set of C++ headers. The headers can be input in one (or both) of
two methods:
1. specify module_root and modules
Given a path to the OpenCV module root and a list of module names,
the headers to parse are implicitly constructed.
2. specifiy header locations explicitly in extras
Each element in the list of extras must be of the form:
'namespace=/full/path/to/extra/header.hpp' where 'namespace' is
the namespace in which the definitions should be added.
The output_dir specifies the directory to write the generated sources
to.
"""
# parse each of the files and store in a dictionary
# as a separate "namespace"
parser = CppHeaderParser()
ns = dict((key, []) for key in modules)
doc = dict((key, []) for key in modules)
path_template = Template('${module}/include/opencv2/${module}.hpp')
for module in modules:
# construct a header path from the module root and a path template
header = os.path.join(module_root,
path_template.substitute(module=module))
# parse the definitions
ns[module] = parser.parse(header)
for extra in extras:
module = extra.split("=")[0]
header = extra.split("=")[1]
ns[module] = ns[module] + parser.parse(
header) if module in ns else parser.parse(header)
# cleanify the parser output
parse_tree = ParseTree()
parse_tree.build(ns)
# populate templates
for namespace in parse_tree.namespaces:
# functions
print("// " + namespace.name)
self.buf_xml.write("<category name=\"" + namespace.name + "\">\n")
for method in namespace.methods:
if method.name in self.IGNORE: continue
longname = namespace.name + "_" + method.name
if self.overload(longname): continue
self.gen_method(method, None, namespace.name, "")
self.buf_xml.write(" <block type=\"" + longname +
"\"></block>\n")
# classes
for clss in namespace.classes:
if clss.name in self.BLACK:
print("// " + namespace.name + "." + clss.name +
" skipped.")
continue
#~ print(namespace.name + "." + clss.name)
#print(clss)
for method in clss.methods:
longname = namespace.name + "_" + clss.name + "_" + method.name
if self.overload(longname): continue
self.gen_method(method, clss.name, namespace.name, "")
self.buf_xml.write(" <block type=\"" + longname +
"\"></block>\n")
self.buf_xml.write("</category>\n")
# create a global constants lookup table
# const = dict(constants(todict(parse_tree.namespaces)))
# create a global constants lookup table
const = dict(constants(todict(parse_tree.namespaces)))
f = open("gen/const.js", "wb")
f.write("[\n")
for c in sorted(const):
f.write(" [ " + c + ", cv2." + c + "],\n")
f.write("]\n")
self.save(output_dir, out_h, self.buf_decl)
self.save(output_dir, out_c, self.buf_def)
self.save(output_dir, out_xml, self.buf_xml)
def parse_tree(self):
return ParseTree.from_parse_string(self.parse)
def parse(self):
seqNode = self.get_stmt_seq_node()
self.parseTree = ParseTree()
self.parseTree.set_root(seqNode)
