def parse(self):
""" Run the parser and get the AST"""
if self.debug:
print(self.code)
logging.debug("Running the parser ...")
parser = c_parser.CParser(lex_optimize=True,
lextab='lextab',
yacc_optimize=True,
yacctab='yacctab',
yacc_debug=False)
self.ast = parser.parse(self.code,
filename=self.filename,
debuglevel=self.parse_dbg_lvl)
if self.ast is None:
logging.error("Parsing failed ...")
raise RuntimeError("Parsing failed ...")
if debug:
self.ast.show()
def explain_c_declaration(c_decl):
""" Parses the declaration in c_decl and returns a text
explanation as a string.
The last external node of the string is used, to allow
earlier typedefs for used types.
"""
parser = c_parser.CParser()
try:
node = parser.parse(c_decl, filename='<stdin>')
except c_parser.ParseError:
e = sys.exc_info()[1]
return "Parse error:" + str(e)
if (not isinstance(node, c_ast.FileAST) or
not isinstance(node.ext[-1], c_ast.Decl)
):
return "Not a valid declaration"
return _explain_decl_node(node.ext[-1])
def __init__(self):
self._parser = c_parser.CParser(lex_optimize=True,
lextab='jsoncc.lextab',
yacc_optimize=True,
yacctab='jsoncc.yacctab',
yacc_debug=False)
self._asts = {
'strdup': self.parse(strdup),
'realloc': self.parse(realloc),
'get_int': self.parse(get_int),
'get_double': self.parse(get_double),
'get_str': self.parse(get_str),
'get_str_array': self.parse(get_str_array),
'get_array': self.parse(get_array),
'decode_func': self.parse(decode_func),
'encode_func': self.parse(encode_func),
'free_func': self.parse(free_func),
'free_item': self.parse(free_item),
'free_str_arr': self.parse(free_str_arr),
'find_opt_token': self.parse(find_opt_token),
'find_token': self.parse(find_token),
'emit_key': self.parse(emit_key),
'emit_quoted': self.parse(emit_quoted),
'emit_number': self.parse(emit_number),
'emit_bool': self.parse(emit_bool),
'emit_str': self.parse(emit_str),
'emit_optional': self.parse(emit_optional),
}
self._type_fmt_map = {
"unsigned int": "%u",
"unsigned short int": "%hu",
"int": "%d",
"short int": "%hd",
"unsigned long": "%lu",
"long": "%ld",
"float": "%f",
"double": "%lf",
}
def sm_test(filename):
print("Strip Mining Test")
with open(filename, "r") as source:
# reading the c file
parser = c_parser.CParser()
astc = parser.parse(source.read())
# convert to python
pysrc = CtoPy(astc)
tree = ast.parse(pysrc.getPy())
analyze = Analyze(tree)
xform = Transform(analyze)
# Input program
print("\nInput Program")
print(xform.codegen())
# Dimensions
in_dim = xform.analyze.dims
out_dim = in_dim + 1
# Type of dimensions
in_dim_type = xform.analyze.getdimtype()
# Input alphabet and order
in_alp = xform.analyze.getalp()
in_ord = xform.analyze.getord()
# strip dimension
strip_dim = 0
# strip size
strip_size = 2
# Transform
xf = Transformation(name='sm',
in_dim=in_dim,
out_dim=out_dim,
in_dim_type=in_dim_type,
in_alp=in_alp,
in_ord=in_ord,
dim_strip=strip_dim,
strip_size=strip_size)
xform.transform(xf)
# Output program
print("\nOutput Program")
print(xform.codegen())
def parse_prepro(filename):
os.system("sed -i 's/asm volatile/asm/' '%s'" % filename)
os.system("sed -i 's/asm __volatile__/asm/' '%s'" % filename)
cpp_path = "arm-none-eabi-cpp"
cpp_args = [
'-Dasm(...)=',
'-D__asm__(...)=',
'-D__extension__=',
'-D__attribute__(...)=',
'-D__builtin_va_list=int',
'-D__builtin_va_arg(a,b)=0',
]
text = preprocess_file(filename, cpp_path, cpp_args)
# pycparser doesn't like a C file to begin with ;
# (happens after removing the first asm block from arm-mcr.h)
text = "struct __dummy__;\n" + text
#~ print(text)
return c_parser.CParser().parse(text, lookup_c(filename))
def check_loop_types(self, function):
"""Check the loops used in a function
Create the ast of a C function, recursively browse it and produce the
numbers of loops that is contained in the function.
Returns:
tuple: (#While, #For, #Do...While)
Raises:
Parsing Error stuffs from pycparser if the student coded with his ass
rather than hands.
"""
if not self.parser:
self.parser = c_parser.CParser()
ast = self.parser.parse(function, filename='<none>')
lc = LoopCounter()
lc.visit(ast)
return (lc.while_count, lc.for_count, lc.dowhile_count)
def deptest_cm(filename):
print("Code Motion Test")
with open(filename, "r") as source:
# reading the c file
parser = c_parser.CParser()
astc = parser.parse(source.read())
# convert to python
pysrc = CtoPy(astc)
tree = ast.parse(pysrc.getPy())
analyze = Analyze(tree)
xform = Transform(analyze)
# Input program
print("\nInput Program")
print(xform.codegen())
# Dimensions
in_dim = xform.analyze.dims
out_dim = in_dim
# Type of dimensions
in_dim_type = xform.analyze.getdimtype()
# Input alphabet and order
in_alp = xform.analyze.getalp()
in_ord = xform.analyze.getord()
# Output order
out_ord = [['e', 't1', 'r1'], ['e', 's1', 'r2l', 'r2r']]
xf = Transformation(
name ='cm',
in_dim = in_dim,
out_dim = out_dim,
in_dim_type = in_dim_type,
in_alp = in_alp,
in_ord = in_ord,
out_ord = out_ord)
xform.analyze.depanalyze()
print(xform.analyze.getdeps())
for i, wt in enumerate(xform.analyze.deps):
Dep = Dependence(wt)
print("Witness Tuple", i, ": ", Dep.test(xf))
def var_copy_assignExpDetail(self):
# find the type, size, and parentFunction of variables referred in each copy expression
# and append it to the existing expression
text = self.var_parseForYacc(self.code)
#print text
# create a pycparser
parser = c_parser.CParser()
ast = parser.parse(text, filename='<none>')
# generate the XML tree
#ast.show()
codeAstXml = open('code_ast.xml','w')
ast.showXml(codeAstXml)
codeAstXml.close()
tree = ET.parse('code_ast.xml')
os.remove('code_ast.xml')
root = tree.getroot()
for i in range(0,len(self.oacc_copys)):
#print i
list=(self.oacc_copys[i]).split('\n')
self.oacc_copys[i]=''
for j in range(0,len(list)-1):
regex = re.compile(r'([a-zA-Z0-9_]*)([=])(\")([a-zA-Z0-9_:\(\)\*]*)(\")')
#print 'regex='+regex.findall(list[j])
for (a, b, c, d, e) in regex.findall(list[j]):
if a=='varname':
try:
# find the `d` variable in the region
varNameList=self.oacc_copysVarNams[i]
varTypeList=self.oacc_copysVarTyps[i]
#print varTypeList[varNameList.index(d)]
list[j]=list[j]+' type="'+varTypeList[varNameList.index(d)]+'"'
except:
print "fatal error! variable "+d+"is undefined!"
list[j]=list[j]+' dname="'+self.varmapper_getDeviceName_elseCreate(self.oacc_copysParent[i],d)+'"'
list[j]=list[j]+' size="'+self.var_find_size(d,self.oacc_copysParent[i],root)+'"'
list[j]=list[j]+' parentFunc="'+self.oacc_copysParent[i]+'"'
self.oacc_copys[i]='\n'.join(list[0:len(list)-1])
def main():
# preprocess Python.h and build the AST
python_h = preprocess_python_headers()
parser = c_parser.CParser()
ast = parser.parse(python_h)
# extract struct members from the AST
ast_parser = AstParser()
ast_parser.visit(ast)
writer = Writer()
# generate the C# code
gen_interop_head(writer)
gen_heap_type_members(ast_parser, writer)
slots_types = [
"PyNumberMethods",
"PySequenceMethods",
"PyMappingMethods",
"PyAsyncMethods",
"PyBufferProcs",
]
supported_types = []
indent = 1
for type_name in slots_types:
if not gen_structure_code(ast_parser, writer, type_name, indent):
continue
supported_types.append(type_name)
gen_supported_slot_record(writer, supported_types, indent)
gen_interop_tail(writer)
interop_cs = writer.to_string()
if len(sys.argv) > 1:
with open(sys.argv[1], "w") as fh:
fh.write(interop_cs)
else:
print(interop_cs)
def makeC(text, GP_result,line):
global result
global stack_to_ast
line=str(line)
parser = c_parser.CParser()
ast = parser.parse(text)
ast_dict=to_dict(ast)
stack_to_ast={}
make_dict(GP_result)
pos=find_line(ast_dict,line)
#print(pos)
#position=find_pos(pos,ast_dict,stack_to_ast)
#print(position)
#print(stack_to_ast)
#position=copy.deepcopy(stack_to_ast)
find_pos(pos,ast_dict,stack_to_ast)
generator = c_generator.CGenerator()
#print(ast_dict)
try:
ast=from_dict(ast_dict)
except:
return False
return generator.visit(ast)
def parse_jstruct(filename, include_paths=[], defines=[]):
parser = c_parser.CParser()
with open(filename, 'r') as infile:
text = infile.read()
define_map = {
'JSTRUCT_H': '<jstruct/jstruct.h>',
'JSON_OBJECT_H': '<json-c/json_object.h>',
'ARRAYLIST_H': '<json-c/arraylist.h>'
}
define_map.update({ds[0]: ds[1] for ds in (d.split('=') for d in defines)})
defines = ['{0}={1}'.format(*kv) for kv in define_map.iteritems()]
# insert some header includes and a 'do not modify'
text = re.sub(GUARD_HEADERS_EXPR,
r'\g<0>' + GENERATED + PREPEND_HEADERS(**define_map),
text,
count=1)
pptext, err = preprocess(text,
include_paths=include_paths,
defines=['__attribute__(x)='] + defines)
if err:
import os
rel_filename = os.path.relpath(filename)
err = err.replace('<stdin>', rel_filename)
print(repr(defines))
raise Exception('C Preprocessor: ' + err)
try:
ast = parser.parse(pptext, filename=filename)
return (ast, text)
except plyparser.ParseError as ex:
import os
rel_filename = os.path.relpath(filename)
message = ex.message.replace('<stdin>', rel_filename)
raise plyparser.ParseError(message)
def explain_c_declaration(c_decl,
adapter,
expand_struct=False,
expand_typedef=False):
""" Parses the declaration in c_decl and returns a text
explanation as a string.
The last external node of the string is used, to allow earlier typedefs
for used types.
expand_struct=True will spell out struct definitions recursively.
expand_typedef=True will expand typedef'd types.
"""
parser = c_parser.CParser()
try:
node = parser.parse(c_decl, filename='<stdin>')
except c_parser.ParseError:
e = sys.exc_info()[1]
return "Parse error:" + str(e)
if (not isinstance(node, c_ast.FileAST)
or not isinstance(node.ext[-1], c_ast.Decl)):
return "Not a valid declaration"
explanations = io.StringIO()
for child in node.ext:
try:
expanded = expand_element(child,
node,
expand_struct=expand_struct,
expand_typedef=expand_typedef)
explanation = adapter.explain(expanded)
explanations.write(explanation + "\n")
except Exception as e:
raise e
return "Not a valid declaration: " + str(e)
return explanations.getvalue()
def _parse_function_from_prototype(self, prototype):
''' Parse the function from the prototype '''
# Thanks to cdecl.py from pycparser
parser = c_parser.CParser()
try:
ast = parser.parse(prototype)
except c_parser.ParseError as exc:
raise MockError("Parse error: '{0}' with input: '{1}'".format(
str(exc), prototype))
decl = ast.ext[-1]
if not isinstance(decl, c_ast.Decl):
raise MockError("Not a valid declaration: " + prototype)
# decl.show(); print("")
if not isinstance(decl.type, c_ast.FuncDecl):
raise MockError("Not a function declaration: " + prototype)
# storage is, for example, "static" in "static void f();"
if decl.storage:
storage = ' '.join(decl.storage)
raise MockError("Cannot mock a function with storage: " + storage)
self._func_decl = decl.type
self.name = decl.name
def parse(code, extra_cpp_args=None, whitelist=None, debug=False, regex=None):
if extra_cpp_args is None:
extra_cpp_args = []
if regex is None:
regex = []
preprocessed = preprocess(code, extra_cpp_args=extra_cpp_args, debug=debug)
parser = c_parser.CParser()
for r in regex:
assert r[0] == "s" and r[-1] == "g" and r[1] == r[-2], 'Only search/replace is allowed: "s/.../.../g"'
delimiter = r[1]
assert r.count(delimiter) == 3, 'Malformed regex. Only search/replace is allowed: "s/.../.../g"'
_, search, replace, _ = r.split(delimiter)
preprocessed = re.sub(search, replace, preprocessed)
ast = parser.parse(preprocessed)
decls = []
for decl in ast.ext:
if not hasattr(decl, "name") or decl.name not in IGNORE_DECLARATIONS:
if not whitelist or decl.coord.file in whitelist:
decls.append(decl)
ast.ext = decls
return ast
def extract_enums(toc, source):
"Yield all enum definitions belonging to the given header."
typedefs = []
in_typedef = False
typedef = []
for line in source.splitlines():
if line and not line.startswith('#'):
if in_typedef:
typedef.append(line)
if line.startswith('}'):
in_typedef = False
typedefs.append(typedef)
typedef = []
elif line.startswith('typedef enum'):
in_typedef = True
typedef.append(line)
parser = c_parser.CParser()
for typedef in typedefs:
td = parser.parse(''.join(typedef))
if td.ext[0].name in toc:
yield td
def cm_test(filename):
print("Code Motion Test")
with open(filename, "r") as source:
# reading the c file
parser = c_parser.CParser()
astc = parser.parse(source.read())
# convert to python
pysrc = CtoPy(astc)
tree = ast.parse(pysrc.getPy())
analyze = Analyze(tree)
xform = Transform(analyze)
# Input program
print("\nInput Program")
print(xform.codegen())
# Dimensions
in_dim = xform.analyze.dims
out_dim = in_dim
# Type of dimensions
in_dim_type = xform.analyze.getdimtype()
# Input alphabet and order
in_alp = xform.analyze.getalp()
in_ord = xform.analyze.getord()
# Output order
out_ord = [['e', 't1', 'r1'], ['e', 's1', 'r2l', 'r2r']]
# Transform
xf = Transformation(name='cm',
in_dim=in_dim,
out_dim=out_dim,
in_dim_type=in_dim_type,
in_alp=in_alp,
in_ord=in_ord,
out_ord=out_ord)
xform.transform(xf)
# Output program
print("\nOutput Program")
print(xform.codegen())
#
# A dummy for generating the lexing/parsing tables and and
# compiling them into .pyc for faster execution in optimized mode.
# Also generates AST code from the configuration file.
# Should be called from the pycparser directory.
#
# Eli Bendersky [https://eli.thegreenplace.net/]
# License: BSD
# -----------------------------------------------------------------
# Generate c_ast.py
from _ast_gen import ASTCodeGenerator
ast_gen = ASTCodeGenerator('_c_ast.cfg')
ast_gen.generate(open('c_ast.py', 'w'))
import sys
sys.path[0:0] = ['.', '..']
from pycparser import c_parser
# Generates the tables
#
c_parser.CParser(lex_optimize=True, yacc_debug=False, yacc_optimize=True)
# Load to compile into .pyc
#
import lextab
import yacctab
import c_ast
def grad_with_split(ast, expression, variables, func = 'function',
reverse_diff = False, second_der = False, output_filename = 'c_code',
output_func = 'compute', split_by = 20, parallel = False, num_threads = 1,
differentiate_pow_exp = True):
"""
Returns a function which computes gradient of `fun` with respect to
positional argument number `x`. The returned function takes the
same arguments as `fun` , but returns the gradient instead. The function
`fun` is expected to be scalar valued. The gradient has the same type as argument."""
global ext_index
fun = None
for funs in range(len(ast.ext)):
if 'decl' not in dir(ast.ext[funs]):
continue
fun_name = ast.ext[funs].decl.name
if fun_name == func:
ext_index = funs
break
assert fun_name == func
der_vars = ast.ext[ext_index].decl.type.args.params
global curr_base_variable
if(reverse_diff and second_der):
print('Computing Hessian with Reverse Differentiation')
c_code = c_generator.CGenerator(filename = output_filename+'0', variable_count = len(variables), derivative_count = len(variables)*len(variables), split=True, parallel = parallel, num_threads = num_threads)
elif (not reverse_diff and second_der):
print('Computing Hessian with Forward Differentiation')
c_code = c_generator.CGenerator(filename = output_filename+'0', variable_count = len(variables), derivative_count = (len(variables)*(len(variables))), split=True, parallel = parallel, num_threads = num_threads)
else:
if reverse_diff:
print('Computing Gradient with Reverse Differentiation')
else:
print('Computing Gradient with Forward Differentiation')
c_code = c_generator.CGenerator(filename = output_filename+'0', variable_count = len(variables), derivative_count = len(variables), split=True, parallel = parallel, num_threads = num_threads)
file_pointer = open(output_filename+'0.c','a')
file_pointer = c_code._make_header(output_func, file_pointer)
dict_ = {}
# looks for an assignment
for blocks in range(len(ast.ext[ext_index].body.block_items)):
if ast.ext[ext_index].body.block_items[blocks].__class__.__name__ == 'Decl':
if ast.ext[ext_index].body.block_items[blocks].type.__class__.__name__ == 'ArrayDecl':
expr_name = ""
arr = ast.ext[ext_index].body.block_items[blocks].type
while arr.__class__.__name__!="TypeDecl":
expr_name += "[{}]".format(arr.dim.value)
arr = arr.type
expr_name = arr.declname+expr_name
else:
expr_name = ast.ext[ext_index].body.block_items[blocks].name
if expr_name in dict_.keys():
substite_in_fun = ast.ext[ext_index].body.block_items[blocks].init
dict_[expr_name] = expand_equation(substite_in_fun, dict_)
else:
dict_[expr_name] = expand_equation(ast.ext[ext_index].body.block_items[blocks].init, dict_)
if expr_name != expression:
continue
fun = dict_[expr_name]
if ast.ext[ext_index].body.block_items[blocks].__class__.__name__ == 'Assignment':
# dealing with array names
if ast.ext[ext_index].body.block_items[blocks].lvalue.__class__.__name__ == 'ArrayRef':
expr_name = ""
arr = ast.ext[ext_index].body.block_items[blocks].lvalue
while arr.__class__.__name__!="ID":
expr_name += "[{}]".format(arr.subscript.value)
arr = arr.name
expr_name = arr.name+expr_name
# scalar variables
else:
expr_name = ast.ext[ext_index].body.block_items[blocks].lvalue.name
# has the variable been encountered before
if expr_name in dict_.keys():
substite_in_fun = ast.ext[ext_index].body.block_items[blocks].rvalue
dict_[expr_name] = expand_equation(substite_in_fun, dict_)
else:
dict_[expr_name] = expand_equation(ast.ext[ext_index].body.block_items[blocks].rvalue, dict_)
if expr_name != expression:
continue
fun = dict_[expr_name]
assert fun != None
job_finished = False
if not reverse_diff:
if second_der:
grad = {}
for i, vars_ in enumerate(variables):
grad[vars_] = ''
ctr=0
split_ctr = 0
# matrix size: i x j
# matrix size flattened : i*j values
# produce sub arrays after splitting
for i,vars_ in enumerate(variables):
curr_base_variable = Variable(vars_)
primary_base_variable = Variable(vars_)
derivative = Expr(fun,differentiate_pow_exp=differentiate_pow_exp)._forward_diff()
derivative = simplify_equation(derivative)
# produce new ast of derivative
new_parser = c_parser.CParser()
new_ast = new_parser.parse("double f = {};".format(derivative), filename='<none>')
# compute upper triangular (with diagonal)
for j in range(i, len(variables)):
vars_second = variables[j]
curr_base_variable = Variable(vars_second)
secondary_base_variable = Variable(vars_second)
second_derivative = Expr(new_ast.ext[0].init,differentiate_pow_exp=differentiate_pow_exp)._forward_diff()
print("[{},{} / {}] Second derivative : df / d{} d{}:".format(i,j,len(variables),vars_, vars_second))
flattened_mat_idx = i*len(variables) + j
flattened_mat_idx_mirror = i + j*len(variables)
# file_pointer returned to ensure file finishes writing this block
file_pointer = c_code._generate_expr([primary_base_variable._get(), secondary_base_variable._get()], second_derivative,index = split_ctr, mirrored_index = None, file_pointer = file_pointer)
ctr+=1
split_ctr += 1
if ctr % split_by==0:
tmp = int(ctr//split_by)
print("Splitting file . Producing file ",tmp)
c_code._make_footer(file_pointer)
if not (i==(len(variables)-1) and j==(len(variables)-1)):
c_code = c_generator.CGenerator(filename = output_filename+str(tmp), variable_count = len(variables),
derivative_count = (len(variables)*(len(variables))), split=True, parallel = parallel, num_threads = num_threads)
file_pointer = open(output_filename+str(tmp)+'.c','a')
file_pointer = c_code._make_header(output_func, file_pointer)
job_finished = True
split_ctr = 0
else:
ctr=0
split_ctr = 0
for i,vars_ in enumerate(variables):
ctr += 1
split_ctr += 1
curr_base_variable = Variable(vars_)
derivative = Expr(fun,differentiate_pow_exp=differentiate_pow_exp)._forward_diff()
c_code._generate_expr(curr_base_variable._get(), derivative, index=i, file_pointer = file_pointer)
if not job_finished:
c_code._make_footer(file_pointer)
elif reverse_diff:
grad = {}
for i, vars_ in enumerate(variables):
# c_code._declare_vars(vars_,i)
grad[vars_] = '0'
if second_der:
ctr=0
split_ctr = 0
Expr(fun,differentiate_pow_exp=differentiate_pow_exp)._reverse_diff("1.",grad)
for i,vars_ in enumerate(variables):
primary_base_variable = Variable(vars_)
k = vars_
v = grad[vars_]
derivative = simplify_equation(v)
new_parser = c_parser.CParser()
new_ast = new_parser.parse("double f = {};".format(derivative), filename='<none>')
grad_hess = {}
for i_ctr, vars_ in enumerate(variables):
grad_hess[vars_] = '0'
Expr(new_ast.ext[0].init,differentiate_pow_exp=differentiate_pow_exp)._reverse_diff("1.",grad_hess)
for j in range(i, len(variables)):
k_hess = variables[j]
v_hess = grad_hess[k_hess]
secondary_base_variable = Variable(k_hess)
print("[{},{} / {}] Second derivative : df / d{} d{}:".format(i,j,len(variables),k, k_hess))
file_pointer = c_code._generate_expr([primary_base_variable._get(), secondary_base_variable._get()], v_hess, index = split_ctr, mirrored_index = None, file_pointer = file_pointer)
ctr+=1
split_ctr += 1
if ctr % split_by==0:
tmp = int(ctr//split_by)
print("Splitting file . Producing file ",tmp)
c_code._make_footer(file_pointer)
if not (i==(len(variables)-1) and j==(len(variables)-1)):
c_code = c_generator.CGenerator(filename = output_filename+str(tmp), variable_count = len(variables),
derivative_count = (len(variables)*(len(variables))), split=True, parallel = parallel, num_threads = num_threads)
file_pointer = open(output_filename+str(tmp)+'.c','a')
file_pointer = c_code._make_header(output_func, file_pointer)
job_finished = True
split_ctr = 0
else:
ctr=0
split_ctr = 0
Expr(fun,differentiate_pow_exp=differentiate_pow_exp)._reverse_diff("1.",grad)
for i,vars_ in enumerate(variables):
ctr += 1
split_ctr += 1
curr_base_variable = Variable(vars_)
v_grad = grad[vars_]
file_pointer = c_code._generate_expr([primary_base_variable._get()], v_grad, index = split_ctr, mirrored_index = None, file_pointer = file_pointer)
if not job_finished:
c_code._make_footer(file_pointer)
def grad(ast, expression, variables, func = 'function',
reverse_diff = False, second_der = False, output_filename = 'c_code',
output_func = 'compute', parallel = False, num_threads = 1,
differentiate_pow_exp = True):
"""
Returns a function which computes gradient of `fun` with respect to
positional argument number `x`. The returned function takes the
same arguments as `fun` , but returns the gradient instead. The function
`fun` is expected to be scalar valued. The gradient has the same type as argument."""
global ext_index
fun = None
for funs in range(len(ast.ext)):
if 'decl' not in dir(ast.ext[funs]):
continue
fun_name = ast.ext[funs].decl.name
if fun_name == func:
ext_index = funs
break
assert fun_name == func
der_vars = ast.ext[ext_index].decl.type.args.params
global curr_base_variable
if(reverse_diff and second_der):
print('Computing Hessian with Reverse Differentiation')
c_code = c_generator.CGenerator(filename = output_filename, variable_count = len(variables), derivative_count = len(variables)*len(variables), parallel = parallel, num_threads = num_threads)
elif (not reverse_diff and second_der):
print('Computing Hessian with Forward Differentiation')
c_code = c_generator.CGenerator(filename = output_filename, variable_count = len(variables), derivative_count = (len(variables)*(len(variables))), parallel = parallel, num_threads = num_threads)
else:
if reverse_diff:
print('Computing Gradient with Reverse Differentiation')
else:
print('Computing Gradient with Forward Differentiation')
c_code = c_generator.CGenerator(filename = output_filename, variable_count = len(variables), derivative_count = len(variables), parallel = parallel, num_threads = num_threads)
file_pointer = open(output_filename+'.c','a')
file_pointer = c_code._make_header(output_func, file_pointer)
dict_ = {}
# looks for an assignment
for blocks in range(len(ast.ext[ext_index].body.block_items)):
if ast.ext[ext_index].body.block_items[blocks].__class__.__name__ == 'Decl':
if ast.ext[ext_index].body.block_items[blocks].type.__class__.__name__ == 'ArrayDecl':
expr_name = ""
arr = ast.ext[ext_index].body.block_items[blocks].type
while arr.__class__.__name__!="TypeDecl":
expr_name += "[{}]".format(arr.dim.value)
arr = arr.type
expr_name = arr.declname+expr_name
else:
expr_name = ast.ext[ext_index].body.block_items[blocks].name
if expr_name in dict_.keys():
substite_in_fun = ast.ext[ext_index].body.block_items[blocks].init
dict_[expr_name] = expand_equation(substite_in_fun, dict_)
else:
dict_[expr_name] = expand_equation(ast.ext[ext_index].body.block_items[blocks].init, dict_)
if expr_name != expression:
continue
fun = dict_[expr_name]
if ast.ext[ext_index].body.block_items[blocks].__class__.__name__ == 'Assignment':
# dealing with array names
if ast.ext[ext_index].body.block_items[blocks].lvalue.__class__.__name__ == 'ArrayRef':
expr_name = ""
arr = ast.ext[ext_index].body.block_items[blocks].lvalue
while arr.__class__.__name__!="ID":
expr_name += "[{}]".format(arr.subscript.value)
arr = arr.name
expr_name = arr.name+expr_name
# scalar variables
else:
expr_name = ast.ext[ext_index].body.block_items[blocks].lvalue.name
# has the variable been encountered before
if expr_name in dict_.keys():
substite_in_fun = ast.ext[ext_index].body.block_items[blocks].rvalue
dict_[expr_name] = expand_equation(substite_in_fun, dict_)
else:
dict_[expr_name] = expand_equation(ast.ext[ext_index].body.block_items[blocks].rvalue, dict_)
if expr_name != expression:
continue
fun = dict_[expr_name]
assert fun != None
grad = {}
for i, vars_ in enumerate(variables):
file_pointer = c_code._declare_vars(vars_,i,file_pointer=file_pointer)
grad[vars_] = '0'
if reverse_diff:
if second_der:
Expr(fun,differentiate_pow_exp=differentiate_pow_exp)._reverse_diff("1.",grad)
ctr=0
for i, vars_ in enumerate(variables):
primary_base_variable = Variable(vars_)
k = vars_
v = grad[vars_]
simplified = simplify_equation(v)
new_parser = c_parser.CParser()
new_ast = new_parser.parse("double f = {};".format(simplified), filename='<none>')
grad_hess = {}
for i_ctr, vars_ in enumerate(variables):
grad_hess[vars_] = '0'
Expr(new_ast.ext[0].init,differentiate_pow_exp=differentiate_pow_exp)._reverse_diff("1.",grad_hess)
for j in range(i, len(variables)):
k_hess = variables[j]
v_hess = grad_hess[k_hess]
secondary_base_variable = Variable(k_hess)
file_pointer = c_code._generate_expr([primary_base_variable._get(), secondary_base_variable._get()], v_hess,index=ctr, file_pointer=file_pointer)
ctr+=1
else:
Expr(fun,differentiate_pow_exp=differentiate_pow_exp)._reverse_diff("1.",grad)
i = 0
for k,v in grad.items():
file_pointer=c_code._generate_expr(k, v,index=i, file_pointer=file_pointer)
i += 1
# forward hessian
elif second_der:
ctr=0
dictionary = {}
for i,vars_ in enumerate(variables):
curr_base_variable = Variable(vars_)
primary_base_variable = Variable(vars_)
derivative = Expr(fun)._forward_diff()
derivative = simplify_equation(derivative)
new_parser = c_parser.CParser()
new_ast = new_parser.parse("double f = {};".format(derivative), filename='<none>')
for j in range(i, len(variables)):
vars_second = variables[j]
curr_base_variable = Variable(vars_second)
secondary_base_variable = Variable(vars_second)
second_derivative = Expr(new_ast.ext[0].init,differentiate_pow_exp=differentiate_pow_exp)._forward_diff()
c_code._generate_expr([primary_base_variable._get(), secondary_base_variable._get()], second_derivative,index=ctr, file_pointer=file_pointer)
string = str(i)+','+str(j)
dictionary[string] = ctr
ctr+=1
pointer_index = 1
for i,vars_ in enumerate(variables):
curr_base_variable = Variable(vars_)
primary_base_variable = Variable(vars_)
for j in range(0,i):
vars_second = variables[j]
curr_base_variable = Variable(vars_second)
secondary_base_variable = Variable(vars_second)
string = str(j)+','+str(i)
pointer_index = dictionary[string]
c_code._generate_copy([primary_base_variable._get(), secondary_base_variable._get()], pointer_index=pointer_index,index=ctr,file_pointer=file_pointer)
ctr += 1
else:
for i,vars_ in enumerate(variables):
curr_base_variable = Variable(vars_)
derivative = Expr(fun,differentiate_pow_exp=differentiate_pow_exp)._forward_diff()
c_code._generate_expr(curr_base_variable._get(), derivative,index=i,file_pointer=file_pointer)
c_code._make_footer(file_pointer)
def parseCode(df):
parser = c_parser.CParser()
ast = parser.parse(df.iloc[0]["code"])
def test_node_finder():
ast = c_parser.CParser().parse(STRUCT, filename='<none>')
matches = NodeFinder(c_parser.c_ast.IdentifierType).find(ast)
assert len(matches) > 0
def cm_sm_ic_test(filename):
print("CM-SM-IC Test")
with open(filename, "r") as source:
# reading the c file
parser = c_parser.CParser()
astc = parser.parse(source.read())
# convert to python
pysrc = CtoPy(astc)
tree = ast.parse(pysrc.getPy())
analyze = Analyze(tree)
xform = Transform(analyze)
# Input program
print("\nInput Program")
print(xform.codegen())
# code-motion
# Dimensions
in_dim = xform.analyze.dims
out_dim = in_dim
# Type of dimensions
in_dim_type = xform.analyze.getdimtype()
# Input alphabet and order
in_alp = xform.analyze.getalp()
in_ord = xform.analyze.getord()
# Output order
out_ord = [['e', 't1', 'r1'], ['e', 's1', 'r2l', 'r2r']]
xf1 = Transformation(name='cm',
in_dim=in_dim,
out_dim=out_dim,
in_dim_type=in_dim_type,
in_alp=in_alp,
in_ord=in_ord,
out_ord=out_ord)
xform.transform(xf1)
# strip mining
# strip dimension
strip_dim = 0
# strip size
strip_size = 2
# Transform
xf2 = Transformation(name='sm',
in_dim=xf1.out_dim,
out_dim=xf1.out_dim + 1,
in_dim_type=xf1.in_dim_type,
in_alp=xf1.out_alp,
in_ord=xf1.out_ord,
dim_strip=strip_dim,
strip_size=strip_size)
xform.transform(xf2)
# interchange
# dimensions
dim1 = 1
dim2 = 2
xf3 = Transformation(name='ic',
in_dim=xf2.out_dim,
out_dim=xf2.out_dim,
in_dim_type=xf2.out_dim_type,
in_alp=xf2.out_alp,
in_ord=xf2.out_ord,
dim_i1=dim1,
dim_i2=dim2)
xform.transform(xf3)
# Output program
print("\nOutput Program")
print(xform.codegen())
def show_id_type(code, typ):
parser = c_parser.CParser()
ast = parser.parse(code)
v = IdentifierTypeVisitor(typ)
v.visit(ast)
from main_statistic import AST_Statistics
problem = 'SUBINC' # FLOW016, MNMX, SUBINC, SUMTRIAN
datapath = '/home/s1520015/Experiment/CodeChef/' #params.datapath # 'Z:/Experiment/CodeChef/'
astjson = '_AST.json'
graphjson = '_AstGraph.json'
codejson = '_Code.json'
randomseed = 314159
labels_dict = {}
labels_dict['tick-icon.gif'] = 0 # correct answer
labels_dict['cross-icon.gif'] = 1 # wrong answer
labels_dict['alert-icon.gif'] = 2 #
labels_dict['clock_error.png'] = 3 #
labels_dict['runtime-error.png'] = 4 #
parser = c_parser.CParser()
def getSourceCode(loc): # lines of code
# preprovessing: remove comments before parsing into AST
code = ''
m_comment = []
for line in loc:
content = line.strip()
idx = line.find('/*')
if idx >= 0:
m_comment.append(1)
cmd = line[:idx]
line = line[idx + 2:]
code += '\n' + cmd
idx = line.find('*/')
def test_resolver():
ast = c_parser.CParser().parse(TYPEDEF_DECL, filename='<none>')
res = TypeResolver(ast)
t = res.resolve_type(ast.ext[-1].type)
assert t.type.names == ['unsigned', 'int']
def parse_files():
parser = c_parser.CParser()
cpp_path = get_latest_cpp()
for filename in FILES:
filepath = os.path.join(INCLUDE_DIR, filename)
dummy_typedefs = {}
text = preprocess_file(filepath, cpp_path)
if filepath.endswith("GL.h"):
dummy_typedefs = {
"CUresult": "int",
"CUgraphicsResource": "void *",
"CUdevice": "void *",
"CUcontext": "void *",
"CUdeviceptr": "void *",
"CUstream": "void *"
}
text = "typedef int GLint;\n" + text
text = "typedef unsigned int GLuint;\n" + text
text = "typedef unsigned int GLenum;\n" + text
text = "typedef long size_t;\n" + text
for typedef in sorted(dummy_typedefs):
text = "typedef " + dummy_typedefs[typedef] + " " + \
typedef + ";\n" + text
ast = parser.parse(text, filepath)
with open(filepath) as f:
lines = f.readlines()
for line in lines:
if line.startswith("#define"):
line = line[8:-1]
token = line.split()
if token[0] not in ("__cuda_cuda_h__",
"CUDA_CB",
"CUDAAPI",
"CUDAGL_H",
"__NVRTC_H__",
"CUDA_ENABLE_DEPRECATED",
"__CUDA_DEPRECATED"):
DEFINES.append(token)
for line in lines:
# TODO(sergey): Use better matching rule for _v2 symbols.
if line[0].isspace() and line.lstrip().startswith("#define"):
line = line[12:-1]
token = line.split()
if len(token) == 2 and (token[1].endswith("_v2") or
token[1].endswith("_v2)")):
if token[1].startswith('__CUDA_API_PTDS') or \
token[1].startswith('__CUDA_API_PTSZ'):
token[1] = token[1][16:-1]
DEFINES_V2.append(token)
v = FuncDefVisitor()
for typedef in dummy_typedefs:
v.dummy_typedefs.append(typedef)
v.visit(ast)
FUNC_TYPEDEFS.append('')
SYMBOLS.append('')
def dump_ast(filename):
with open(filename) as c_file:
c_text = c_file.read()
parser = c_parser.CParser()
ast = parser.parse(c_text, filename=filename)
print(dumps(to_dict(ast), sort_keys=False, indent=2))
def main(fname, unique, F):
global extern_funcs, externs
global advapi_funcs, kernel32_funcs, user32_funcs, msvcrt_funcs
parser = c_parser.CParser()
text = open(fname, 'r').read()
L = []
for i in text:
if i == '\r': L.append('')
else: L.append(i)
text = ''.join(L)
ast = parser.parse(text, filename=fname, debuglevel=0)
#ast.show()
p1 = Printer1()
p1.visit(ast)
if unique:
unique = binascii.a2b_hex(unique)
else:
while True:
unique = os.urandom(32)
if F and not F.has_key(unique):
F[unique] = time.time()
break
else:
break
mt = Mutagen(unique)
def EfLrandomize(L, mt):
R = list(L[0])
for i in L[1:]:
q = mt.SeqBit()
if q:
R.extend(i)
return R
mt.ResetBits()
advapi_funcs = EfLrandomize(advapi_funcs, mt)
kernel32_funcs = EfLrandomize(kernel32_funcs, mt)
user32_funcs = EfLrandomize(user32_funcs, mt)
msvcrt_funcs = EfLrandomize(msvcrt_funcs, mt)
mt.ResetBits()
extern_funcs = [
(None, msvcrt_funcs),
('__stdcall', advapi_funcs),
('__stdcall', kernel32_funcs),
('__stdcall', user32_funcs),
]
externs = reduce(lambda x, y: x + y, map(lambda z: z[1], extern_funcs))
ph = Polymorph(mt)
p2 = Printer2(ph)
p2.visit(ast)
funcs = [j for i, j in functions.items()]
fakes = ph.GetFakes()
ph.PrintGlobals()
DeclareExternFuncs(mt, funcs)
mt.RandomList(funcs)
mt.RandomList(fakes)
while funcs or fakes:
q = mt.SeqBit()
f = None
if (q and funcs) or (not q and not fakes):
f = funcs[0]
funcs = funcs[1:]
elif (q and not funcs) or (not q and fakes):
f = fakes[0]
fakes = fakes[1:]
f.PrintFunc()
import sys
import textwrap
import unittest
# Run from the root dir
sys.path.insert(0, '.')
from pycparser import c_parser, c_generator, c_ast
_c_parser = c_parser.CParser(lex_optimize=False,
yacc_debug=True,
yacc_optimize=False,
yacctab='yacctab')
def compare_asts(ast1, ast2):
if type(ast1) != type(ast2):
return False
if isinstance(ast1, tuple) and isinstance(ast2, tuple):
if ast1[0] != ast2[0]:
return False
ast1 = ast1[1]
ast2 = ast2[1]
return compare_asts(ast1, ast2)
for attr in ast1.attr_names:
if getattr(ast1, attr) != getattr(ast2, attr):
return False
for i, c1 in enumerate(ast1.children()):
if compare_asts(c1, ast2.children()[i]) == False:
return False
return True
def grad_without_traversal(ast, x=0):
"""
Returns a function which computes gradient of `fun` with respect to
positional argument number `x`. The returned function takes the
same arguments as `fun` , but returns the gradient instead. The function
`fun` is expected to be scalar valued. The gradient has the same type as argument."""
assert type(x) in (int, tuple, list), x
global ext_index
fun = None
for funs in range(len(ast.ext)):
if 'decl' not in dir(ast.ext[funs]):
continue
fun_name = ast.ext[funs].decl.name
if fun_name == parser.func:
ext_index = funs
break
assert fun_name == parser.func
der_vars = ast.ext[ext_index].decl.type.args.params
global curr_base_variable
if(reverse_diff and second_der):
c_code = c_generator.CGenerator(filename = output_filename, variable_count = len(variables), derivative_count = len(variables)*len(variables), c_code = ccode, ispc = ispc)
elif (not reverse_diff and second_der):
c_code = c_generator.CGenerator(filename = output_filename, variable_count = len(variables), derivative_count = (len(variables)*(len(variables))), c_code = ccode, ispc = ispc)
else:
c_code = c_generator.CGenerator(filename = output_filename, variable_count = len(variables), derivative_count = len(variables), c_code = ccode, ispc = ispc)
c_code._make_header()
# for vars_ in der_vars:
# c_code._make_decls(vars_.name)
# look for function to differentiate.
dict_ = {}
# fix the expanding eqn here
for blocks in range(len(ast.ext[ext_index].body.block_items)):
if 'name' not in dir(ast.ext[ext_index].body.block_items[blocks]):
continue
if ast.ext[ext_index].body.block_items[blocks].type.__class__.__name__ == 'ArrayDecl':
if ast.ext[ext_index].body.block_items[blocks].type.type.__class__.__name__ == 'ArrayDecl':
expr_name = ast.ext[ext_index].body.block_items[blocks].name+'[{}][{}]'.format(ast.ext[ext_index].body.block_items[blocks].type.dim.value,ast.ext[ext_index].body.block_items[blocks].type.type.dim.value)
else:
expr_name = ast.ext[ext_index].body.block_items[blocks].name+'[{}]'.format(ast.ext[ext_index].body.block_items[blocks].type.dim.value)
else:
expr_name = ast.ext[ext_index].body.block_items[blocks].name
if expr_name != expression:
dict_[expr_name] = ast.ext[ext_index].body.block_items[blocks].init
continue
fun = ast.ext[ext_index].body.block_items[blocks].init
assert fun != None
# fun.show()
# print("dictionary: ")
# print(dict_)
# print("==============================================")
# print("Function: ")
# fun.show()
fun = expand_equation(fun, dict_)
# print("Expanded equation:")
# fun.show()
grad = {}
for i, vars_ in enumerate(variables):
c_code._declare_vars(vars_,i)
grad[vars_] = ''
# print(grad)
if reverse_diff:
if second_der:
Expr(fun)._reverse_diff("1.",grad)
ctr=0
for i, vars_ in enumerate(variables):
primary_base_variable = Variable(vars_)
k = vars_
v = grad[vars_]
# print("First derivative: ")
# print(vars_)
# print(v)
simplified = simplify_equation(v)
# print("Simplified equation: ")
# print(simplified)
new_parser = c_parser.CParser()
new_ast = new_parser.parse("double f = {};".format(simplified), filename='<none>')
grad_hess = {}
for i_ctr, vars_ in enumerate(variables):
grad_hess[vars_] = ''
Expr(new_ast.ext[0].init)._reverse_diff("1.",grad_hess)
# print("Second Derivative: ")
for j in range(i, len(variables)):
k_hess = variables[j]
v_hess = grad_hess[k_hess]
secondary_base_variable = Variable(k_hess)
# print("Second derivative : df / d{} d{}:".format(k, k_hess))
# print(k_hess)
# print(v_hess)
c_code._generate_expr([primary_base_variable._get(), secondary_base_variable._get()], v_hess,index=ctr)
ctr+=1
else:
Expr(fun)._reverse_diff("1.",grad)
# print(grad)
i = 0
for k,v in grad.items():
c_code._generate_expr(k, v,index=i)
i += 1
elif second_der:
ctr=0
dictionary = {}
for i,vars_ in enumerate(variables):
curr_base_variable = Variable(vars_)
# print("current base var: ",curr_base_variable. _get())
primary_base_variable = Variable(vars_)
derivative = Expr(fun)._forward_diff()
# print("First derivative: ")
# print(derivative)
derivative = simplify_equation(derivative)
new_parser = c_parser.CParser()
new_ast = new_parser.parse("double f = {};".format(derivative), filename='<none>')
for j in range(i, len(variables)):
vars_second = variables[j]
curr_base_variable = Variable(vars_second)
secondary_base_variable = Variable(vars_second)
second_derivative = Expr(new_ast.ext[0].init)._forward_diff()
# print("Second derivative : df / d{} d{}:".format(vars_, vars_second))
# print(second_derivative)
c_code._generate_expr([primary_base_variable._get(), secondary_base_variable._get()], second_derivative,index=ctr)
string = str(i)+','+str(j)
dictionary[string] = ctr
ctr+=1
pointer_index = 1
for i,vars_ in enumerate(variables):
curr_base_variable = Variable(vars_)
# print("current base var: ",curr_base_variable. _get())
primary_base_variable = Variable(vars_)
for j in range(0,i):
vars_second = variables[j]
curr_base_variable = Variable(vars_second)
secondary_base_variable = Variable(vars_second)
string = str(j)+','+str(i)
pointer_index = dictionary[string]
c_code._generate_copy([primary_base_variable._get(), secondary_base_variable._get()], pointer_index=pointer_index,index=ctr)
ctr += 1
else:
for i,vars_ in enumerate(variables):
curr_base_variable = Variable(vars_)
derivative = Expr(fun)._forward_diff()
# print(derivative)
c_code._generate_expr(curr_base_variable._get(), derivative,index=i)
c_code._make_footer()
