def pmt_main(script_path, stdout, stderr, args=None):
parser = argparse.ArgumentParser()
parser.add_argument("program_name", \
help="name of the prometeo script to be executed")
parser.add_argument("--cgen", type=str2bool, default="False", \
help="generate, compile and execute C code?")
parser.add_argument("--out", type=str, default=None, \
help="redirect output to file")
parser.add_argument("--debug", type=str, default=False, \
help="call raise instead of exit() upon Exception")
args = parser.parse_args()
filename = args.program_name
cgen = args.cgen
red_stdout = args.out
debug = args.debug
if cgen is False:
post = '''main()'''
# code = open(filename).read() + post
# exec(code, globals(), globals())
code_no_hints = strip_file_to_string(filename) + post
exec(code_no_hints, globals(), globals())
else:
pmt_path = os.path.dirname(prometeo.__file__)
# cmd = 'export MYPYPATH=' + pmt_path + ' & mypy ' + filename
# os.system(cmd)
filename_ = filename.split('.')[0]
tree = ast.parse(''.join(open(filename)))
# tree will be slightly modified by
# to source(). Store a copy, for heap usage
# analysis:
tree_copy = deepcopy(tree)
# astpretty.pprint(tree)
# import pdb; pdb.set_trace()
# result = prometeo.cgen.code_gen_c.to_source(tree, filename_, \
# main=True, ___c_pmt_8_heap_size=10000, \
# ___c_pmt_64_heap_size=1000000, \
# size_of_pointer = size_of_pointer, \
# size_of_int = size_of_int, \
# size_of_double = size_of_double)
try:
result = prometeo.cgen.code_gen_c.to_source(tree, filename_, \
main=True, ___c_pmt_8_heap_size=10000, \
___c_pmt_64_heap_size=1000000, \
size_of_pointer = size_of_pointer, \
size_of_int = size_of_int, \
size_of_double = size_of_double)
except prometeo.cgen.code_gen_c.cgenException as e:
print('\n > Exception -- prometeo code-gen: ', e.message)
code = ''.join(open(filename))
print(' > @ line {}:'.format(e.lineno) + '\033[34m' +
code.splitlines()[e.lineno - 1] + '\033[0;0m')
print(' > Exiting.\n')
if debug:
raise
else:
exit()
dest_file = open(filename_ + '.c', 'w')
dest_file.write(prometeo.cgen.source_repr.pretty_source(result.source))
dest_file.close()
dest_file = open(filename_ + '.h', 'w')
dest_file.write(prometeo.cgen.source_repr.pretty_source(result.header))
dest_file.close()
# generate Makefile
makefile_code = makefile_template.replace('{{ filename }}', filename_)
# execname_ = re.sub('\.c$', '', filename_)
# makefile_code = makefile_template.replace('{{ execname }}', execname_)
makefile_code = makefile_code.replace('\n', '', 1)
makefile_code = makefile_code.replace(
'@INSTALL_DIR@',
os.path.dirname(__file__) + '/..')
dest_file = open('Makefile', 'w+')
dest_file.write(makefile_code)
dest_file.close()
# compute heap usage
visitor = ast_visitor()
# import pdb; pdb.set_trace()
visitor.visit(tree_copy)
call_graph = visitor.callees
typed_record = visitor.typed_record
# print('\ncall graph:\n\n', call_graph, '\n\n')
reach_map = compute_reach_graph(call_graph, typed_record)
# print('reach_map:\n\n', reach_map, '\n\n')
# check that there are no cycles containing memory allocations
for method in reach_map:
if '*' in reach_map[method] and typed_record[method] != dict():
raise Exception('\n\nDetected cycle {} containing memory'
' allocation.\n'.format(reach_map[method]))
proc = subprocess.Popen(["make", "clean"], stdout=subprocess.PIPE)
try:
outs, errs = proc.communicate(timeout=20)
except TimeOutExpired:
proc.kill()
outs, errs = proc.communicate()
print('Command \'make\' timed out.')
if proc.returncode:
raise Exception('Command \'make\' failed with the above error.'
' Full command is:\n\n {}'.format(outs.decode()))
proc = subprocess.Popen(["make"], stdout=subprocess.PIPE)
try:
outs, errs = proc.communicate(timeout=20)
except TimeOutExpired:
proc.kill()
outs, errs = proc.communicate()
print('Command \'make\' timed out.')
if proc.returncode:
raise Exception('Command \'make\' failed with the above error.'
' Full command is:\n\n {}'.format(outs.decode()))
INSTALL_DIR = os.path.dirname(__file__) + '/..'
running_on = platform.system()
if running_on == 'Linux':
cmd = 'export LD_LIBRARY_PATH=' + INSTALL_DIR + '/lib/prometeo:' + \
INSTALL_DIR + '/lib/blasfeo:$LD_LIBRARY_PATH ; ./' + filename_
elif running_on == 'Darwin':
cmd = 'export DYLD_LIBRARY_PATH=' + INSTALL_DIR + '/lib/prometeo:' + \
INSTALL_DIR + '/lib/blasfeo:$DYLD_LIBRARY_PATH ; ./' + filename_
else:
raise Exception(
'Running on unsupported operating system {}'.format(
running_on))
if red_stdout is not None:
proc = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE)
else:
proc = subprocess.Popen([cmd], shell=True)
try:
outs, errs = proc.communicate()
except TimeOutExpired:
proc.kill()
outs, errs = proc.communicate()
print('Command \'make\' timed out.')
if red_stdout is not None:
with open(red_stdout, 'w') as f:
f.write(outs)
if red_stdout is not None:
with open(red_stdout, 'w') as f:
f.write(outs.decode())
if proc.returncode:
raise Exception('Command {} failed with the above error.'
' Full command is:\n\n {}'.format(
cmd, outs.decode()))
def pmt_main():
"""
Method called by prometeo's command line utility `pmt`
"""
parser = argparse.ArgumentParser()
parser.add_argument("program_name", \
help="name of the prometeo script to be executed")
parser.add_argument("--cgen", type=str2bool, default="False", \
help="generate, compile and execute C code?")
parser.add_argument("--out", type=str, default=None, \
help="redirect output to file")
parser.add_argument("--debug", type=str, default=False, \
help="call raise instead of exit() upon Exception")
args = parser.parse_args()
filename = args.program_name
cgen = args.cgen
red_stdout = args.out
debug = args.debug
if cgen is False:
post = '''main()'''
code_no_hints = strip_file_to_string(filename) + post
tic = time.time()
exec(code_no_hints, globals(), globals())
toc = time.time() - tic
print('Execution time = {:0.3f} sec'.format(toc))
else:
try:
f = open(filename)
f.close()
except FileNotFoundError:
print('\n\033[;1m > prometeo:\033[0;0m \033[91m file {} not found.\033[0;0m'.format(filename))
exit()
print('\n\033[;1m > prometeo:\033[0;0m starting transpilation')
pmt_path = os.path.dirname(prometeo.__file__)
filename_ = filename.split('.')[0]
sed_cmd = "sed '/# pure >/,/# pure </d' " + filename_ + '.py'
code = ''.join(os.popen(sed_cmd).read())
tree = ast.parse(code)
# ap.pprint(tree)
tree_copy = deepcopy(tree)
try:
result = prometeo.cgen.code_gen_c.to_source(tree, filename_, \
main=True,
size_of_pointer = size_of_pointer, \
size_of_int = size_of_int, \
size_of_double = size_of_double)
except prometeo.cgen.code_gen_c.cgenException as e:
print('\n > Exception -- prometeo code-gen: ', e.message)
code = ''.join(open(filename))
print(' > @ line {}:'.format(e.lineno) + '\033[34m' + \
code.splitlines()[e.lineno-1] + '\033[0;0m')
print(' > Exiting.\n')
if debug:
raise
else:
return 1
dest_file = open('__pmt_cache__/' + filename_ + '.c', 'w')
dest_file.write(prometeo.cgen.source_repr.pretty_source(result.source))
dest_file.close()
dest_file = open('__pmt_cache__/' + filename_ + '.h', 'w')
dest_file.write(prometeo.cgen.source_repr.pretty_source(result.header))
dest_file.close()
print('\033[;1m > prometeo:\033[0;0m code-generation successfully completed')
print('\033[;1m > prometeo:\033[0;0m starting worst-case heap usage analysis')
# compute heap usage
# load log file
with open('__pmt_cache__/dim_record.json') as f:
dim_vars = json.load(f, object_pairs_hook=OrderedDict)
# reverse ordered dictionary to apply iterative resolution of expressions
dim_vars = OrderedDict(reversed(list(dim_vars.items())))
# resolve dims and dimv values
dim_vars = resolve_dims_value(dim_vars)
# evaluate numerical expressions
for key, value in dim_vars.items():
if isinstance(value, list):
for i in range(len(value)):
for j in range(len(value[i])):
dim_vars[key][i][j] = str(numexpr.evaluate(str(value[i][j])))
else:
dim_vars[key] = str(numexpr.evaluate(str(value)))
# load log file
with open('__pmt_cache__/heap64.json') as f:
heap64_data = json.load(f)
# resolve values of heap usage in calls
for key, value in heap64_data.items():
for item in dim_vars:
if item in heap64_data[key]:
heap64_data[key] = re.sub(r'\b' + item + r'\b', dim_vars[item], heap64_data[key])
# evaluate numerical expressions
for key, value in heap64_data.items():
heap64_data[key] = str(numexpr.evaluate(str(value)))
# load log file (8-byte aligned)
with open('__pmt_cache__/heap8.json') as f:
heap8_data = json.load(f)
# resolve values of heap usage in calls
for key, value in heap8_data.items():
for item in dim_vars:
if item in heap8_data[key]:
heap8_data[key] = re.sub(r'\b' + item + r'\b', dim_vars[item], heap8_data[key])
# evaluate numerical expressions
for key, value in heap8_data.items():
heap8_data[key] = str(numexpr.evaluate(str(value)))
visitor = ast_visitor()
visitor.visit(tree_copy)
call_graph = visitor.callees
typed_record = visitor.typed_record
meta_info = visitor.meta_info
# print('\ncall graph:\n\n', call_graph, '\n\n')
reach_map, call_graph = compute_reach_graph(\
call_graph, typed_record, meta_info)
# check that there are no cycles containing memory allocations
for method in reach_map:
if '*' in reach_map[method] and typed_record[method] != dict():
raise Exception('\n\nDetected cycle {} containing memory'
' allocation.\n'.format(reach_map[method]))
# update heap usage with memory associated with
# constructors (escape memory)
# load log file
with open('__pmt_cache__/constructor_record.json') as f:
constructors_list = json.load(f)
for caller, callees in call_graph.items():
for callee in callees:
# if call is a constructor, then account for
# escaped memory
if callee in constructors_list:
heap64_data[caller] = str(int(heap64_data[caller])
+ int(heap64_data[callee]))
heap8_data[caller] = str(int(heap8_data[caller])
+ int(heap8_data[callee]))
# print('reach_map:\n\n', reach_map, '\n\n')
# Bellman-Ford algorithm
# build memory graph (64-bytes aligned)
nodes = []
edges = []
for key, value in call_graph.items():
nodes.append(key)
for key, value in call_graph.items():
# if leaf node
if not value:
value = ['end']
for node in value:
if node in heap64_data:
heap_usage = -int(heap64_data[node])
else:
heap_usage = 0
# import pdb; pdb.set_trace()
edges.append([(key,node), heap_usage])
# add artificial end node
nodes.append('end')
if 'global@main' in heap64_data:
heap_main = -int(heap64_data['global@main'])
else:
heap_main = 0
mem_graph = Graph(nodes, edges, 'global@main', 'end', heap_main)
worst_case_heap_usage_64 = -mem_graph.compute_shortes_path()
# build memory graph (8-bytes aligned)
nodes = []
edges = []
for key, value in call_graph.items():
nodes.append(key)
for key, value in call_graph.items():
# if leaf node
if not value:
value = ['end']
for node in value:
if node in heap8_data:
heap_usage = -int(heap8_data[node])
else:
heap_usage = 0
# import pdb; pdb.set_trace()
edges.append([(key,node), heap_usage])
# add artificial end node
nodes.append('end')
if 'global@main' in heap8_data:
heap_main = -int(heap8_data['global@main'])
else:
heap_main = 0
mem_graph = Graph(nodes, edges, 'global@main', 'end', heap_main)
worst_case_heap_usage_8 = -mem_graph.compute_shortes_path()
print('\033[;1m > prometeo:\033[0;0m heap usage analysis completed successfully\n \
\033[34m{}\033[0;0m(\033[34m{}\033[0;0m) 64(8)-bytes aligned\n'.format(\
worst_case_heap_usage_64, worst_case_heap_usage_8))
# generate Makefile
makefile_code = makefile_template.replace('{{ filename }}', filename_)
makefile_code = makefile_code.replace('{{ HEAP8_SIZE }}', str(2*worst_case_heap_usage_8))
# NOTE: factor 2 due to alignment
makefile_code = makefile_code.replace('{{ HEAP64_SIZE }}', str(2*worst_case_heap_usage_64))
makefile_code = makefile_code.replace('\n','', 1)
makefile_code = makefile_code.replace('{{ INSTALL_DIR }}', os.path.dirname(__file__) + '/..')
with open('__pmt_cache__/casadi_funs.json') as f:
casadi_funs = json.load(f, object_pairs_hook=OrderedDict)
casadi_target_code = '\nOBJS = '
for item in casadi_funs:
fun_name = item.replace('@', '_')
casadi_target_code = casadi_target_code + ' ' + 'casadi_wrapper_' + fun_name + '.o ' + fun_name + '.o'
casadi_target_code = casadi_target_code + '\n\ncasadi: '
for item in casadi_funs:
fun_name = item.replace('@', '_')
casadi_target_code = casadi_target_code + ' ' + fun_name
for item in casadi_funs:
fun_name = item.replace('@', '_')
casadi_target_code = casadi_target_code + '\n\n'
casadi_target_code = casadi_target_code + fun_name + ':\n'
casadi_target_code = casadi_target_code + "\t$(CC) -c " + fun_name + '.c ' + 'casadi_wrapper_' + fun_name + '.c\n'
makefile_code = makefile_code.replace('{{ CASADI_TARGET }}', casadi_target_code)
dest_file = open('__pmt_cache__/Makefile', 'w+')
dest_file.write(makefile_code)
dest_file.close()
print('\033[;1m > prometeo:\033[0;0m building C code')
os.chdir('__pmt_cache__')
proc = subprocess.Popen(["make", "clean"], stdout=subprocess.PIPE)
try:
outs, errs = proc.communicate(timeout=20)
except TimeOutExpired:
proc.kill()
outs, errs = proc.communicate()
print('Command \'make\' timed out.')
if proc.returncode:
raise Exception('Command \'make\' failed with the above error.'
' Full command is:\n\n {}'.format(outs.decode()))
proc = subprocess.Popen(["make", "all"], stdout=subprocess.PIPE)
try:
outs, errs = proc.communicate(timeout=20)
except TimeOutExpired:
proc.kill()
outs, errs = proc.communicate()
print('Command \'make \' timed out.')
if proc.returncode:
raise Exception('Command \'make\' failed with the above error.'
' Full command is:\n\n {}'.format(outs.decode()))
print('\033[;1m > prometeo:\033[0;0m successfully built C code')
print('\033[;1m > prometeo:\033[0;0m running compiled C code:\n')
INSTALL_DIR = os.path.dirname(__file__) + '/..'
running_on = platform.system()
if running_on == 'Linux':
cmd = 'export LD_LIBRARY_PATH=' + INSTALL_DIR + '/lib/prometeo:' + \
INSTALL_DIR + '/lib/blasfeo:$LD_LIBRARY_PATH ; ./' + filename_
elif running_on == 'Darwin':
cmd = 'export DYLD_LIBRARY_PATH=' + INSTALL_DIR + '/lib/prometeo:' + \
INSTALL_DIR + '/lib/blasfeo:$DYLD_LIBRARY_PATH ; ./' + filename_
else:
raise Exception('Running on unsupported operating system {}'.format(running_on))
tic = time.time()
proc = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE)
try:
outs, errs = proc.communicate()
except TimeOutExpired:
proc.kill()
outs, errs = proc.communicate()
print('Command {} timed out.'.format(cmd))
if red_stdout is not None:
with open(red_stdout, 'w') as f:
f.write(outs)
toc = time.time() - tic
if red_stdout is not None:
with open(red_stdout, 'w') as f:
f.write(outs.decode())
else:
print(outs.decode())
if proc.returncode:
raise Exception('Command {} failed with the above error.'
' Full command is:\n\n {}'.format(cmd, outs.decode()))
print('\n\033[;1m > prometeo:\033[0;0m exiting\n')
os.chdir('..')
# from prometeo.mem.ast_analyzer import get_call_graph
from prometeo.mem.ast_analyzer import compute_reach_graph
from prometeo.mem.ast_analyzer_2 import ast_visitor
from prometeo.mem.ast_analyzer_2 import compute_reach_graph
# from prometeo.cgen.code_gen import to_source
import ast
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-i', '--input', help='Input .py file', required=True)
args = parser.parse_args()
tree = ast.parse(open(args.input).read())
# call_graph = get_call_graph(tree)
visitor = ast_visitor()
# import pdb; pdb.set_trace()
visitor.visit(tree)
call_graph = visitor.callees
print(call_graph)
# to_source(tree)
# print('call graph:\n', call_graph)
# import pdb; pdb.set_trace()
reach_map = compute_reach_graph(call_graph)
print('reach_map:\n', reach_map)