def strip_file(file_path):
from strip_hints import strip_file_to_string
if file_path.endswith('.py'):
stripped = strip_file_to_string(file_path,
to_empty=True,
only_assigns_and_defs=True)
print(file_path)
with open(file_path, 'w') as file_obj:
file_obj.write(stripped)
def _strip_hints():
base_path = os.path.relpath("ascii_canvas")
for path in _find_files(base_path, "*.py"):
file_str = strip_file_to_string(
path,
to_empty=False,
no_ast=False,
no_colon_move=False,
only_assigns_and_defs=False,
)
print(file_str)
with open(path, "w") as f:
f.write(file_str)
def strip_all_type_hints(root_dir):
"""Strip all type hints in place from the directory rooted from root_dir."""
for root, _, files in os.walk(root_dir):
for filename in files:
if not filename.endswith('.py'):
continue
filepath = os.path.join(root, filename)
print('Processing python file %s' % filepath)
code_string = strip_hints.strip_file_to_string(
filepath,
to_empty=False,
no_ast=False,
no_colon_move=False,
only_assigns_and_defs=False,
only_test_for_changes=False)
with open(filepath, 'w') as f:
f.write(code_string)
def run() -> None:
input_dir = Path("../plugins")
output_dir = Path("../plugins_py36")
output_dir.mkdir(exist_ok=True, parents=True)
output_files = []
for input_file in input_dir.iterdir():
source = strip_hints.strip_file_to_string(input_file)
source = clean_up_type_hints(source)
output_file = output_dir / input_file.name
with output_file.open("w", encoding="utf8") as file_obj:
file_obj.write(source)
output_files.append(output_file)
subprocess.run(["python", "-m", "black"] + [str(f) for f in output_files])
def copy_src(source, destination, filename):
full_src_path = os.path.join(source, filename)
full_dest_path = os.path.join(destination, filename)
with open(full_dest_path, "w") as f:
code_string = strip_file_to_string(full_src_path, to_empty=True, strip_nl=True)
code_string = re.sub(r"from typing import .*\n", "", code_string)
code_string = re.sub(r"\n.*INPUT_SIZE_TYPE = .*\n", "", code_string)
code_string = re.sub(r"DETECTED_OUTPUT_TYPES = .*\n", "", code_string)
code_string = re.sub(r" *,", ",", code_string)
code_string = re.sub(r" +=", " =", code_string)
code_string = re.sub(r",\n\s*", ", ", code_string)
code_string = re.sub(r"\(\n\s*", "(", code_string)
code_string = re.sub(r",?\s*\)\s*:", "):", code_string)
code_string = code_string.replace("CORRECTED_INPUT_SIZE_TYPE, ", "")
code_string = code_string.replace("from __future__ import annotations\n", "")
code_string = f2format.convert(code_string)
f.write(code_string)
def strip_all_type_hints(root_dir):
"""Strip all type hints in place from the directory rooted from root_dir."""
for root, _, files in os.walk(root_dir):
for filename in files:
if not filename.endswith('.py'):
continue
filepath = os.path.join(root, filename)
name = filename.split('/')[-1]
if name in _FILES_TO_SKIP:
continue
print('Processing python file %s' % filepath)
code_string = strip_hints.strip_file_to_string(
filepath,
to_empty=False,
no_ast=False,
no_colon_move=False,
only_assigns_and_defs=False,
only_test_for_changes=False)
code_string = code_string.replace(
'from typing import',
'from tensorflow_data_validation.types_compat import')
with open(filepath, 'w') as f:
f.write(code_string)
def process_file(f):
print(os.path.abspath(f))
out = strip_file_to_string(f)
with open(f, 'w') as fh:
fh.write(out)
"""
from __future__ import print_function, division, absolute_import
import codecs
import sys
sys.path.insert(1, "../src")
import strip_hints
if len(sys.argv) < 2:
print("Filename required as the argument.", file=sys.stderr)
sys.exit(1)
filename = sys.argv[1]
with codecs.open(filename, encoding='utf-8') as code_file:
code_string = code_file.read()
stripped_string_from_file = strip_hints.strip_file_to_string(
filename, only_test_for_changes=False)
stripped_string_from_string = strip_hints.strip_string_to_string(
code_string, only_test_for_changes=False)
#print(stripped_string_from_string, end="", sep="")
if stripped_string_from_file != stripped_string_from_string:
print("Error, the final stripped strings differ.")
else:
print("The final stripped strings are identical.")
from strip_hints import strip_file_to_string
import os
import fnmatch
# From https://stackoverflow.com/questions/2186525/use-a-glob-to-find-files-recursively-in-python # noqa: E501
def find_files(directory, pattern):
for root, dirs, files in os.walk(directory):
if "stress_tests" not in root:
for basename in files:
if fnmatch.fnmatch(basename, pattern):
filename = os.path.join(root, basename)
yield filename
# Go up to modin root
modin_path = os.path.relpath("../../")
for path in find_files(modin_path, "*.py"):
string = strip_file_to_string(
path,
to_empty=False,
no_ast=False,
no_colon_move=False,
only_assigns_and_defs=False,
)
with open(path, "w") as f:
f.write(string)
def pmt_main():
"""
Method called by prometeo's command line utility `pmt`
"""
parser = argparse.ArgumentParser()
parser.add_argument("program_name", \
help="name of the prometeo script to be executed")
parser.add_argument("--cgen", type=str2bool, default="False", \
help="generate, compile and execute C code?")
parser.add_argument("--out", type=str, default=None, \
help="redirect output to file")
parser.add_argument("--debug", type=str, default=False, \
help="call raise instead of exit() upon Exception")
args = parser.parse_args()
filename = args.program_name
cgen = args.cgen
red_stdout = args.out
debug = args.debug
if cgen is False:
post = '''main()'''
code_no_hints = strip_file_to_string(filename) + post
tic = time.time()
exec(code_no_hints, globals(), globals())
toc = time.time() - tic
print('Execution time = {:0.3f} sec'.format(toc))
else:
try:
f = open(filename)
f.close()
except FileNotFoundError:
print('\n\033[;1m > prometeo:\033[0;0m \033[91m file {} not found.\033[0;0m'.format(filename))
exit()
print('\n\033[;1m > prometeo:\033[0;0m starting transpilation')
pmt_path = os.path.dirname(prometeo.__file__)
filename_ = filename.split('.')[0]
sed_cmd = "sed '/# pure >/,/# pure </d' " + filename_ + '.py'
code = ''.join(os.popen(sed_cmd).read())
tree = ast.parse(code)
# ap.pprint(tree)
tree_copy = deepcopy(tree)
try:
result = prometeo.cgen.code_gen_c.to_source(tree, filename_, \
main=True,
size_of_pointer = size_of_pointer, \
size_of_int = size_of_int, \
size_of_double = size_of_double)
except prometeo.cgen.code_gen_c.cgenException as e:
print('\n > Exception -- prometeo code-gen: ', e.message)
code = ''.join(open(filename))
print(' > @ line {}:'.format(e.lineno) + '\033[34m' + \
code.splitlines()[e.lineno-1] + '\033[0;0m')
print(' > Exiting.\n')
if debug:
raise
else:
return 1
dest_file = open('__pmt_cache__/' + filename_ + '.c', 'w')
dest_file.write(prometeo.cgen.source_repr.pretty_source(result.source))
dest_file.close()
dest_file = open('__pmt_cache__/' + filename_ + '.h', 'w')
dest_file.write(prometeo.cgen.source_repr.pretty_source(result.header))
dest_file.close()
print('\033[;1m > prometeo:\033[0;0m code-generation successfully completed')
print('\033[;1m > prometeo:\033[0;0m starting worst-case heap usage analysis')
# compute heap usage
# load log file
with open('__pmt_cache__/dim_record.json') as f:
dim_vars = json.load(f, object_pairs_hook=OrderedDict)
# reverse ordered dictionary to apply iterative resolution of expressions
dim_vars = OrderedDict(reversed(list(dim_vars.items())))
# resolve dims and dimv values
dim_vars = resolve_dims_value(dim_vars)
# evaluate numerical expressions
for key, value in dim_vars.items():
if isinstance(value, list):
for i in range(len(value)):
for j in range(len(value[i])):
dim_vars[key][i][j] = str(numexpr.evaluate(str(value[i][j])))
else:
dim_vars[key] = str(numexpr.evaluate(str(value)))
# load log file
with open('__pmt_cache__/heap64.json') as f:
heap64_data = json.load(f)
# resolve values of heap usage in calls
for key, value in heap64_data.items():
for item in dim_vars:
if item in heap64_data[key]:
heap64_data[key] = re.sub(r'\b' + item + r'\b', dim_vars[item], heap64_data[key])
# evaluate numerical expressions
for key, value in heap64_data.items():
heap64_data[key] = str(numexpr.evaluate(str(value)))
# load log file (8-byte aligned)
with open('__pmt_cache__/heap8.json') as f:
heap8_data = json.load(f)
# resolve values of heap usage in calls
for key, value in heap8_data.items():
for item in dim_vars:
if item in heap8_data[key]:
heap8_data[key] = re.sub(r'\b' + item + r'\b', dim_vars[item], heap8_data[key])
# evaluate numerical expressions
for key, value in heap8_data.items():
heap8_data[key] = str(numexpr.evaluate(str(value)))
visitor = ast_visitor()
visitor.visit(tree_copy)
call_graph = visitor.callees
typed_record = visitor.typed_record
meta_info = visitor.meta_info
# print('\ncall graph:\n\n', call_graph, '\n\n')
reach_map, call_graph = compute_reach_graph(\
call_graph, typed_record, meta_info)
# check that there are no cycles containing memory allocations
for method in reach_map:
if '*' in reach_map[method] and typed_record[method] != dict():
raise Exception('\n\nDetected cycle {} containing memory'
' allocation.\n'.format(reach_map[method]))
# update heap usage with memory associated with
# constructors (escape memory)
# load log file
with open('__pmt_cache__/constructor_record.json') as f:
constructors_list = json.load(f)
for caller, callees in call_graph.items():
for callee in callees:
# if call is a constructor, then account for
# escaped memory
if callee in constructors_list:
heap64_data[caller] = str(int(heap64_data[caller])
+ int(heap64_data[callee]))
heap8_data[caller] = str(int(heap8_data[caller])
+ int(heap8_data[callee]))
# print('reach_map:\n\n', reach_map, '\n\n')
# Bellman-Ford algorithm
# build memory graph (64-bytes aligned)
nodes = []
edges = []
for key, value in call_graph.items():
nodes.append(key)
for key, value in call_graph.items():
# if leaf node
if not value:
value = ['end']
for node in value:
if node in heap64_data:
heap_usage = -int(heap64_data[node])
else:
heap_usage = 0
# import pdb; pdb.set_trace()
edges.append([(key,node), heap_usage])
# add artificial end node
nodes.append('end')
if 'global@main' in heap64_data:
heap_main = -int(heap64_data['global@main'])
else:
heap_main = 0
mem_graph = Graph(nodes, edges, 'global@main', 'end', heap_main)
worst_case_heap_usage_64 = -mem_graph.compute_shortes_path()
# build memory graph (8-bytes aligned)
nodes = []
edges = []
for key, value in call_graph.items():
nodes.append(key)
for key, value in call_graph.items():
# if leaf node
if not value:
value = ['end']
for node in value:
if node in heap8_data:
heap_usage = -int(heap8_data[node])
else:
heap_usage = 0
# import pdb; pdb.set_trace()
edges.append([(key,node), heap_usage])
# add artificial end node
nodes.append('end')
if 'global@main' in heap8_data:
heap_main = -int(heap8_data['global@main'])
else:
heap_main = 0
mem_graph = Graph(nodes, edges, 'global@main', 'end', heap_main)
worst_case_heap_usage_8 = -mem_graph.compute_shortes_path()
print('\033[;1m > prometeo:\033[0;0m heap usage analysis completed successfully\n \
\033[34m{}\033[0;0m(\033[34m{}\033[0;0m) 64(8)-bytes aligned\n'.format(\
worst_case_heap_usage_64, worst_case_heap_usage_8))
# generate Makefile
makefile_code = makefile_template.replace('{{ filename }}', filename_)
makefile_code = makefile_code.replace('{{ HEAP8_SIZE }}', str(2*worst_case_heap_usage_8))
# NOTE: factor 2 due to alignment
makefile_code = makefile_code.replace('{{ HEAP64_SIZE }}', str(2*worst_case_heap_usage_64))
makefile_code = makefile_code.replace('\n','', 1)
makefile_code = makefile_code.replace('{{ INSTALL_DIR }}', os.path.dirname(__file__) + '/..')
with open('__pmt_cache__/casadi_funs.json') as f:
casadi_funs = json.load(f, object_pairs_hook=OrderedDict)
casadi_target_code = '\nOBJS = '
for item in casadi_funs:
fun_name = item.replace('@', '_')
casadi_target_code = casadi_target_code + ' ' + 'casadi_wrapper_' + fun_name + '.o ' + fun_name + '.o'
casadi_target_code = casadi_target_code + '\n\ncasadi: '
for item in casadi_funs:
fun_name = item.replace('@', '_')
casadi_target_code = casadi_target_code + ' ' + fun_name
for item in casadi_funs:
fun_name = item.replace('@', '_')
casadi_target_code = casadi_target_code + '\n\n'
casadi_target_code = casadi_target_code + fun_name + ':\n'
casadi_target_code = casadi_target_code + "\t$(CC) -c " + fun_name + '.c ' + 'casadi_wrapper_' + fun_name + '.c\n'
makefile_code = makefile_code.replace('{{ CASADI_TARGET }}', casadi_target_code)
dest_file = open('__pmt_cache__/Makefile', 'w+')
dest_file.write(makefile_code)
dest_file.close()
print('\033[;1m > prometeo:\033[0;0m building C code')
os.chdir('__pmt_cache__')
proc = subprocess.Popen(["make", "clean"], stdout=subprocess.PIPE)
try:
outs, errs = proc.communicate(timeout=20)
except TimeOutExpired:
proc.kill()
outs, errs = proc.communicate()
print('Command \'make\' timed out.')
if proc.returncode:
raise Exception('Command \'make\' failed with the above error.'
' Full command is:\n\n {}'.format(outs.decode()))
proc = subprocess.Popen(["make", "all"], stdout=subprocess.PIPE)
try:
outs, errs = proc.communicate(timeout=20)
except TimeOutExpired:
proc.kill()
outs, errs = proc.communicate()
print('Command \'make \' timed out.')
if proc.returncode:
raise Exception('Command \'make\' failed with the above error.'
' Full command is:\n\n {}'.format(outs.decode()))
print('\033[;1m > prometeo:\033[0;0m successfully built C code')
print('\033[;1m > prometeo:\033[0;0m running compiled C code:\n')
INSTALL_DIR = os.path.dirname(__file__) + '/..'
running_on = platform.system()
if running_on == 'Linux':
cmd = 'export LD_LIBRARY_PATH=' + INSTALL_DIR + '/lib/prometeo:' + \
INSTALL_DIR + '/lib/blasfeo:$LD_LIBRARY_PATH ; ./' + filename_
elif running_on == 'Darwin':
cmd = 'export DYLD_LIBRARY_PATH=' + INSTALL_DIR + '/lib/prometeo:' + \
INSTALL_DIR + '/lib/blasfeo:$DYLD_LIBRARY_PATH ; ./' + filename_
else:
raise Exception('Running on unsupported operating system {}'.format(running_on))
tic = time.time()
proc = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE)
try:
outs, errs = proc.communicate()
except TimeOutExpired:
proc.kill()
outs, errs = proc.communicate()
print('Command {} timed out.'.format(cmd))
if red_stdout is not None:
with open(red_stdout, 'w') as f:
f.write(outs)
toc = time.time() - tic
if red_stdout is not None:
with open(red_stdout, 'w') as f:
f.write(outs.decode())
else:
print(outs.decode())
if proc.returncode:
raise Exception('Command {} failed with the above error.'
' Full command is:\n\n {}'.format(cmd, outs.decode()))
print('\n\033[;1m > prometeo:\033[0;0m exiting\n')
os.chdir('..')
#!/usr/bin/python3
import os
from strip_hints import strip_file_to_string
with open('transference.tmp', 'w') as out:
print(strip_file_to_string('transference.py'), file=out)
os.chmod('transference.tmp', 0o755)
os.rename('transference.tmp', 'transference')
def pmt_main(script_path, stdout, stderr, args=None):
parser = argparse.ArgumentParser()
parser.add_argument("program_name", \
help="name of the prometeo script to be executed")
parser.add_argument("--cgen", type=str2bool, default="False", \
help="generate, compile and execute C code?")
parser.add_argument("--out", type=str, default=None, \
help="redirect output to file")
parser.add_argument("--debug", type=str, default=False, \
help="call raise instead of exit() upon Exception")
args = parser.parse_args()
filename = args.program_name
cgen = args.cgen
red_stdout = args.out
debug = args.debug
if cgen is False:
post = '''main()'''
# code = open(filename).read() + post
# exec(code, globals(), globals())
code_no_hints = strip_file_to_string(filename) + post
exec(code_no_hints, globals(), globals())
else:
pmt_path = os.path.dirname(prometeo.__file__)
# cmd = 'export MYPYPATH=' + pmt_path + ' & mypy ' + filename
# os.system(cmd)
filename_ = filename.split('.')[0]
tree = ast.parse(''.join(open(filename)))
# tree will be slightly modified by
# to source(). Store a copy, for heap usage
# analysis:
tree_copy = deepcopy(tree)
# astpretty.pprint(tree)
# import pdb; pdb.set_trace()
# result = prometeo.cgen.code_gen_c.to_source(tree, filename_, \
# main=True, ___c_pmt_8_heap_size=10000, \
# ___c_pmt_64_heap_size=1000000, \
# size_of_pointer = size_of_pointer, \
# size_of_int = size_of_int, \
# size_of_double = size_of_double)
try:
result = prometeo.cgen.code_gen_c.to_source(tree, filename_, \
main=True, ___c_pmt_8_heap_size=10000, \
___c_pmt_64_heap_size=1000000, \
size_of_pointer = size_of_pointer, \
size_of_int = size_of_int, \
size_of_double = size_of_double)
except prometeo.cgen.code_gen_c.cgenException as e:
print('\n > Exception -- prometeo code-gen: ', e.message)
code = ''.join(open(filename))
print(' > @ line {}:'.format(e.lineno) + '\033[34m' +
code.splitlines()[e.lineno - 1] + '\033[0;0m')
print(' > Exiting.\n')
if debug:
raise
else:
exit()
dest_file = open(filename_ + '.c', 'w')
dest_file.write(prometeo.cgen.source_repr.pretty_source(result.source))
dest_file.close()
dest_file = open(filename_ + '.h', 'w')
dest_file.write(prometeo.cgen.source_repr.pretty_source(result.header))
dest_file.close()
# generate Makefile
makefile_code = makefile_template.replace('{{ filename }}', filename_)
# execname_ = re.sub('\.c$', '', filename_)
# makefile_code = makefile_template.replace('{{ execname }}', execname_)
makefile_code = makefile_code.replace('\n', '', 1)
makefile_code = makefile_code.replace(
'@INSTALL_DIR@',
os.path.dirname(__file__) + '/..')
dest_file = open('Makefile', 'w+')
dest_file.write(makefile_code)
dest_file.close()
# compute heap usage
visitor = ast_visitor()
# import pdb; pdb.set_trace()
visitor.visit(tree_copy)
call_graph = visitor.callees
typed_record = visitor.typed_record
# print('\ncall graph:\n\n', call_graph, '\n\n')
reach_map = compute_reach_graph(call_graph, typed_record)
# print('reach_map:\n\n', reach_map, '\n\n')
# check that there are no cycles containing memory allocations
for method in reach_map:
if '*' in reach_map[method] and typed_record[method] != dict():
raise Exception('\n\nDetected cycle {} containing memory'
' allocation.\n'.format(reach_map[method]))
proc = subprocess.Popen(["make", "clean"], stdout=subprocess.PIPE)
try:
outs, errs = proc.communicate(timeout=20)
except TimeOutExpired:
proc.kill()
outs, errs = proc.communicate()
print('Command \'make\' timed out.')
if proc.returncode:
raise Exception('Command \'make\' failed with the above error.'
' Full command is:\n\n {}'.format(outs.decode()))
proc = subprocess.Popen(["make"], stdout=subprocess.PIPE)
try:
outs, errs = proc.communicate(timeout=20)
except TimeOutExpired:
proc.kill()
outs, errs = proc.communicate()
print('Command \'make\' timed out.')
if proc.returncode:
raise Exception('Command \'make\' failed with the above error.'
' Full command is:\n\n {}'.format(outs.decode()))
INSTALL_DIR = os.path.dirname(__file__) + '/..'
running_on = platform.system()
if running_on == 'Linux':
cmd = 'export LD_LIBRARY_PATH=' + INSTALL_DIR + '/lib/prometeo:' + \
INSTALL_DIR + '/lib/blasfeo:$LD_LIBRARY_PATH ; ./' + filename_
elif running_on == 'Darwin':
cmd = 'export DYLD_LIBRARY_PATH=' + INSTALL_DIR + '/lib/prometeo:' + \
INSTALL_DIR + '/lib/blasfeo:$DYLD_LIBRARY_PATH ; ./' + filename_
else:
raise Exception(
'Running on unsupported operating system {}'.format(
running_on))
if red_stdout is not None:
proc = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE)
else:
proc = subprocess.Popen([cmd], shell=True)
try:
outs, errs = proc.communicate()
except TimeOutExpired:
proc.kill()
outs, errs = proc.communicate()
print('Command \'make\' timed out.')
if red_stdout is not None:
with open(red_stdout, 'w') as f:
f.write(outs)
if red_stdout is not None:
with open(red_stdout, 'w') as f:
f.write(outs.decode())
if proc.returncode:
raise Exception('Command {} failed with the above error.'
' Full command is:\n\n {}'.format(
cmd, outs.decode()))
def process_file(f):
print(f)
out = strip_file_to_string(f)
with open(f, 'w') as fh:
fh.write(out)
def file_has_incompatible_typehints(file_path):
return strip_hints.strip_file_to_string(file_path,
to_empty=True,
only_assigns_and_defs=True,
only_test_for_changes=True)
def strip_hints_and_overwrite(file_path):
transformed = strip_hints.strip_file_to_string(file_path,
to_empty=True,
only_assigns_and_defs=True)
with open(file_path, "w") as f:
f.write(transformed)
