def strip_file(file_path): from strip_hints import strip_file_to_string if file_path.endswith('.py'): stripped = strip_file_to_string(file_path, to_empty=True, only_assigns_and_defs=True) print(file_path) with open(file_path, 'w') as file_obj: file_obj.write(stripped)
def _strip_hints(): base_path = os.path.relpath("ascii_canvas") for path in _find_files(base_path, "*.py"): file_str = strip_file_to_string( path, to_empty=False, no_ast=False, no_colon_move=False, only_assigns_and_defs=False, ) print(file_str) with open(path, "w") as f: f.write(file_str)
def strip_all_type_hints(root_dir): """Strip all type hints in place from the directory rooted from root_dir.""" for root, _, files in os.walk(root_dir): for filename in files: if not filename.endswith('.py'): continue filepath = os.path.join(root, filename) print('Processing python file %s' % filepath) code_string = strip_hints.strip_file_to_string( filepath, to_empty=False, no_ast=False, no_colon_move=False, only_assigns_and_defs=False, only_test_for_changes=False) with open(filepath, 'w') as f: f.write(code_string)
def run() -> None: input_dir = Path("../plugins") output_dir = Path("../plugins_py36") output_dir.mkdir(exist_ok=True, parents=True) output_files = [] for input_file in input_dir.iterdir(): source = strip_hints.strip_file_to_string(input_file) source = clean_up_type_hints(source) output_file = output_dir / input_file.name with output_file.open("w", encoding="utf8") as file_obj: file_obj.write(source) output_files.append(output_file) subprocess.run(["python", "-m", "black"] + [str(f) for f in output_files])
def copy_src(source, destination, filename): full_src_path = os.path.join(source, filename) full_dest_path = os.path.join(destination, filename) with open(full_dest_path, "w") as f: code_string = strip_file_to_string(full_src_path, to_empty=True, strip_nl=True) code_string = re.sub(r"from typing import .*\n", "", code_string) code_string = re.sub(r"\n.*INPUT_SIZE_TYPE = .*\n", "", code_string) code_string = re.sub(r"DETECTED_OUTPUT_TYPES = .*\n", "", code_string) code_string = re.sub(r" *,", ",", code_string) code_string = re.sub(r" +=", " =", code_string) code_string = re.sub(r",\n\s*", ", ", code_string) code_string = re.sub(r"\(\n\s*", "(", code_string) code_string = re.sub(r",?\s*\)\s*:", "):", code_string) code_string = code_string.replace("CORRECTED_INPUT_SIZE_TYPE, ", "") code_string = code_string.replace("from __future__ import annotations\n", "") code_string = f2format.convert(code_string) f.write(code_string)
def strip_all_type_hints(root_dir): """Strip all type hints in place from the directory rooted from root_dir.""" for root, _, files in os.walk(root_dir): for filename in files: if not filename.endswith('.py'): continue filepath = os.path.join(root, filename) name = filename.split('/')[-1] if name in _FILES_TO_SKIP: continue print('Processing python file %s' % filepath) code_string = strip_hints.strip_file_to_string( filepath, to_empty=False, no_ast=False, no_colon_move=False, only_assigns_and_defs=False, only_test_for_changes=False) code_string = code_string.replace( 'from typing import', 'from tensorflow_data_validation.types_compat import') with open(filepath, 'w') as f: f.write(code_string)
def process_file(f): print(os.path.abspath(f)) out = strip_file_to_string(f) with open(f, 'w') as fh: fh.write(out)
""" from __future__ import print_function, division, absolute_import import codecs import sys sys.path.insert(1, "../src") import strip_hints if len(sys.argv) < 2: print("Filename required as the argument.", file=sys.stderr) sys.exit(1) filename = sys.argv[1] with codecs.open(filename, encoding='utf-8') as code_file: code_string = code_file.read() stripped_string_from_file = strip_hints.strip_file_to_string( filename, only_test_for_changes=False) stripped_string_from_string = strip_hints.strip_string_to_string( code_string, only_test_for_changes=False) #print(stripped_string_from_string, end="", sep="") if stripped_string_from_file != stripped_string_from_string: print("Error, the final stripped strings differ.") else: print("The final stripped strings are identical.")
from strip_hints import strip_file_to_string import os import fnmatch # From https://stackoverflow.com/questions/2186525/use-a-glob-to-find-files-recursively-in-python # noqa: E501 def find_files(directory, pattern): for root, dirs, files in os.walk(directory): if "stress_tests" not in root: for basename in files: if fnmatch.fnmatch(basename, pattern): filename = os.path.join(root, basename) yield filename # Go up to modin root modin_path = os.path.relpath("../../") for path in find_files(modin_path, "*.py"): string = strip_file_to_string( path, to_empty=False, no_ast=False, no_colon_move=False, only_assigns_and_defs=False, ) with open(path, "w") as f: f.write(string)
def pmt_main(): """ Method called by prometeo's command line utility `pmt` """ parser = argparse.ArgumentParser() parser.add_argument("program_name", \ help="name of the prometeo script to be executed") parser.add_argument("--cgen", type=str2bool, default="False", \ help="generate, compile and execute C code?") parser.add_argument("--out", type=str, default=None, \ help="redirect output to file") parser.add_argument("--debug", type=str, default=False, \ help="call raise instead of exit() upon Exception") args = parser.parse_args() filename = args.program_name cgen = args.cgen red_stdout = args.out debug = args.debug if cgen is False: post = '''main()''' code_no_hints = strip_file_to_string(filename) + post tic = time.time() exec(code_no_hints, globals(), globals()) toc = time.time() - tic print('Execution time = {:0.3f} sec'.format(toc)) else: try: f = open(filename) f.close() except FileNotFoundError: print('\n\033[;1m > prometeo:\033[0;0m \033[91m file {} not found.\033[0;0m'.format(filename)) exit() print('\n\033[;1m > prometeo:\033[0;0m starting transpilation') pmt_path = os.path.dirname(prometeo.__file__) filename_ = filename.split('.')[0] sed_cmd = "sed '/# pure >/,/# pure </d' " + filename_ + '.py' code = ''.join(os.popen(sed_cmd).read()) tree = ast.parse(code) # ap.pprint(tree) tree_copy = deepcopy(tree) try: result = prometeo.cgen.code_gen_c.to_source(tree, filename_, \ main=True, size_of_pointer = size_of_pointer, \ size_of_int = size_of_int, \ size_of_double = size_of_double) except prometeo.cgen.code_gen_c.cgenException as e: print('\n > Exception -- prometeo code-gen: ', e.message) code = ''.join(open(filename)) print(' > @ line {}:'.format(e.lineno) + '\033[34m' + \ code.splitlines()[e.lineno-1] + '\033[0;0m') print(' > Exiting.\n') if debug: raise else: return 1 dest_file = open('__pmt_cache__/' + filename_ + '.c', 'w') dest_file.write(prometeo.cgen.source_repr.pretty_source(result.source)) dest_file.close() dest_file = open('__pmt_cache__/' + filename_ + '.h', 'w') dest_file.write(prometeo.cgen.source_repr.pretty_source(result.header)) dest_file.close() print('\033[;1m > prometeo:\033[0;0m code-generation successfully completed') print('\033[;1m > prometeo:\033[0;0m starting worst-case heap usage analysis') # compute heap usage # load log file with open('__pmt_cache__/dim_record.json') as f: dim_vars = json.load(f, object_pairs_hook=OrderedDict) # reverse ordered dictionary to apply iterative resolution of expressions dim_vars = OrderedDict(reversed(list(dim_vars.items()))) # resolve dims and dimv values dim_vars = resolve_dims_value(dim_vars) # evaluate numerical expressions for key, value in dim_vars.items(): if isinstance(value, list): for i in range(len(value)): for j in range(len(value[i])): dim_vars[key][i][j] = str(numexpr.evaluate(str(value[i][j]))) else: dim_vars[key] = str(numexpr.evaluate(str(value))) # load log file with open('__pmt_cache__/heap64.json') as f: heap64_data = json.load(f) # resolve values of heap usage in calls for key, value in heap64_data.items(): for item in dim_vars: if item in heap64_data[key]: heap64_data[key] = re.sub(r'\b' + item + r'\b', dim_vars[item], heap64_data[key]) # evaluate numerical expressions for key, value in heap64_data.items(): heap64_data[key] = str(numexpr.evaluate(str(value))) # load log file (8-byte aligned) with open('__pmt_cache__/heap8.json') as f: heap8_data = json.load(f) # resolve values of heap usage in calls for key, value in heap8_data.items(): for item in dim_vars: if item in heap8_data[key]: heap8_data[key] = re.sub(r'\b' + item + r'\b', dim_vars[item], heap8_data[key]) # evaluate numerical expressions for key, value in heap8_data.items(): heap8_data[key] = str(numexpr.evaluate(str(value))) visitor = ast_visitor() visitor.visit(tree_copy) call_graph = visitor.callees typed_record = visitor.typed_record meta_info = visitor.meta_info # print('\ncall graph:\n\n', call_graph, '\n\n') reach_map, call_graph = compute_reach_graph(\ call_graph, typed_record, meta_info) # check that there are no cycles containing memory allocations for method in reach_map: if '*' in reach_map[method] and typed_record[method] != dict(): raise Exception('\n\nDetected cycle {} containing memory' ' allocation.\n'.format(reach_map[method])) # update heap usage with memory associated with # constructors (escape memory) # load log file with open('__pmt_cache__/constructor_record.json') as f: constructors_list = json.load(f) for caller, callees in call_graph.items(): for callee in callees: # if call is a constructor, then account for # escaped memory if callee in constructors_list: heap64_data[caller] = str(int(heap64_data[caller]) + int(heap64_data[callee])) heap8_data[caller] = str(int(heap8_data[caller]) + int(heap8_data[callee])) # print('reach_map:\n\n', reach_map, '\n\n') # Bellman-Ford algorithm # build memory graph (64-bytes aligned) nodes = [] edges = [] for key, value in call_graph.items(): nodes.append(key) for key, value in call_graph.items(): # if leaf node if not value: value = ['end'] for node in value: if node in heap64_data: heap_usage = -int(heap64_data[node]) else: heap_usage = 0 # import pdb; pdb.set_trace() edges.append([(key,node), heap_usage]) # add artificial end node nodes.append('end') if 'global@main' in heap64_data: heap_main = -int(heap64_data['global@main']) else: heap_main = 0 mem_graph = Graph(nodes, edges, 'global@main', 'end', heap_main) worst_case_heap_usage_64 = -mem_graph.compute_shortes_path() # build memory graph (8-bytes aligned) nodes = [] edges = [] for key, value in call_graph.items(): nodes.append(key) for key, value in call_graph.items(): # if leaf node if not value: value = ['end'] for node in value: if node in heap8_data: heap_usage = -int(heap8_data[node]) else: heap_usage = 0 # import pdb; pdb.set_trace() edges.append([(key,node), heap_usage]) # add artificial end node nodes.append('end') if 'global@main' in heap8_data: heap_main = -int(heap8_data['global@main']) else: heap_main = 0 mem_graph = Graph(nodes, edges, 'global@main', 'end', heap_main) worst_case_heap_usage_8 = -mem_graph.compute_shortes_path() print('\033[;1m > prometeo:\033[0;0m heap usage analysis completed successfully\n \ \033[34m{}\033[0;0m(\033[34m{}\033[0;0m) 64(8)-bytes aligned\n'.format(\ worst_case_heap_usage_64, worst_case_heap_usage_8)) # generate Makefile makefile_code = makefile_template.replace('{{ filename }}', filename_) makefile_code = makefile_code.replace('{{ HEAP8_SIZE }}', str(2*worst_case_heap_usage_8)) # NOTE: factor 2 due to alignment makefile_code = makefile_code.replace('{{ HEAP64_SIZE }}', str(2*worst_case_heap_usage_64)) makefile_code = makefile_code.replace('\n','', 1) makefile_code = makefile_code.replace('{{ INSTALL_DIR }}', os.path.dirname(__file__) + '/..') with open('__pmt_cache__/casadi_funs.json') as f: casadi_funs = json.load(f, object_pairs_hook=OrderedDict) casadi_target_code = '\nOBJS = ' for item in casadi_funs: fun_name = item.replace('@', '_') casadi_target_code = casadi_target_code + ' ' + 'casadi_wrapper_' + fun_name + '.o ' + fun_name + '.o' casadi_target_code = casadi_target_code + '\n\ncasadi: ' for item in casadi_funs: fun_name = item.replace('@', '_') casadi_target_code = casadi_target_code + ' ' + fun_name for item in casadi_funs: fun_name = item.replace('@', '_') casadi_target_code = casadi_target_code + '\n\n' casadi_target_code = casadi_target_code + fun_name + ':\n' casadi_target_code = casadi_target_code + "\t$(CC) -c " + fun_name + '.c ' + 'casadi_wrapper_' + fun_name + '.c\n' makefile_code = makefile_code.replace('{{ CASADI_TARGET }}', casadi_target_code) dest_file = open('__pmt_cache__/Makefile', 'w+') dest_file.write(makefile_code) dest_file.close() print('\033[;1m > prometeo:\033[0;0m building C code') os.chdir('__pmt_cache__') proc = subprocess.Popen(["make", "clean"], stdout=subprocess.PIPE) try: outs, errs = proc.communicate(timeout=20) except TimeOutExpired: proc.kill() outs, errs = proc.communicate() print('Command \'make\' timed out.') if proc.returncode: raise Exception('Command \'make\' failed with the above error.' ' Full command is:\n\n {}'.format(outs.decode())) proc = subprocess.Popen(["make", "all"], stdout=subprocess.PIPE) try: outs, errs = proc.communicate(timeout=20) except TimeOutExpired: proc.kill() outs, errs = proc.communicate() print('Command \'make \' timed out.') if proc.returncode: raise Exception('Command \'make\' failed with the above error.' ' Full command is:\n\n {}'.format(outs.decode())) print('\033[;1m > prometeo:\033[0;0m successfully built C code') print('\033[;1m > prometeo:\033[0;0m running compiled C code:\n') INSTALL_DIR = os.path.dirname(__file__) + '/..' running_on = platform.system() if running_on == 'Linux': cmd = 'export LD_LIBRARY_PATH=' + INSTALL_DIR + '/lib/prometeo:' + \ INSTALL_DIR + '/lib/blasfeo:$LD_LIBRARY_PATH ; ./' + filename_ elif running_on == 'Darwin': cmd = 'export DYLD_LIBRARY_PATH=' + INSTALL_DIR + '/lib/prometeo:' + \ INSTALL_DIR + '/lib/blasfeo:$DYLD_LIBRARY_PATH ; ./' + filename_ else: raise Exception('Running on unsupported operating system {}'.format(running_on)) tic = time.time() proc = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE) try: outs, errs = proc.communicate() except TimeOutExpired: proc.kill() outs, errs = proc.communicate() print('Command {} timed out.'.format(cmd)) if red_stdout is not None: with open(red_stdout, 'w') as f: f.write(outs) toc = time.time() - tic if red_stdout is not None: with open(red_stdout, 'w') as f: f.write(outs.decode()) else: print(outs.decode()) if proc.returncode: raise Exception('Command {} failed with the above error.' ' Full command is:\n\n {}'.format(cmd, outs.decode())) print('\n\033[;1m > prometeo:\033[0;0m exiting\n') os.chdir('..')
#!/usr/bin/python3 import os from strip_hints import strip_file_to_string with open('transference.tmp', 'w') as out: print(strip_file_to_string('transference.py'), file=out) os.chmod('transference.tmp', 0o755) os.rename('transference.tmp', 'transference')
def pmt_main(script_path, stdout, stderr, args=None): parser = argparse.ArgumentParser() parser.add_argument("program_name", \ help="name of the prometeo script to be executed") parser.add_argument("--cgen", type=str2bool, default="False", \ help="generate, compile and execute C code?") parser.add_argument("--out", type=str, default=None, \ help="redirect output to file") parser.add_argument("--debug", type=str, default=False, \ help="call raise instead of exit() upon Exception") args = parser.parse_args() filename = args.program_name cgen = args.cgen red_stdout = args.out debug = args.debug if cgen is False: post = '''main()''' # code = open(filename).read() + post # exec(code, globals(), globals()) code_no_hints = strip_file_to_string(filename) + post exec(code_no_hints, globals(), globals()) else: pmt_path = os.path.dirname(prometeo.__file__) # cmd = 'export MYPYPATH=' + pmt_path + ' & mypy ' + filename # os.system(cmd) filename_ = filename.split('.')[0] tree = ast.parse(''.join(open(filename))) # tree will be slightly modified by # to source(). Store a copy, for heap usage # analysis: tree_copy = deepcopy(tree) # astpretty.pprint(tree) # import pdb; pdb.set_trace() # result = prometeo.cgen.code_gen_c.to_source(tree, filename_, \ # main=True, ___c_pmt_8_heap_size=10000, \ # ___c_pmt_64_heap_size=1000000, \ # size_of_pointer = size_of_pointer, \ # size_of_int = size_of_int, \ # size_of_double = size_of_double) try: result = prometeo.cgen.code_gen_c.to_source(tree, filename_, \ main=True, ___c_pmt_8_heap_size=10000, \ ___c_pmt_64_heap_size=1000000, \ size_of_pointer = size_of_pointer, \ size_of_int = size_of_int, \ size_of_double = size_of_double) except prometeo.cgen.code_gen_c.cgenException as e: print('\n > Exception -- prometeo code-gen: ', e.message) code = ''.join(open(filename)) print(' > @ line {}:'.format(e.lineno) + '\033[34m' + code.splitlines()[e.lineno - 1] + '\033[0;0m') print(' > Exiting.\n') if debug: raise else: exit() dest_file = open(filename_ + '.c', 'w') dest_file.write(prometeo.cgen.source_repr.pretty_source(result.source)) dest_file.close() dest_file = open(filename_ + '.h', 'w') dest_file.write(prometeo.cgen.source_repr.pretty_source(result.header)) dest_file.close() # generate Makefile makefile_code = makefile_template.replace('{{ filename }}', filename_) # execname_ = re.sub('\.c$', '', filename_) # makefile_code = makefile_template.replace('{{ execname }}', execname_) makefile_code = makefile_code.replace('\n', '', 1) makefile_code = makefile_code.replace( '@INSTALL_DIR@', os.path.dirname(__file__) + '/..') dest_file = open('Makefile', 'w+') dest_file.write(makefile_code) dest_file.close() # compute heap usage visitor = ast_visitor() # import pdb; pdb.set_trace() visitor.visit(tree_copy) call_graph = visitor.callees typed_record = visitor.typed_record # print('\ncall graph:\n\n', call_graph, '\n\n') reach_map = compute_reach_graph(call_graph, typed_record) # print('reach_map:\n\n', reach_map, '\n\n') # check that there are no cycles containing memory allocations for method in reach_map: if '*' in reach_map[method] and typed_record[method] != dict(): raise Exception('\n\nDetected cycle {} containing memory' ' allocation.\n'.format(reach_map[method])) proc = subprocess.Popen(["make", "clean"], stdout=subprocess.PIPE) try: outs, errs = proc.communicate(timeout=20) except TimeOutExpired: proc.kill() outs, errs = proc.communicate() print('Command \'make\' timed out.') if proc.returncode: raise Exception('Command \'make\' failed with the above error.' ' Full command is:\n\n {}'.format(outs.decode())) proc = subprocess.Popen(["make"], stdout=subprocess.PIPE) try: outs, errs = proc.communicate(timeout=20) except TimeOutExpired: proc.kill() outs, errs = proc.communicate() print('Command \'make\' timed out.') if proc.returncode: raise Exception('Command \'make\' failed with the above error.' ' Full command is:\n\n {}'.format(outs.decode())) INSTALL_DIR = os.path.dirname(__file__) + '/..' running_on = platform.system() if running_on == 'Linux': cmd = 'export LD_LIBRARY_PATH=' + INSTALL_DIR + '/lib/prometeo:' + \ INSTALL_DIR + '/lib/blasfeo:$LD_LIBRARY_PATH ; ./' + filename_ elif running_on == 'Darwin': cmd = 'export DYLD_LIBRARY_PATH=' + INSTALL_DIR + '/lib/prometeo:' + \ INSTALL_DIR + '/lib/blasfeo:$DYLD_LIBRARY_PATH ; ./' + filename_ else: raise Exception( 'Running on unsupported operating system {}'.format( running_on)) if red_stdout is not None: proc = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE) else: proc = subprocess.Popen([cmd], shell=True) try: outs, errs = proc.communicate() except TimeOutExpired: proc.kill() outs, errs = proc.communicate() print('Command \'make\' timed out.') if red_stdout is not None: with open(red_stdout, 'w') as f: f.write(outs) if red_stdout is not None: with open(red_stdout, 'w') as f: f.write(outs.decode()) if proc.returncode: raise Exception('Command {} failed with the above error.' ' Full command is:\n\n {}'.format( cmd, outs.decode()))
def process_file(f): print(f) out = strip_file_to_string(f) with open(f, 'w') as fh: fh.write(out)
def file_has_incompatible_typehints(file_path): return strip_hints.strip_file_to_string(file_path, to_empty=True, only_assigns_and_defs=True, only_test_for_changes=True)
def strip_hints_and_overwrite(file_path): transformed = strip_hints.strip_file_to_string(file_path, to_empty=True, only_assigns_and_defs=True) with open(file_path, "w") as f: f.write(transformed)