def do_tests(src_dir, patterns, target_dir, start_with=None, do_verify=False, max_files=200): def visitor(files, dirname, names): files.extend([ os.path.normpath(os.path.join(dirname, n)) for n in names for pat in patterns if fnmatch(n, pat) ]) files = [] cwd = os.getcwd() os.chdir(src_dir) if PYTHON3: for root, dirname, names in os.walk(os.curdir): files.extend([ os.path.normpath(os.path.join(root, n)) for n in names for pat in patterns if fnmatch(n, pat) ]) pass pass else: os.path.walk(os.curdir, visitor, files) os.chdir(cwd) files.sort() if start_with: try: start_with = files.index(start_with) files = files[start_with:] print(">>> starting with file", files[0]) except ValueError: pass if len(files) > max_files: files = [ file for file in files if not "site-packages" in file and ( file.endswith(".pyo") or file.endswith(".pyc")) ] files = [ file for file in files if not "test" in file and ( file.endswith(".pyo") or file.endswith(".pyc")) ] if len(files) > max_files: # print("Number of files %d - truncating to last 200" % len(files)) print("Number of files %d - truncating to first %s" % (len(files), max_files)) files = files[:max_files] print(time.ctime()) (tot_files, okay_files, failed_files, verify_failed_files) = main.main(src_dir, target_dir, files, [], do_verify=do_verify) print(time.ctime()) return verify_failed_files + failed_files
def do_tests(src_dir, patterns, target_dir, start_with=None, do_verify=False): def visitor(files, dirname, names): files.extend( [os.path.normpath(os.path.join(dirname, n)) for n in names for pat in patterns if fnmatch(n, pat)]) files = [] cwd = os.getcwd() os.chdir(src_dir) if PYTHON3: for root, dirname, names in os.walk(os.curdir): files.extend( [os.path.normpath(os.path.join(root, n)) for n in names for pat in patterns if fnmatch(n, pat)]) pass pass else: os.path.walk(os.curdir, visitor, files) os.chdir(cwd) files.sort() if start_with: try: start_with = files.index(start_with) files = files[start_with:] print('>>> starting with file', files[0]) except ValueError: pass if len(files) > 200: files = [file for file in files if not 'site-packages' in file] files = [file for file in files if not 'test' in file] if len(files) > 200: files = files[:200] print(time.ctime()) main.main(src_dir, target_dir, files, [], do_verify=do_verify) print(time.ctime())
def do_tests(src_dir, patterns, target_dir, start_with=None, do_verify=False): def visitor(files, dirname, names): files.extend([ os.path.normpath(os.path.join(dirname, n)) for n in names for pat in patterns if fnmatch(n, pat) ]) files = [] cwd = os.getcwd() os.chdir(src_dir) if PYTHON3: for root, dirname, names in os.walk(os.curdir): files.extend([ os.path.normpath(os.path.join(root, n)) for n in names for pat in patterns if fnmatch(n, pat) ]) pass pass else: os.path.walk(os.curdir, visitor, files) os.chdir(cwd) files.sort() if start_with: try: start_with = files.index(start_with) files = files[start_with:] print('>>> starting with file', files[0]) except ValueError: pass if len(files) > 200: files = [file for file in files if not 'site-packages' in file] files = [file for file in files if not 'test' in file] if len(files) > 200: files = files[:200] print(time.ctime()) main.main(src_dir, target_dir, files, [], do_verify=do_verify) print(time.ctime())
def process_func(): try: (tot_files, okay_files, failed_files, verify_failed_files) = (0, 0, 0, 0) while 1: f = fqueue.get() if f is None: break (t, o, f, v) = main(src_base, out_base, [f], codes, outfile, **options) tot_files += t okay_files += o failed_files += f verify_failed_files += v except (Empty, KeyboardInterrupt): pass rqueue.put((tot_files, okay_files, failed_files, verify_failed_files)) rqueue.close()
def process_func(): try: (tot_files, okay_files, failed_files, verify_failed_files) = (0, 0, 0, 0) while 1: f = fqueue.get() if f is None: break (t, o, f, v) = \ main(src_base, out_base, [f], None, outfile, **options) tot_files += t okay_files += o failed_files += f verify_failed_files += v except (Empty, KeyboardInterrupt): pass rqueue.put((tot_files, okay_files, failed_files, verify_failed_files)) rqueue.close()
def main_bin(): if not (sys.version_info[0:2] in ((2, 6), (2, 7), (3, 0), (3, 1), (3, 2), (3, 3), (3, 4), (3, 5), (3, 6), (3, 7), (3, 8))): print('Error: %s requires Python 2.6-3.8' % program, file=sys.stderr) sys.exit(-1) do_verify = recurse_dirs = False numproc = 0 outfile = '-' out_base = None source_paths = [] timestamp = False timestampfmt = "# %Y.%m.%d %H:%M:%S %Z" try: opts, pyc_paths = getopt.getopt( sys.argv[1:], 'hac:gtTdrVo:p:', 'help asm compile= grammar linemaps recurse ' 'timestamp tree= tree+ ' 'fragments verify verify-run version ' 'syntax-verify ' 'showgrammar encoding='.split(' ')) except getopt.GetoptError as e: print('%s: %s' % (os.path.basename(sys.argv[0]), e), file=sys.stderr) sys.exit(-1) options = {} for opt, val in opts: if opt in ('-h', '--help'): print(__doc__) sys.exit(0) elif opt in ('-V', '--version'): print("%s %s" % (program, __version__)) sys.exit(0) elif opt == '--verify': options['do_verify'] = 'strong' elif opt == '--syntax-verify': options['do_verify'] = 'weak' elif opt == '--fragments': options['do_fragments'] = True elif opt == '--verify-run': options['do_verify'] = 'verify-run' elif opt == '--linemaps': options['do_linemaps'] = True elif opt in ('--asm', '-a'): options['showasm'] = 'after' options['do_verify'] = None elif opt in ('--tree', '-t'): if 'showast' not in options: options['showast'] = {} if val == 'before': options['showast'][val] = True elif val == 'after': options['showast'][val] = True else: options['showast']['before'] = True options['do_verify'] = None elif opt in ('--tree+', '-T'): if 'showast' not in options: options['showast'] = {} options['showast']['Full'] = True options['do_verify'] = None elif opt in ('--grammar', '-g'): options['showgrammar'] = True elif opt == '-o': outfile = val elif opt in ('--timestamp', '-d'): timestamp = True elif opt in ('--compile', '-c'): source_paths.append(val) elif opt == '-p': numproc = int(val) elif opt in ('--recurse', '-r'): recurse_dirs = True elif opt == '--encoding': options['source_encoding'] = val else: print(opt, file=sys.stderr) usage() # expand directory if specified if recurse_dirs: expanded_files = [] for f in pyc_paths: if os.path.isdir(f): for root, _, dir_files in os.walk(f): for df in dir_files: if df.endswith('.pyc') or df.endswith('.pyo'): expanded_files.append(os.path.join(root, df)) pyc_paths = expanded_files # argl, commonprefix works on strings, not on path parts, # thus we must handle the case with files in 'some/classes' # and 'some/cmds' src_base = os.path.commonprefix(pyc_paths) if src_base[-1:] != os.sep: src_base = os.path.dirname(src_base) if src_base: sb_len = len(os.path.join(src_base, '')) pyc_paths = [f[sb_len:] for f in pyc_paths] if not pyc_paths and not source_paths: print("No input files given to decompile", file=sys.stderr) usage() if outfile == '-': outfile = None # use stdout elif outfile and os.path.isdir(outfile): out_base = outfile outfile = None elif outfile and len(pyc_paths) > 1: out_base = outfile outfile = None if timestamp: print(time.strftime(timestampfmt)) if numproc <= 1: try: result = main(src_base, out_base, pyc_paths, source_paths, outfile, **options) result = list(result) + [options.get('do_verify', None)] if len(pyc_paths) > 1: mess = status_msg(do_verify, *result) print('# ' + mess) pass except (KeyboardInterrupt): pass except verify.VerifyCmpError: raise else: from multiprocessing import Process, Queue try: from Queue import Empty except ImportError: from queue import Empty fqueue = Queue(len(pyc_paths) + numproc) for f in pyc_paths: fqueue.put(f) for i in range(numproc): fqueue.put(None) rqueue = Queue(numproc) def process_func(): try: (tot_files, okay_files, failed_files, verify_failed_files) = (0, 0, 0, 0) while 1: f = fqueue.get() if f is None: break (t, o, f, v) = \ main(src_base, out_base, [f], None, outfile, **options) tot_files += t okay_files += o failed_files += f verify_failed_files += v except (Empty, KeyboardInterrupt): pass rqueue.put( (tot_files, okay_files, failed_files, verify_failed_files)) rqueue.close() try: procs = [Process(target=process_func) for i in range(numproc)] for p in procs: p.start() for p in procs: p.join() try: (tot_files, okay_files, failed_files, verify_failed_files) = (0, 0, 0, 0) while True: (t, o, f, v) = rqueue.get(False) tot_files += t okay_files += o failed_files += f verify_failed_files += v except Empty: pass print( '# decompiled %i files: %i okay, %i failed, %i verify failed' % (tot_files, okay_files, failed_files, verify_failed_files)) except (KeyboardInterrupt, OSError): pass if timestamp: print(time.strftime(timestampfmt)) return
def do_tests(src_dir, obj_patterns, target_dir, opts): def file_matches(files, root, basenames, patterns): files.extend( [os.path.normpath(os.path.join(root, n)) for n in basenames for pat in patterns if fnmatch(n, pat)]) files = [] # Change directories so use relative rather than # absolute paths. This speeds up things, and allows # main() to write to a relative-path destination. cwd = os.getcwd() os.chdir(src_dir) if opts['do_compile']: compiled_version = opts['compiled_version'] if compiled_version and PYTHON_VERSION != compiled_version: print("Not compiling: desired Python version is %s but we are running %s" % (compiled_version, PYTHON_VERSION), file=sys.stderr) else: for root, dirs, basenames in os.walk(src_dir): file_matches(files, root, basenames, PY) for sfile in files: py_compile.compile(sfile) pass pass files = [] pass pass for root, dirs, basenames in os.walk('.'): # Turn root into a relative path dirname = root[2:] # 2 = len('.') + 1 file_matches(files, dirname, basenames, obj_patterns) if not files: print("Didn't come up with any files to test! Try with --compile?", file=sys.stderr) exit(1) os.chdir(cwd) files.sort() if opts['start_with']: try: start_with = files.index(opts['start_with']) files = files[start_with:] print('>>> starting with file', files[0]) except ValueError: pass print(time.ctime()) print('Source directory: ', src_dir) print('Output directory: ', target_dir) try: _, _, failed_files, failed_verify = \ main(src_dir, target_dir, files, [], do_verify=opts['do_verify']) if failed_files != 0: exit(2) elif failed_verify != 0: exit(3) except (KeyboardInterrupt, OSError): print() exit(1) if test_opts['rmtree']: parent_dir = os.path.dirname(target_dir) print("Everything good, removing %s" % parent_dir) shutil.rmtree(parent_dir)
def main_bin(): if not (sys.version_info[0:2] in ((2, 6), (2, 7), (3, 1), (3, 2), (3, 3), (3, 4), (3, 5), (3, 6))): print('Error: %s requires Python 2.6-2.7, or 3.1-3.6' % program, file=sys.stderr) sys.exit(-1) do_verify = recurse_dirs = False numproc = 0 outfile = '-' out_base = None codes = [] timestamp = False timestampfmt = "# %Y.%m.%d %H:%M:%S %Z" try: opts, files = getopt.getopt(sys.argv[1:], 'hagtdrVo:c:p:', 'help asm grammar linemaps recurse timestamp tree ' 'verify version showgrammar'.split(' ')) except getopt.GetoptError as e: print('%s: %s' % (os.path.basename(sys.argv[0]), e), file=sys.stderr) sys.exit(-1) options = {} for opt, val in opts: if opt in ('-h', '--help'): print(__doc__) sys.exit(0) elif opt in ('-V', '--version'): print("%s %s" % (program, VERSION)) sys.exit(0) elif opt == '--verify': options['do_verify'] = True elif opt == '--linemaps': options['do_linemaps'] = True elif opt in ('--asm', '-a'): options['showasm'] = 'after' options['do_verify'] = False elif opt in ('--tree', '-t'): options['showast'] = True options['do_verify'] = False elif opt in ('--grammar', '-g'): options['showgrammar'] = True elif opt == '-o': outfile = val elif opt in ('--timestamp', '-d'): timestamp = True elif opt == '-c': codes.append(val) elif opt == '-p': numproc = int(val) elif opt in ('--recurse', '-r'): recurse_dirs = True else: print(opt, file=sys.stderr) usage() # expand directory if specified if recurse_dirs: expanded_files = [] for f in files: if os.path.isdir(f): for root, _, dir_files in os.walk(f): for df in dir_files: if df.endswith('.pyc') or df.endswith('.pyo'): expanded_files.append(os.path.join(root, df)) files = expanded_files # argl, commonprefix works on strings, not on path parts, # thus we must handle the case with files in 'some/classes' # and 'some/cmds' src_base = os.path.commonprefix(files) if src_base[-1:] != os.sep: src_base = os.path.dirname(src_base) if src_base: sb_len = len( os.path.join(src_base, '') ) files = [f[sb_len:] for f in files] if not files: print("No files given", file=sys.stderr) usage() if outfile == '-': outfile = None # use stdout elif outfile and os.path.isdir(outfile): out_base = outfile; outfile = None elif outfile and len(files) > 1: out_base = outfile; outfile = None if timestamp: print(time.strftime(timestampfmt)) if numproc <= 1: try: result = main(src_base, out_base, files, codes, outfile, **options) if len(files) > 1: mess = status_msg(do_verify, *result) print('# ' + mess) pass except (KeyboardInterrupt): pass except verify.VerifyCmpError: raise else: from multiprocessing import Process, Queue try: from Queue import Empty except ImportError: from Queue import Empty fqueue = Queue(len(files)+numproc) for f in files: fqueue.put(f) for i in range(numproc): fqueue.put(None) rqueue = Queue(numproc) def process_func(): try: (tot_files, okay_files, failed_files, verify_failed_files) = (0, 0, 0, 0) while 1: f = fqueue.get() if f is None: break (t, o, f, v) = \ main(src_base, out_base, [f], codes, outfile, **options) tot_files += t okay_files += o failed_files += f verify_failed_files += v except (Empty, KeyboardInterrupt): pass rqueue.put((tot_files, okay_files, failed_files, verify_failed_files)) rqueue.close() try: procs = [Process(target=process_func) for i in range(numproc)] for p in procs: p.start() for p in procs: p.join() try: (tot_files, okay_files, failed_files, verify_failed_files) = (0, 0, 0, 0) while True: (t, o, f, v) = rqueue.get(False) tot_files += t okay_files += o failed_files += f verify_failed_files += v except Empty: pass print('# decompiled %i files: %i okay, %i failed, %i verify failed' % (tot_files, okay_files, failed_files, verify_failed_files)) except (KeyboardInterrupt, OSError): pass if timestamp: print(time.strftime(timestampfmt)) return
def main_bin(): if not (sys.version_info[0:2] in ((2, 6), (2, 7), (3, 2), (3, 3), (3, 4), (3, 5), (3, 6))): print("Error: %s requires Python 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, or 3.6" % program, file=sys.stderr) sys.exit(-1) do_verify = recurse_dirs = False numproc = 0 outfile = "-" out_base = None codes = [] timestamp = False timestampfmt = "# %Y.%m.%d %H:%M:%S %Z" try: opts, files = getopt.getopt( sys.argv[1:], "hagtdrVo:c:p:", "help asm grammar recurse timestamp tree verify version " "showgrammar".split(" "), ) except getopt.GetoptError as e: print("%s: %s" % (os.path.basename(sys.argv[0]), e), file=sys.stderr) sys.exit(-1) options = {} for opt, val in opts: if opt in ("-h", "--help"): print(__doc__) sys.exit(0) elif opt in ("-V", "--version"): print("%s %s" % (program, VERSION)) sys.exit(0) elif opt == "--verify": options["do_verify"] = True elif opt in ("--asm", "-a"): options["showasm"] = True options["do_verify"] = False elif opt in ("--tree", "-t"): options["showast"] = True options["do_verify"] = False elif opt in ("--grammar", "-g"): options["showgrammar"] = True elif opt == "-o": outfile = val elif opt in ("--timestamp", "-d"): timestamp = True elif opt == "-c": codes.append(val) elif opt == "-p": numproc = int(val) elif opt in ("--recurse", "-r"): recurse_dirs = True else: print(opt, file=sys.stderr) usage() # expand directory if specified if recurse_dirs: expanded_files = [] for f in files: if os.path.isdir(f): for root, _, dir_files in os.walk(f): for df in dir_files: if df.endswith(".pyc") or df.endswith(".pyo"): expanded_files.append(os.path.join(root, df)) files = expanded_files # argl, commonprefix works on strings, not on path parts, # thus we must handle the case with files in 'some/classes' # and 'some/cmds' src_base = os.path.commonprefix(files) if src_base[-1:] != os.sep: src_base = os.path.dirname(src_base) if src_base: sb_len = len(os.path.join(src_base, "")) files = [f[sb_len:] for f in files] del sb_len if not files: print("No files given", file=sys.stderr) usage() if outfile == "-": if "do_verify" in options and len(files) == 1: junk, outfile = tempfile.mkstemp(suffix=".pyc", prefix=files[0][0:-4] + "-") else: outfile = None # use stdout elif outfile and os.path.isdir(outfile): out_base = outfile outfile = None elif outfile and len(files) > 1: out_base = outfile outfile = None if timestamp: print(time.strftime(timestampfmt)) if numproc <= 1: try: result = main(src_base, out_base, files, codes, outfile, **options) if len(files) > 1: mess = status_msg(do_verify, *result) print("# " + mess) pass except (KeyboardInterrupt): pass except verify.VerifyCmpError: raise else: from multiprocessing import Process, Queue try: from Queue import Empty except ImportError: from Queue import Empty fqueue = Queue(len(files) + numproc) for f in files: fqueue.put(f) for i in range(numproc): fqueue.put(None) rqueue = Queue(numproc) def process_func(): try: (tot_files, okay_files, failed_files, verify_failed_files) = (0, 0, 0, 0) while 1: f = fqueue.get() if f is None: break (t, o, f, v) = main(src_base, out_base, [f], codes, outfile, **options) tot_files += t okay_files += o failed_files += f verify_failed_files += v except (Empty, KeyboardInterrupt): pass rqueue.put((tot_files, okay_files, failed_files, verify_failed_files)) rqueue.close() try: procs = [Process(target=process_func) for i in range(numproc)] for p in procs: p.start() for p in procs: p.join() try: (tot_files, okay_files, failed_files, verify_failed_files) = (0, 0, 0, 0) while True: (t, o, f, v) = rqueue.get(False) tot_files += t okay_files += o failed_files += f verify_failed_files += v except Empty: pass print( "# decompiled %i files: %i okay, %i failed, %i verify failed" % (tot_files, okay_files, failed_files, verify_failed_files) ) except (KeyboardInterrupt, OSError): pass if timestamp: print(time.strftime(timestampfmt)) return