def do_tests(src_dir,
patterns,
target_dir,
start_with=None,
do_verify=False,
max_files=200):
def visitor(files, dirname, names):
files.extend([
os.path.normpath(os.path.join(dirname, n)) for n in names
for pat in patterns if fnmatch(n, pat)
])
files = []
cwd = os.getcwd()
os.chdir(src_dir)
if PYTHON3:
for root, dirname, names in os.walk(os.curdir):
files.extend([
os.path.normpath(os.path.join(root, n)) for n in names
for pat in patterns if fnmatch(n, pat)
])
pass
pass
else:
os.path.walk(os.curdir, visitor, files)
os.chdir(cwd)
files.sort()
if start_with:
try:
start_with = files.index(start_with)
files = files[start_with:]
print('>>> starting with file', files[0])
except ValueError:
pass
if len(files) > max_files:
files = [file for file in files if not 'site-packages' in file]
files = [file for file in files if not 'test' in file]
if len(files) > max_files:
# print("Number of files %d - truncating to last 200" % len(files))
print("Number of files %d - truncating to first %s" %
(len(files), max_files))
files = files[:max_files]
print(time.ctime())
(tot_files, okay_files, failed_files,
verify_failed_files) = main.main(src_dir,
target_dir,
files, [],
do_verify=do_verify)
print(time.ctime())
return verify_failed_files + failed_files
def process_func():
try:
(tot_files, okay_files, failed_files, verify_failed_files) = (0, 0, 0, 0)
while 1:
f = fqueue.get()
if f is None:
break
(t, o, f, v) = \
main(src_base, out_base, [f], None, outfile, **options)
tot_files += t
okay_files += o
failed_files += f
verify_failed_files += v
except (Empty, KeyboardInterrupt):
pass
rqueue.put((tot_files, okay_files, failed_files, verify_failed_files))
rqueue.close()
def main_bin():
if not (sys.version_info[0:2] in ((3, 7), (3, 8))):
print("Error: %s requires Python 3.7-3.8" % program, file=sys.stderr)
sys.exit(-1)
do_verify = recurse_dirs = False
numproc = 0
outfile = "-"
out_base = None
source_paths = []
timestamp = False
timestampfmt = "# %Y.%m.%d %H:%M:%S %Z"
try:
opts, pyc_paths = getopt.getopt(
sys.argv[1:],
"hac:gtTdrVo:p:",
"help asm compile= grammar linemaps recurse "
"timestamp tree= tree+ "
"fragments verify verify-run version "
"syntax-verify "
"showgrammar".split(" "),
)
except getopt.GetoptError as e:
print("%s: %s" % (os.path.basename(sys.argv[0]), e), file=sys.stderr)
sys.exit(-1)
options = {}
for opt, val in opts:
if opt in ("-h", "--help"):
print(__doc__)
sys.exit(0)
elif opt in ("-V", "--version"):
print("%s %s" % (program, VERSION))
sys.exit(0)
elif opt == "--syntax-verify":
options["do_verify"] = "weak"
elif opt == "--fragments":
options["do_fragments"] = True
elif opt == "--verify-run":
options["do_verify"] = "verify-run"
elif opt == "--linemaps":
options["do_linemaps"] = True
elif opt in ("--asm", "-a"):
options["showasm"] = "after"
options["do_verify"] = None
elif opt in ("--tree", "-t"):
if "showast" not in options:
options["showast"] = {}
if val == "before":
options["showast"][val] = True
elif val == "after":
options["showast"][val] = True
else:
options["showast"]["before"] = True
options["do_verify"] = None
elif opt in ("--tree+", "-T"):
if "showast" not in options:
options["showast"] = {}
options["showast"]["Full"] = True
options["do_verify"] = None
elif opt in ("--grammar", "-g"):
options["showgrammar"] = True
elif opt == "-o":
outfile = val
elif opt in ("--timestamp", "-d"):
timestamp = True
elif opt in ("--compile", "-c"):
source_paths.append(val)
elif opt == "-p":
numproc = int(val)
elif opt in ("--recurse", "-r"):
recurse_dirs = True
elif opt == "--encoding":
options["source_encoding"] = val
else:
print(opt, file=sys.stderr)
usage()
# expand directory if specified
if recurse_dirs:
expanded_files = []
for f in pyc_paths:
if os.path.isdir(f):
for root, _, dir_files in os.walk(f):
for df in dir_files:
if df.endswith(".pyc") or df.endswith(".pyo"):
expanded_files.append(os.path.join(root, df))
pyc_paths = expanded_files
# argl, commonprefix works on strings, not on path parts,
# thus we must handle the case with files in 'some/classes'
# and 'some/cmds'
src_base = os.path.commonprefix(pyc_paths)
if src_base[-1:] != os.sep:
src_base = os.path.dirname(src_base)
if src_base:
sb_len = len(os.path.join(src_base, ""))
pyc_paths = [f[sb_len:] for f in pyc_paths]
if not pyc_paths and not source_paths:
print("No input files given to decompile", file=sys.stderr)
usage()
if outfile == "-":
outfile = None # use stdout
elif outfile and os.path.isdir(outfile):
out_base = outfile
outfile = None
elif outfile and len(pyc_paths) > 1:
out_base = outfile
outfile = None
if timestamp:
print(time.strftime(timestampfmt))
if numproc <= 1:
try:
result = main(src_base, out_base, pyc_paths, source_paths, outfile,
**options)
result = list(result) + [options.get("do_verify", None)]
if len(pyc_paths) > 1:
mess = status_msg(do_verify, *result)
print("# " + mess)
pass
except (KeyboardInterrupt):
pass
else:
from multiprocessing import Process, Queue
try:
from Queue import Empty
except ImportError:
from queue import Empty
fqueue = Queue(len(pyc_paths) + numproc)
for f in pyc_paths:
fqueue.put(f)
for i in range(numproc):
fqueue.put(None)
rqueue = Queue(numproc)
def process_func():
try:
(tot_files, okay_files, failed_files, verify_failed_files) = (
0,
0,
0,
0,
)
while 1:
f = fqueue.get()
if f is None:
break
(t, o, f, v) = main(src_base, out_base, [f], None, outfile,
**options)
tot_files += t
okay_files += o
failed_files += f
verify_failed_files += v
except (Empty, KeyboardInterrupt):
pass
rqueue.put(
(tot_files, okay_files, failed_files, verify_failed_files))
rqueue.close()
try:
procs = [Process(target=process_func) for i in range(numproc)]
for p in procs:
p.start()
for p in procs:
p.join()
try:
(tot_files, okay_files, failed_files, verify_failed_files) = (
0,
0,
0,
0,
)
while True:
(t, o, f, v) = rqueue.get(False)
tot_files += t
okay_files += o
failed_files += f
verify_failed_files += v
except Empty:
pass
print(
"# decompiled %i files: %i okay, %i failed, %i verify failed" %
(tot_files, okay_files, failed_files, verify_failed_files))
except (KeyboardInterrupt, OSError):
pass
if timestamp:
print(time.strftime(timestampfmt))
return
def main_bin():
if not (sys.version_info[0:2] in ((2, 6), (2, 7), (3, 0), (3, 1), (3, 2),
(3, 3), (3, 4), (3, 5), (3, 6), (3, 7),
(3, 8))):
print('Error: %s requires Python 2.6-3.8' % program, file=sys.stderr)
sys.exit(-1)
do_verify = recurse_dirs = False
numproc = 0
outfile = '-'
out_base = None
source_paths = []
timestamp = False
timestampfmt = "# %Y.%m.%d %H:%M:%S %Z"
try:
opts, pyc_paths = getopt.getopt(
sys.argv[1:], 'hac:gtTdrVo:p:',
'help asm compile= grammar linemaps recurse '
'timestamp tree tree+ '
'fragments verify verify-run version '
'weak-verify '
'showgrammar'.split(' '))
except getopt.GetoptError as e:
print('%s: %s' % (os.path.basename(sys.argv[0]), e), file=sys.stderr)
sys.exit(-1)
options = {}
for opt, val in opts:
if opt in ('-h', '--help'):
print(__doc__)
sys.exit(0)
elif opt in ('-V', '--version'):
print("%s %s" % (program, VERSION))
sys.exit(0)
elif opt == '--verify':
options['do_verify'] = 'strong'
elif opt == '--weak-verify':
options['do_verify'] = 'weak'
elif opt == '--fragments':
options['do_fragments'] = True
elif opt == '--verify-run':
options['do_verify'] = 'verify-run'
elif opt == '--linemaps':
options['do_linemaps'] = True
elif opt in ('--asm', '-a'):
options['showasm'] = 'after'
options['do_verify'] = None
elif opt in ('--tree', '-t'):
options['showast'] = True
options['do_verify'] = None
elif opt in ('--tree+', '-T'):
options['showast'] = 'Full'
options['do_verify'] = None
elif opt in ('--grammar', '-g'):
options['showgrammar'] = True
elif opt == '-o':
outfile = val
elif opt in ('--timestamp', '-d'):
timestamp = True
elif opt in ('--compile', '-c'):
source_paths.append(val)
elif opt == '-p':
numproc = int(val)
elif opt in ('--recurse', '-r'):
recurse_dirs = True
else:
print(opt, file=sys.stderr)
usage()
# expand directory if specified
if recurse_dirs:
expanded_files = []
for f in pyc_paths:
if os.path.isdir(f):
for root, _, dir_files in os.walk(f):
for df in dir_files:
if df.endswith('.pyc') or df.endswith('.pyo'):
expanded_files.append(os.path.join(root, df))
pyc_paths = expanded_files
# argl, commonprefix works on strings, not on path parts,
# thus we must handle the case with files in 'some/classes'
# and 'some/cmds'
src_base = os.path.commonprefix(pyc_paths)
if src_base[-1:] != os.sep:
src_base = os.path.dirname(src_base)
if src_base:
sb_len = len(os.path.join(src_base, ''))
pyc_paths = [f[sb_len:] for f in pyc_paths]
if not pyc_paths and not source_paths:
print("No input files given to decompile", file=sys.stderr)
usage()
if outfile == '-':
outfile = None # use stdout
elif outfile and os.path.isdir(outfile):
out_base = outfile
outfile = None
elif outfile and len(pyc_paths) > 1:
out_base = outfile
outfile = None
if timestamp:
print(time.strftime(timestampfmt))
if numproc <= 1:
try:
result = main(src_base, out_base, pyc_paths, source_paths, outfile,
**options)
result = list(result) + [options.get('do_verify', None)]
if len(pyc_paths) > 1:
mess = status_msg(do_verify, *result)
print('# ' + mess)
pass
except (KeyboardInterrupt):
pass
except verify.VerifyCmpError:
raise
else:
from multiprocessing import Process, Queue
try:
from Queue import Empty
except ImportError:
from queue import Empty
fqueue = Queue(len(pyc_paths) + numproc)
for f in pyc_paths:
fqueue.put(f)
for i in range(numproc):
fqueue.put(None)
rqueue = Queue(numproc)
def process_func():
try:
(tot_files, okay_files, failed_files,
verify_failed_files) = (0, 0, 0, 0)
while 1:
f = fqueue.get()
if f is None:
break
(t, o, f, v) = \
main(src_base, out_base, [f], None, outfile, **options)
tot_files += t
okay_files += o
failed_files += f
verify_failed_files += v
except (Empty, KeyboardInterrupt):
pass
rqueue.put(
(tot_files, okay_files, failed_files, verify_failed_files))
rqueue.close()
try:
procs = [Process(target=process_func) for i in range(numproc)]
for p in procs:
p.start()
for p in procs:
p.join()
try:
(tot_files, okay_files, failed_files,
verify_failed_files) = (0, 0, 0, 0)
while True:
(t, o, f, v) = rqueue.get(False)
tot_files += t
okay_files += o
failed_files += f
verify_failed_files += v
except Empty:
pass
print(
'# decompiled %i files: %i okay, %i failed, %i verify failed' %
(tot_files, okay_files, failed_files, verify_failed_files))
except (KeyboardInterrupt, OSError):
pass
if timestamp:
print(time.strftime(timestampfmt))
return
def do_tests(src_dir, obj_patterns, target_dir, opts):
def file_matches(files, root, basenames, patterns):
files.extend(
[os.path.normpath(os.path.join(root, n))
for n in basenames
for pat in patterns
if fnmatch(n, pat)])
files = []
# Change directories so use relative rather than
# absolute paths. This speeds up things, and allows
# main() to write to a relative-path destination.
cwd = os.getcwd()
os.chdir(src_dir)
if opts['do_compile']:
compiled_version = opts['compiled_version']
if compiled_version and PYTHON_VERSION != compiled_version:
print("Not compiling: desired Python version is %s but we are running %s" %
(compiled_version, PYTHON_VERSION), file=sys.stderr)
else:
for root, dirs, basenames in os.walk(src_dir):
file_matches(files, root, basenames, PY)
for sfile in files:
py_compile.compile(sfile)
pass
pass
files = []
pass
pass
for root, dirs, basenames in os.walk('.'):
# Turn root into a relative path
dirname = root[2:] # 2 = len('.') + 1
file_matches(files, dirname, basenames, obj_patterns)
if not files:
print("Didn't come up with any files to test! Try with --compile?",
file=sys.stderr)
exit(1)
os.chdir(cwd)
files.sort()
if opts['start_with']:
try:
start_with = files.index(opts['start_with'])
files = files[start_with:]
print('>>> starting with file', files[0])
except ValueError:
pass
print(time.ctime())
print('Source directory: ', src_dir)
print('Output directory: ', target_dir)
try:
_, _, failed_files, failed_verify = \
main(src_dir, target_dir, files, [],
do_verify=opts['do_verify'])
if failed_files != 0:
sys.exit(2)
elif failed_verify != 0:
sys.exit(3)
except (KeyboardInterrupt, OSError):
print()
sys.exit(1)
if test_opts['rmtree']:
parent_dir = os.path.dirname(target_dir)
print("Everything good, removing %s" % parent_dir)
shutil.rmtree(parent_dir)