def execute_on(self, view: ProjectView, build_output_dir: str,
build_manager):
try:
import python_minifier
except ImportError:
subprocess.Popen(
[sys.executable, "-m", "pip", "install", "python-minifier"])
import python_minifier
for file in view.modified_files:
if file.endswith(".py"):
if file not in self.special_config:
try:
view.write(
file,
python_minifier.minify(
view.read(file).decode("utf-8"),
preserve_locals=["NAME"],
remove_literal_statements=True,
).encode("utf-8"),
)
except:
print(file)
raise
else:
view.write(
file,
python_minifier.minify(
view.read(file).decode("utf-8"),
**self.special_config[file]).encode("utf-8"),
)
def import_lambda(name):
TK_IN_LBD = "IBOX_CODE_IN_LAMBDA"
parent_dir_name = os.path.dirname(os.path.realpath(__file__))
lambda_file = os.path.join(os.getcwd(), "lib/lambdas/%s.code" % name)
if not os.path.exists(lambda_file):
lambda_file = os.path.join(parent_dir_name, "lambdas/%s.code" % name)
try:
lambda_file_trunk = os.path.join(*PurePath(lambda_file).parts[-3:])
with open(lambda_file, "r") as f:
fdata = f.read()
try:
# try to minify using python_minifier
code = python_minifier.minify(fdata)
if len(code) > 4096:
logging.warning(
f"{lambda_file_trunk} > 4096, trying to minify it using a more aggressive option [rename_globals=True]"
)
code = python_minifier.minify(
fdata,
rename_globals=True,
preserve_globals=["lambda_handler"])
except Exception:
code = fdata
if len(code) > 4096:
logging.warning(f"Inline lambda {lambda_file_trunk} > 4096")
code_lines = code.splitlines(keepends=True)
file_lines = []
# parse lambda code for Token IBOX CODE
for x in code_lines:
if x.startswith(cfg.EVAL_FUNCTIONS_IN_CFG):
value = eval(x)
elif x.startswith(TK_IN_LBD):
value = '"'
elif TK_IN_LBD in x:
# parse minified code
tks = x.split(TK_IN_LBD)
file_lines.extend([f"{tks[0]}", eval(tks[1]), tks[2]])
continue
else:
value = "".join(x)
file_lines.append(value)
return file_lines
except IOError:
logging.warning(f"Lambda code {name} not found")
exit(1)
except Exception as e:
logging.error(e)
def _minify_and_compile(file):
with open(file) as f:
buff = f.read()
with open(file, "w") as f:
mnf = python_minifier.minify(buff)
f.write(mnf)
compileall.compile_file(file, force=True, quiet=True, legacy=True)
def run(self):
"""Run command."""
import python_minifier
for file in os.listdir(str(__dir__ / 'src' / 'umqtt')):
with open(str(__dir__ / 'src' / 'umqtt' / file)) as f:
print('Minify: %s' % file)
source = f.read()
filename = file.split('/')[-1]
out = python_minifier.minify(
source,
filename,
remove_annotations=True,
remove_pass=True,
remove_literal_statements=True,
combine_imports=False,
hoist_literals=False,
rename_locals=True,
preserve_locals=None,
rename_globals=False,
preserve_globals=None,
remove_object_base=False,
convert_posargs_to_args=False
)
with open(str(Path(self.minified_dir) / 'umqtt' / file), 'w') as f:
f.write(out)
def run_test(self):
from sh import Command, ErrorReturnCode
ErrorReturnCode.truncate_cap = 1000
def execute(python, path):
python = Command(python)
python(path)
try:
with open(self.test_path, 'rb') as f:
source = f.read()
shutil.copy(self.test_path, self.test_path + '.bak')
with open(self.test_path, 'wb') as f:
f.write(minify(source, self.test_path, **self.options).encode())
execute(sys.executable, self.test_path)
except ErrorReturnCode as e:
print(self.test_path)
print(e.stderr)
raise
finally:
shutil.copy(self.test_path + '.bak', self.test_path)
def process(filename, exclude_dirs=['test', 'site-packages']):
"""Process a VFS filename for Brython."""
print("Generating {}".format(filename))
nb = 0
nb_err = 0
main_root = os.path.dirname(filename)
VFS = {}
for stdlib_dir in ("libs", "Lib"):
for root, _dir, files in os.walk(os.path.join(main_root, stdlib_dir)):
flag = False
root_elts = root.split(os.sep)
for exclude in exclude_dirs:
if exclude in root_elts:
flag = True
continue
if flag:
continue # skip these modules
if '__pycache__' in root:
continue
nb += 1
for _file in files:
ext = os.path.splitext(_file)[1]
if ext not in ('.js', '.py'):
continue
if re.match(r'^module\d+\..*$', _file):
continue
nb += 1
file_name = os.path.join(root, _file)
with open(file_name, encoding='utf-8') as f:
data = f.read()
if ext == '.py':
data = python_minifier.minify(data, preserve_lines=True)
vfs_path = os.path.join(root, _file).replace(main_root, '')
vfs_path = vfs_path.replace("\\", "/")
if vfs_path.startswith('/libs/crypto_js/rollups/'):
if _file not in ('md5.js', 'sha1.js', 'sha3.js',
'sha224.js', 'sha384.js', 'sha512.js'):
continue
mod_name = vfs_path[len(stdlib_dir) + 2:].replace('/', '.')
mod_name, ext = os.path.splitext(mod_name)
is_package = mod_name.endswith('__init__')
if is_package:
mod_name = mod_name[:-9]
VFS[mod_name] = [ext, data, 1]
else:
VFS[mod_name] = [ext, data]
print("adding {}".format(mod_name))
print('{} files, {} errors'.format(nb, nb_err))
with open(filename, "w") as out:
out.write('__BRYTHON__.use_VFS = true;\n')
out.write('__BRYTHON__.VFS={}\n\n'.format(json.dumps(VFS)))
def process(filename, exclude_dirs=[
'unittest',
'test',
]):
"""Process a VFS filename for Brython."""
print("Generating {}".format(filename))
nb = 0
nb_err = 0
_main_root = os.path.dirname(filename)
_VFS = {}
for _mydir in ("libs", "Lib"):
for _root, _dir, _files in os.walk(os.path.join(_main_root, _mydir)):
#if _root.endswith('lib_migration'):
_flag = False
for _exclude in exclude_dirs:
if _exclude in _root: #_root.endswith(_exclude):
_flag = True
continue
if _flag:
continue # skip these modules
if '__pycache__' in _root:
continue
nb += 1
for _file in _files:
_ext = os.path.splitext(_file)[1]
if _ext not in ('.js', '.py'):
continue
nb += 1
file_name = os.path.join(_root, _file)
_data = open(file_name, encoding='utf-8').read()
if _ext == '.py':
_data = python_minifier.minify(_data)
_vfs_filename = os.path.join(_root,
_file).replace(_main_root, '')
_vfs_filename = _vfs_filename.replace("\\", "/")
if _vfs_filename.startswith('/libs/crypto_js/rollups/'):
if _file not in ('md5.js', 'sha1.js', 'sha3.js',
'sha224.js', 'sha384.js', 'sha512.js'):
continue
mod_name = _vfs_filename[len(_mydir) + 2:].replace('/', '.')
mod_name, ext = os.path.splitext(mod_name)
is_package = mod_name.endswith('__init__')
if is_package:
mod_name = mod_name[:-9]
_VFS[mod_name] = [ext, _data, 1]
else:
_VFS[mod_name] = [ext, _data]
print(("adding %s %s" % (mod_name, _vfs_filename)))
print('%s files, %s errors' % (nb, nb_err))
with open(filename, "w") as file_to_write_VFS:
file_to_write_VFS.write('__BRYTHON__.use_VFS = true;\n')
file_to_write_VFS.write('__BRYTHON__.VFS=%s;\n\n' % json.dumps(_VFS))
def test_file(path):
try:
with open(path, 'rb') as f:
source = f.read()
except IOError:
pytest.skip('IOError opening file')
try:
original_ast = ast.parse(source, path)
except SyntaxError:
pytest.skip('Invalid syntax in file')
# Test unparsing
unparse(original_ast)
# Test transforms
minify(source, filename=path)
def process(filename, exclude_dirs=['unittest','test','site-packages']):
"""Process a VFS filename for Brython."""
print("Generating {}".format(filename))
nb = 0
nb_err = 0
_main_root = os.path.dirname(filename)
_VFS = {}
for _mydir in ("libs", "Lib"):
for _root, _dir, _files in os.walk(os.path.join(_main_root, _mydir)):
#if _root.endswith('lib_migration'):
_flag=False
for _exclude in exclude_dirs:
if _exclude in _root: #_root.endswith(_exclude):
_flag=True
continue
if _flag:
continue # skip these modules
if '__pycache__' in _root:
continue
nb += 1
for _file in _files:
_ext = os.path.splitext(_file)[1]
if _ext not in ('.js', '.py'):
continue
if re.match(r'^module\d+\..*$', _file):
continue
nb += 1
file_name = os.path.join(_root, _file)
_data = open(file_name, encoding='utf-8').read()
if _ext == '.py':
_data = python_minifier.minify(_data, preserve_lines=True)
_vfs_filename = os.path.join(_root, _file).replace(_main_root, '')
_vfs_filename = _vfs_filename.replace("\\", "/")
if _vfs_filename.startswith('/libs/crypto_js/rollups/'):
if _file not in ('md5.js', 'sha1.js', 'sha3.js',
'sha224.js', 'sha384.js', 'sha512.js'):
continue
mod_name = _vfs_filename[len(_mydir) + 2:].replace('/', '.')
mod_name, ext = os.path.splitext(mod_name)
is_package = mod_name.endswith('__init__')
if is_package:
mod_name = mod_name[:-9]
_VFS[mod_name] = [ext, _data, 1]
else:
_VFS[mod_name] = [ext, _data]
print(("adding %s %s" % (mod_name, _vfs_filename)))
print('%s files, %s errors' % (nb, nb_err))
with open(filename, "w") as file_to_write_VFS:
file_to_write_VFS.write('__BRYTHON__.use_VFS = true;\n')
file_to_write_VFS.write('__BRYTHON__.VFS=%s;\n\n' % json.dumps(_VFS))
def create_vfs(path,
exclude_dirs=['test', 'site-packages'],
progress_func=print):
ret = {}
def skip(p):
for ex in exclude_dirs:
if ex in p.parts:
return True
if '__pycache__' in p.parts:
return True
if p.is_file():
if p.suffix not in ('.js', '.py'):
return True
if re.match(r'^module\d+\..*$', p.name):
return True
if '/'.join(p.parts[-4:-1]) == 'libs/crypto_js/rollups':
if p.name not in ('md5.js', 'sha1.js', 'sha3.js', 'sha224.js',
'sha384.js', 'sha512.js'):
return True
return False
nb = 0
for subdir in ("libs", "Lib"):
for p in walk(path / subdir, skip):
nb += 1
vfs_path = p.relative_to(path)
data = p.read_text(encoding='utf-8')
if p.suffix == '.py':
tree = ast.parse(data)
visitor = Visitor(path / subdir, vfs_path.pargs[1:])
visitor.visit(tree)
imports = sorted(list(visitor.imports))
data = python_minifier.minify(data, preserve_lines=True)
parent_mod_name = '.'.join(vfs_path.parts[1:-1])
if p.stem == '__init__':
ret[parent_mod_name] = [p.suffix, data, imports, 1]
progress_func("Adding package {}".format(parent_mod_name))
else:
if parent_mod_name:
ret[parent_mod_name + '.' + p.stem] = [p.suffix, data]
if p.suffix == '.py':
ret[parent_mod_name + '.' + p.stem].append(imports)
progress_func("Adding {}".format(parent_mod_name + '.' +
p.stem))
else:
ret[p.stem] = [p.suffix, data]
if p.suffix == '.py':
ret[parent_mod_name + '.' + p.stem].append(imports)
progress_func("Adding {}".format(p.stem))
return ret, nb
def add_state_machine_handler(template: troposphere.Template):
# minify so it fits in 4096 bytes and we don't need to upload to S3 first
code = python_minifier.minify(open("lambda/rds.py").read(),
"lambda/rds.py",
rename_globals=True,
preserve_globals=["handler", "NotReady"])
return add_lambda(
template,
"Handler",
code,
HANDLER_POLICIES,
Timeout=30,
)
def __minify_files(self: object, files_to_minify: str) -> bool:
for file in files_to_minify:
try:
with open(file, 'r') as data:
# , rename_globals=True, rename_locals=True, hoist_literals=True
minified_content = minify(data.read())
if minified_content:
with open(f'build\\{file}', 'w') as data:
data.write(minified_content)
self.__push_msg_(f'Minified {file}')
except Exception as err_obj:
self.__push_msg_(f'Error: {err_obj}', type='err')
return False
return True
def _zip_codedir(self, home, ziph):
home = home.rstrip('/')
total_files = {}
zip_relpaths = set()
for root, _, files in os.walk(home):
for file in files:
filepath = join(root, file)
zip_relpath = relpath(filepath, home)
zip_relpaths.add(zip_relpath)
total_files[zip_relpath] = filepath
src_zip_relpaths = zip_relpaths.copy()
ignore_patterns = self.get_ignore_patterns(home)
for ignore_pattern in ignore_patterns:
if ignore_pattern.startswith('!'):
ignore_pattern = ignore_pattern[1:]
zip_relpaths.update(
fnmatch.filter(src_zip_relpaths, ignore_pattern))
continue
zip_relpaths -= set(fnmatch.filter(zip_relpaths, ignore_pattern))
minify_relpaths = set()
minify_patterns = self.get_minify_patterns(home)
for minify_pattern in minify_patterns:
if minify_pattern.startswith('!'):
minify_pattern = minify_pattern[1:]
minify_relpaths -= set(
fnmatch.filter(minify_relpaths, minify_pattern))
continue
minify_relpaths.update(
fnmatch.filter(src_zip_relpaths, minify_pattern))
# 仅支持混淆 .py 文件
minify_relpaths = fnmatch.filter(minify_relpaths, '*.py')
with console.status('正在打包模型源码...', spinner='dots'):
for zip_relpath in sorted(zip_relpaths):
filepath = total_files[zip_relpath]
if zip_relpath in minify_relpaths:
continue
ziph.write(filepath, zip_relpath)
output('\t{}'.format(zip_relpath))
for zip_relpath in sorted(minify_relpaths):
filepath = total_files[zip_relpath]
with open(filepath, 'r') as fp:
content = fp.read()
with tempfile.NamedTemporaryFile(mode='w') as temp_fp:
content = python_minifier.minify(content)
temp_fp.write(content)
temp_fp.flush()
temp_fp.seek(0)
ziph.write(temp_fp.name, zip_relpath)
output('\t{} [green]MINIFIED[/green]'.format(zip_relpath))
async def minify(self, ctx):
"""Minify a python file.
You need to attach a file to this command, and it's extension needs to be `.py`.
"""
await ctx.trigger_typing()
if not ctx.message.attachments:
return await ctx.send_help()
file = ctx.message.attachments[0]
if not file.filename.lower().endswith(".py"):
return await ctx.send("Must be a python file.")
converted = io.BytesIO(minifier.minify(await file.read()).encode())
content = "Please see the attached file below, with your minimized code."
await ctx.send(content=content,
file=discord.File(converted,
filename=file.filename.lower()))
def create_example(option):
options = {
'combine_imports': False,
'hoist_literals': False,
'remove_annotations': False,
'remove_literal_statements': False,
'remove_pass': False,
'rename_globals': False,
'rename_locals': False,
'remove_object_base': False
}
options[option] = True
with open(f'transforms/{option}.py') as source:
with open(f'transforms/{option}.min.py', 'w') as minified:
minified.write(
minify(source.read(), filename=f'{option}.py', **options))
def minify_python(filepath):
""" Minify a Python File """
try:
if not os.path.isfile(filepath):
return False
if not filepath.endswith(".py"):
return False
print("-- Minifying %s" % filepath)
with open(filepath, "r") as fileread:
filecontent = fileread.read()
fileread.close()
filemini = python_minifier.minify(filecontent)
with open(filepath, "w") as filewrite:
filewrite.write(filemini)
filewrite.close()
return True
except Exception:
print("-- Unable to minify %s" % filepath)
return False
def minify_python(filepath):
""" Minify a Python File """
try:
if not os.path.isfile(filepath):
return False
if not filepath.endswith(".py"):
return False
print("-- Minifying %s" % filepath)
with open(filepath, "r") as fileread:
filecontent = fileread.read()
fileread.close()
filemini = python_minifier.minify(filecontent)
with open(filepath, "w") as filewrite:
filewrite.write(filemini)
filewrite.close()
return True
except Exception:
print("-- Unable to minify %s" % filepath)
return False
async def minify(self, ctx):
"""Minify a python file.
You need to attach a file to this command, and it's extension needs to be `.py`.
"""
await ctx.trigger_typing()
if not ctx.message.attachments:
return await ctx.send_help()
file = ctx.message.attachments[0]
file_name = file.filename.lower()
if not file_name.endswith((".py", ".python")):
return await ctx.send("Must be a python file.")
with contextlib.suppress(UnicodeDecodeError, UnicodeEncodeError):
file = await file.read()
converted = io.BytesIO(minifier.minify(file).encode(encoding="utf-8"))
content = "Please see the attached file below, with your minimized code."
return await ctx.send(
content=content, file=discord.File(converted, filename=file_name)
)
return await ctx.send("The file provided was in an unsupported format.")
async def minify(self, ctx):
"""Minify a python file.
You need to attach a file to this command, and it's extension needs to be `.py`.
"""
await ctx.trigger_typing()
if not ctx.message.attachments:
return await ctx.send_help()
file = ctx.message.attachments[0]
file_name = file.filename.lower()
if not file_name.endswith((".py", ".python")):
return await ctx.send("Must be a python file.")
try:
file = await file.read()
except UnicodeDecodeError:
return await ctx.send("Something went wrong when trying to decode this file.")
converted = io.BytesIO(minifier.minify(file).encode(encoding="utf-8"))
return await ctx.send(
content="Please see the attached file below, with your minified code.",
file=discord.File(converted, filename=file_name),
)
def minify_python(s):
"""
Minify python code.
@param s: python code to minify
@type s: L{str}
@return: the minified python code
@rtype: L{str}
"""
# not working with brython :(
return s
if python_minifier is None:
raise NotImplementedError(
"Dependency 'python-minifier' required, but not found!")
return python_minifier.minify(
s,
remove_annotations=True,
remove_pass=True,
# remove_literal_statements=
combine_imports=True,
hoist_literals=True,
rename_locals=True,
rename_globals=True,
)
def process(filename, exclude_dirs=['test','site-packages']):
"""Process a VFS filename for Brython."""
print("Generating {}".format(filename))
nb = 0
nb_err = 0
main_root = os.path.dirname(filename)
VFS = {}
for stdlib_dir in ("libs", "Lib"):
lib_path = os.path.join(main_root, stdlib_dir)
for root, _dir, files in os.walk(lib_path):
flag = False
root_elts = root.split(os.sep)
for exclude in exclude_dirs:
if exclude in root_elts:
flag = True
continue
if flag:
continue # skip these modules
if '__pycache__' in _dir:
_dir.remove("__pycache__")
nb += 1
if stdlib_dir == "Lib":
if root == lib_path:
package = []
else:
package = root[len(lib_path) + 1:].split(os.sep)
for _file in files:
ext = os.path.splitext(_file)[1]
if ext not in ('.js', '.py'):
continue
if re.match(r'^module\d+\..*$', _file):
continue
if not git.in_index(_file):
continue
nb += 1
file_name = os.path.join(root, _file)
with open(file_name, encoding='utf-8') as f:
data = f.read()
if ext == '.py':
data = python_minifier.minify(data, preserve_lines=True)
path_elts = package[:]
if os.path.basename(filename) != "__init__.py":
path_elts.append(os.path.basename(file_name)[:-3])
fqname = ".".join(path_elts)
with open(os.path.join(root, file_name), encoding="utf-8") as f:
tree = ast.parse(f.read())
visitor = Visitor(lib_path, package)
visitor.visit(tree)
imports = sorted(list(visitor.imports))
vfs_path = os.path.join(root, _file).replace(main_root, '')
vfs_path = vfs_path.replace("\\", "/")
if vfs_path.startswith('/libs/crypto_js/rollups/'):
if _file not in ('md5.js', 'sha1.js', 'sha3.js',
'sha224.js', 'sha256.js', 'sha384.js',
'sha512.js'):
continue
mod_name = vfs_path[len(stdlib_dir) + 2:].replace('/', '.')
mod_name, ext = os.path.splitext(mod_name)
is_package = mod_name.endswith('__init__')
if ext == ".py":
if is_package:
mod_name = mod_name[:-9]
VFS[mod_name] = [ext, data, imports, 1]
else:
VFS[mod_name] = [ext, data, imports]
else:
VFS[mod_name] = [ext, data]
print("adding {}".format(mod_name))
print('{} files, {} errors'.format(nb, nb_err))
with open(filename, "w") as out:
out.write('__BRYTHON__.use_VFS = true;\n')
out.write('__BRYTHON__.VFS={}\n\n'.format(json.dumps(VFS)))
def make(package_name, package_path, exclude_dirs=None):
if not package_name:
raise ValueError("package name is not specified")
print("Generating package {}".format(package_name))
VFS = {"$timestamp": int(1000 * time.time())}
nb = 0
if exclude_dirs is None:
exclude_dirs = []
# If package_path folder has __init__.py, all scripts will be in the
# package named package_name
in_package = "__init__.py" in os.listdir(package_path)
for dirpath, dirnames, filenames in os.walk(package_path):
flag = False
root_elts = dirpath.split(os.sep)
for exclude in exclude_dirs:
if exclude in root_elts:
continue
if '__pycache__' in dirnames:
dirnames.remove("__pycache__")
if dirpath == package_path:
package = []
else:
package = dirpath[len(package_path) + 1:].split(os.sep)
if in_package:
package = [package_name] + package
has_init = "__init__.py" in filenames
has_py = False
for filename in filenames:
name, ext = os.path.splitext(filename)
if ext != '.py':
continue
has_py = True
is_package = name.endswith('__init__')
if is_package:
mod_name = '.'.join(package)
else:
mod_name = '.'.join(package + [name])
nb += 1
absname = os.path.join(dirpath, filename)
with open(absname, encoding='utf-8') as f:
data = f.read()
data = python_minifier.minify(data, preserve_lines=True)
path_elts = package[:]
if os.path.basename(filename) != "__init__.py":
path_elts.append(os.path.basename(filename)[:-3])
fqname = ".".join(path_elts)
with open(absname, encoding="utf-8") as f:
tree = ast.parse(f.read())
visitor = Visitor(package_path, package)
visitor.visit(tree)
imports = sorted(list(visitor.imports))
if is_package:
VFS[mod_name] = [ext, data, imports, 1]
else:
VFS[mod_name] = [ext, data, imports]
print("adding {} from {}".format(mod_name, filename))
if has_py and not has_init:
# Add empty package file
mod_name = '.'.join(package)
VFS[mod_name] = [".py", "", [], 1]
print("adding empty {}".format(mod_name))
if nb == 0:
print("No Python file found in current directory")
else:
print('{} files'.format(nb))
with open(os.path.join(package_path, package_name + ".brython.js"),
"w",
encoding="utf-8") as out:
out.write('__BRYTHON__.use_VFS = true;\n')
out.write('var scripts = {}\n'.format(json.dumps(VFS, indent=4)))
out.write('__BRYTHON__.update_VFS(scripts)\n')
def process_unittest(filename):
"""Process a VFS filename for Brython."""
print("Generating {}".format(filename))
nb = 0
nb_err = 0
_main_root = os.path.dirname(filename)
_VFS = {}
for _mydir in ("Lib",):
for _root, _dir, _files in os.walk(os.path.join(_main_root, _mydir)):
if 'unittest' not in _root:
if 'test' not in _root:
continue
if '__pycache__' in _root:
continue
for _file in _files:
_ext = os.path.splitext(_file)[1]
if _ext not in ('.py'):
continue
nb += 1
file_name = os.path.join(_root, _file)
encoding = "utf-8"
try:
src = open(file_name, encoding=encoding).read()
except:
encoding = "iso-8859-1"
src = open(file_name, encoding=encoding).read()
if _ext.lower() == '.py':
try:
_data = python_minifier.minify(src)
except Exception as error:
print(error)
nb_err += 1
_vfs_filename = os.path.join(_root, _file).replace(_main_root, '')
_vfs_filename = _vfs_filename.replace("\\", "/")
mod_name = _vfs_filename[len(_mydir) + 2:].replace('/', '.')
mod_name, ext = os.path.splitext(mod_name)
is_package = mod_name.endswith('__init__')
if is_package:
mod_name = mod_name[:-9]
_VFS[mod_name] = [_data, 1]
else:
_VFS[mod_name] = [_data]
print(("Adding %s %s" % (mod_name, _vfs_filename)))
print('%s files, %s errors' % (nb, nb_err))
with open(filename, "w") as file_to_write_VFS:
file_to_write_VFS.write('__BRYTHON__.libs = __BRYTHON__.libs || {};\n')
file_to_write_VFS.write("__BRYTHON__.=libs['unittest']=%s;\n\n" % json.dumps(_VFS))
file_to_write_VFS.write("""
__BRYTHON__.import_from_unittest function(mod_name){
var stored = __BRYTHON__.libs['unittest'][mod_name]
if(stored!==undefined){
var module_contents = stored[0]
var $is_package = stored[1]
var path = 'py_unittest'
var module = {name:mod_name,__class__:$B.$ModuleDict,$is_package:$is_package}
if($is_package){var package=mod_name}
else{
var elts = mod_name.split('.')
elts.pop()
var package = elts.join('.')
}
$B.modules[mod_name].$package = $is_package
$B.modules[mod_name].__package__ = package
run_py(module,path,module_contents)
return true
}
return null
}
// add this import function to brython by doing the following:
// <body onload="brython({custom_import_funcs:[__BRYTHON__.import_from_unittest]})">
// this will allow us to import unittest modules.
""")
def main():
parser = argparse.ArgumentParser(description='Minify Python source')
parser.add_argument(
'path',
type=str,
help='The source file to minify. Use "-" to read from stdin')
parser.add_argument(
'--no-combine-imports',
action='store_false',
help='Disable combining adjacent import statements',
dest='combine_imports',
)
parser.add_argument(
'--no-remove-pass',
action='store_false',
default=True,
help='Disable removing Pass statements',
dest='remove_pass',
)
parser.add_argument(
'--remove-literal-statements',
action='store_true',
help=
'Enable removing statements that are just a literal (including docstrings)',
dest='remove_literal_statements',
)
parser.add_argument(
'--no-remove-annotations',
action='store_false',
help='Disable removing function and variable annotations',
dest='remove_annotations',
)
parser.add_argument(
'--no-hoist-literals',
action='store_false',
help='Disable replacing string and bytes literals with variables',
dest='hoist_literals',
)
parser.add_argument('--no-rename-locals',
action='store_false',
help='Disable shortening of local names',
dest='rename_locals')
parser.add_argument(
'--preserve-locals',
type=str,
action='append',
help='Comma separated list of local names that will not be shortened',
dest='preserve_locals')
parser.add_argument('--rename-globals',
action='store_true',
help='Enable shortening of global names',
dest='rename_globals')
parser.add_argument(
'--preserve-globals',
type=str,
action='append',
help='Comma separated list of global names that will not be shortened',
dest='preserve_globals')
parser.add_argument(
'--no-remove-object-base',
action='store_false',
help='Disable removing object from base class list',
dest='remove_object_base',
)
parser.add_argument('-v', '--version', action='version', version=version)
args = parser.parse_args()
if args.path == '-':
source = sys.stdin.read()
else:
with open(args.path, 'rb') as f:
source = f.read()
preserve_globals = []
if args.preserve_globals:
for arg in args.preserve_globals:
names = [name.strip() for name in arg.split(',') if name]
preserve_globals.extend(names)
preserve_locals = []
if args.preserve_locals:
for arg in args.preserve_locals:
names = [name.strip() for name in arg.split(',') if name]
preserve_locals.extend(names)
sys.stdout.write(
minify(source,
filename=args.path,
combine_imports=args.combine_imports,
remove_pass=args.remove_pass,
remove_literal_statements=args.remove_literal_statements,
hoist_literals=args.hoist_literals,
rename_locals=args.rename_locals,
preserve_locals=preserve_locals,
rename_globals=args.rename_globals,
preserve_globals=preserve_globals,
remove_object_base=args.remove_object_base))
TEXTE = python_minifier.minify(
TEXTE,
filename=None,
remove_annotations=True,
remove_pass=True,
remove_literal_statements=True,
combine_imports=True,
hoist_literals=True,
rename_locals=True,
preserve_locals=None,
rename_globals=True,
preserve_globals=[
# Objets
'Neutron',
'Proton',
'Electron',
'Atome',
'Molecule',
'Ion',
'IonMonoAtomique',
'IonPolyAtomique',
'Noyau',
'Quark',
'QUp',
'QDown',
'Equation',
'Reaction',
# Globales
'params',
'u',
'mol',
# Exceptions
'ValeurIncorrecte',
'Instable',
'Incompatible'
],
remove_object_base=True,
convert_posargs_to_args=True)
for dir_path, _, file_names in os.walk('jerryc05-original'):
if 'cache' in dir_path:
continue
new_dir_path = dir_path.replace('jerryc05-original', 'jerryc05')
if not os.path.isdir(new_dir_path):
os.mkdir(new_dir_path)
for file_name in file_names:
with open(f'{dir_path}/{file_name}', 'r', encoding='utf-8') as f1, \
open(f'{new_dir_path}/{file_name}', 'w+', encoding='utf-8') as f2:
f2.write(
python_minifier.minify(f1.read(),
remove_literal_statements=True,
rename_globals=True,
preserve_globals=[
'main',
'station_name',
'ticket_count',
'colored_text',
]) if '.py' == file_name[-3:]
and file_name not in exclude else f1.read())
print('Shrinking finished!!')
rmtree = shutil.rmtree
mp.Process(target=rmtree, args=('dist', True)).start()
mp.Process(target=rmtree, args=('build', True)).start()
argv = sys.argv
argv.append('bdist_wheel') # do not delete import setup
import setup
unicode_error = 0
errdict = defaultdict(int)
syntax_error_total = 0
kind_dict = defaultdict(int)
for kind in p.iterdir():
kind_length = 0
if not kind.is_dir():
continue
for i in kind.iterdir():
kind_length += 1
if '.py' in i.suffixes and i.is_file() and '.clean' not in i.suffixes:
try:
# print (i)
text = i.open().read()
text = python_minifier.minify(text,
remove_literal_statements=True,
rename_globals=True)
with open(i.parent / (i.stem + '.clean' + i.suffix),
"w") as newfile:
newfile.write(text)
except UnicodeDecodeError:
# print ("UnicodeDecodeError",i)
unicode_error += 1
except SyntaxError:
i.unlink()
syntax_error_total += 1
errdict[kind.name] += 1
print(f"removed {kind.name}", i)
finally:
dir_length_total += 1
kind_length += 1
def process(filename, exclude_dirs=['test','site-packages']):
"""Process a VFS filename for Brython."""
print("Generating {}".format(filename))
nb = 0
nb_err = 0
main_root = os.path.dirname(filename)
VFS = {}
for stdlib_dir in ("libs", "Lib"):
lib_path = os.path.join(main_root, stdlib_dir)
for root, _dir, files in os.walk(lib_path):
flag = False
root_elts = root.split(os.sep)
for exclude in exclude_dirs:
if exclude in root_elts:
flag = True
continue
if flag:
continue # skip these modules
if '__pycache__' in _dir:
_dir.remove("__pycache__")
nb += 1
if stdlib_dir == "Lib":
if root == lib_path:
package = []
else:
package = root[len(lib_path) + 1:].split(os.sep)
for _file in files:
ext = os.path.splitext(_file)[1]
if ext not in ('.js', '.py'):
continue
if re.match(r'^module\d+\..*$', _file):
continue
if not git.in_index(_file):
print(_file, "not in git index")
continue
nb += 1
file_name = os.path.join(root, _file)
with open(file_name, encoding='utf-8') as f:
data = f.read()
if ext == '.py':
data = python_minifier.minify(data, preserve_lines=True)
path_elts = package[:]
if os.path.basename(filename) != "__init__.py":
path_elts.append(os.path.basename(file_name)[:-3])
fqname = ".".join(path_elts)
with open(os.path.join(root, file_name), encoding="utf-8") as f:
tree = ast.parse(f.read())
visitor = Visitor(lib_path, package)
visitor.visit(tree)
imports = sorted(list(visitor.imports))
vfs_path = os.path.join(root, _file).replace(main_root, '')
vfs_path = vfs_path.replace("\\", "/")
if vfs_path.startswith('/libs/crypto_js/rollups/'):
if _file not in ('md5.js', 'sha1.js', 'sha3.js',
'sha224.js', 'sha256.js', 'sha384.js',
'sha512.js'):
continue
mod_name = vfs_path[len(stdlib_dir) + 2:].replace('/', '.')
mod_name, ext = os.path.splitext(mod_name)
is_package = mod_name.endswith('__init__')
if ext == ".py":
if is_package:
mod_name = mod_name[:-9]
VFS[mod_name] = [ext, data, imports, 1]
else:
VFS[mod_name] = [ext, data, imports]
else:
VFS[mod_name] = [ext, data]
print("adding {}".format(mod_name))
print('{} files, {} errors'.format(nb, nb_err))
with open(filename, "w") as out:
out.write('__BRYTHON__.use_VFS = true;\n')
out.write('var scripts = {}\n'.format(json.dumps(VFS)))
out.write('__BRYTHON__.update_VFS(scripts)\n')
parser.add_argument("file", type=str, help="target file for obfuscation")
g_output.add_argument("-o", "--output", type=str, help="output destination")
args = parser.parse_args()
if __name__ == "__main__":
filename = args.file
output = args.output if args.output else filename.split(
".")[0] + "_BFUSCATED.py"
start = time.time()
with open(filename) as f:
code = f.read().replace("\"", "'")
minified = python_minifier.minify(code)
if args.method == "lambda":
RESULT = functions.lambda_bfuscate(minified, args)
elif args.method == "len":
RESULT = functions.len_bfuscate(minified, args)
else:
print(f"[×] Unknown obfuscation method '{args.method}'")
sys.exit()
with open(output, "w+") as f:
f.write(RESULT)
print(f"[✔] Successfully obfuscated: {filename}")
print(f"Before\t\t{len(code)} characters")
print(f"Minified\t{len(minified)} characters")
def make(package_name, package_path, exclude_dirs=None):
print("Generating package {}".format(package_name))
VFS = {}
nb = 0
if exclude_dirs is None:
exclude_dirs = []
for dirpath, dirnames, filenames in os.walk(package_path):
flag = False
root_elts = dirpath.split(os.sep)
for exclude in exclude_dirs:
if exclude in root_elts:
continue
if '__pycache__' in dirnames:
dirnames.remove("__pycache__")
if dirpath == package_path:
package = []
else:
package = dirpath[len(package_path) + 1:].split(os.sep)
for filename in filenames:
ext = os.path.splitext(filename)[1]
if ext not in ('.js', '.py'):
continue
if filename.endswith(".brython.js"): # no recursion
continue
nb += 1
absname = os.path.join(dirpath, filename)
with open(absname, encoding='utf-8') as f:
data = f.read()
if ext == '.py':
data = python_minifier.minify(data, preserve_lines=True)
path_elts = package[:]
if os.path.basename(filename) != "__init__.py":
path_elts.append(os.path.basename(filename)[:-3])
fqname = ".".join(path_elts)
with open(absname, encoding="utf-8") as f:
tree = ast.parse(f.read())
visitor = Visitor(package_path, package)
visitor.visit(tree)
imports = sorted(list(visitor.imports))
mod_name = filename.replace(os.sep, '.')
if package_name:
mod_name = package_name + "." + mod_name
mod_name, ext = os.path.splitext(mod_name)
is_package = mod_name.endswith('__init__')
if ext == ".py":
if is_package:
mod_name = mod_name[:-9]
VFS[mod_name] = [ext, data, imports, 1]
else:
VFS[mod_name] = [ext, data, imports]
else:
VFS[mod_name] = [ext, data]
print("adding {}".format(mod_name))
print('{} files'.format(nb))
with open(os.path.join(package_path, package_name + ".brython.js"),
"w",
encoding="utf-8") as out:
out.write('__BRYTHON__.use_VFS = true;\n')
out.write('var scripts = {}\n'.format(json.dumps(VFS)))
out.write('__BRYTHON__.update_VFS(scripts)\n')
def process_unittest(filename):
"""Process a VFS filename for Brython."""
print("Generating {}".format(filename))
nb = 0
nb_err = 0
_main_root = os.path.dirname(filename)
_VFS = {}
for _mydir in ("Lib", ):
for _root, _dir, _files in os.walk(os.path.join(_main_root, _mydir)):
if 'unittest' not in _root:
if 'test' not in _root:
continue
if '__pycache__' in _root:
continue
for _file in _files:
_ext = os.path.splitext(_file)[1]
if _ext not in ('.py'):
continue
nb += 1
file_name = os.path.join(_root, _file)
encoding = "utf-8"
try:
src = open(file_name, encoding=encoding).read()
except:
encoding = "iso-8859-1"
src = open(file_name, encoding=encoding).read()
if _ext.lower() == '.py':
try:
_data = python_minifier.minify(src)
except Exception as error:
print(error)
nb_err += 1
_vfs_filename = os.path.join(_root,
_file).replace(_main_root, '')
_vfs_filename = _vfs_filename.replace("\\", "/")
mod_name = _vfs_filename[len(_mydir) + 2:].replace('/', '.')
mod_name, ext = os.path.splitext(mod_name)
is_package = mod_name.endswith('__init__')
if is_package:
mod_name = mod_name[:-9]
_VFS[mod_name] = [_data, 1]
else:
_VFS[mod_name] = [_data]
print(("Adding %s %s" % (mod_name, _vfs_filename)))
print('%s files, %s errors' % (nb, nb_err))
with open(filename, "w") as file_to_write_VFS:
file_to_write_VFS.write('__BRYTHON__.libs = __BRYTHON__.libs || {};\n')
file_to_write_VFS.write("__BRYTHON__.=libs['unittest']=%s;\n\n" %
json.dumps(_VFS))
file_to_write_VFS.write("""
__BRYTHON__.import_from_unittest function(mod_name){
var stored = __BRYTHON__.libs['unittest'][mod_name]
if(stored!==undefined){
var module_contents = stored[0]
var $is_package = stored[1]
var path = 'py_unittest'
var module = {name:mod_name,__class__:$B.$ModuleDict,$is_package:$is_package}
if($is_package){var package=mod_name}
else{
var elts = mod_name.split('.')
elts.pop()
var package = elts.join('.')
}
$B.modules[mod_name].$package = $is_package
$B.modules[mod_name].__package__ = package
run_py(module,path,module_contents)
return true
}
return null
}
// add this import function to brython by doing the following:
// <body onload="brython({custom_import_funcs:[__BRYTHON__.import_from_unittest]})">
// this will allow us to import unittest modules.
""")
def minify(input_file):
import python_minifier
with open(input_file, 'r') as handler:
return python_minifier.minify(handler.read())
