def execute_on(self, view: ProjectView, build_output_dir: str, build_manager): try: import python_minifier except ImportError: subprocess.Popen( [sys.executable, "-m", "pip", "install", "python-minifier"]) import python_minifier for file in view.modified_files: if file.endswith(".py"): if file not in self.special_config: try: view.write( file, python_minifier.minify( view.read(file).decode("utf-8"), preserve_locals=["NAME"], remove_literal_statements=True, ).encode("utf-8"), ) except: print(file) raise else: view.write( file, python_minifier.minify( view.read(file).decode("utf-8"), **self.special_config[file]).encode("utf-8"), )
def import_lambda(name): TK_IN_LBD = "IBOX_CODE_IN_LAMBDA" parent_dir_name = os.path.dirname(os.path.realpath(__file__)) lambda_file = os.path.join(os.getcwd(), "lib/lambdas/%s.code" % name) if not os.path.exists(lambda_file): lambda_file = os.path.join(parent_dir_name, "lambdas/%s.code" % name) try: lambda_file_trunk = os.path.join(*PurePath(lambda_file).parts[-3:]) with open(lambda_file, "r") as f: fdata = f.read() try: # try to minify using python_minifier code = python_minifier.minify(fdata) if len(code) > 4096: logging.warning( f"{lambda_file_trunk} > 4096, trying to minify it using a more aggressive option [rename_globals=True]" ) code = python_minifier.minify( fdata, rename_globals=True, preserve_globals=["lambda_handler"]) except Exception: code = fdata if len(code) > 4096: logging.warning(f"Inline lambda {lambda_file_trunk} > 4096") code_lines = code.splitlines(keepends=True) file_lines = [] # parse lambda code for Token IBOX CODE for x in code_lines: if x.startswith(cfg.EVAL_FUNCTIONS_IN_CFG): value = eval(x) elif x.startswith(TK_IN_LBD): value = '"' elif TK_IN_LBD in x: # parse minified code tks = x.split(TK_IN_LBD) file_lines.extend([f"{tks[0]}", eval(tks[1]), tks[2]]) continue else: value = "".join(x) file_lines.append(value) return file_lines except IOError: logging.warning(f"Lambda code {name} not found") exit(1) except Exception as e: logging.error(e)
def _minify_and_compile(file): with open(file) as f: buff = f.read() with open(file, "w") as f: mnf = python_minifier.minify(buff) f.write(mnf) compileall.compile_file(file, force=True, quiet=True, legacy=True)
def run(self): """Run command.""" import python_minifier for file in os.listdir(str(__dir__ / 'src' / 'umqtt')): with open(str(__dir__ / 'src' / 'umqtt' / file)) as f: print('Minify: %s' % file) source = f.read() filename = file.split('/')[-1] out = python_minifier.minify( source, filename, remove_annotations=True, remove_pass=True, remove_literal_statements=True, combine_imports=False, hoist_literals=False, rename_locals=True, preserve_locals=None, rename_globals=False, preserve_globals=None, remove_object_base=False, convert_posargs_to_args=False ) with open(str(Path(self.minified_dir) / 'umqtt' / file), 'w') as f: f.write(out)
def run_test(self): from sh import Command, ErrorReturnCode ErrorReturnCode.truncate_cap = 1000 def execute(python, path): python = Command(python) python(path) try: with open(self.test_path, 'rb') as f: source = f.read() shutil.copy(self.test_path, self.test_path + '.bak') with open(self.test_path, 'wb') as f: f.write(minify(source, self.test_path, **self.options).encode()) execute(sys.executable, self.test_path) except ErrorReturnCode as e: print(self.test_path) print(e.stderr) raise finally: shutil.copy(self.test_path + '.bak', self.test_path)
def process(filename, exclude_dirs=['test', 'site-packages']): """Process a VFS filename for Brython.""" print("Generating {}".format(filename)) nb = 0 nb_err = 0 main_root = os.path.dirname(filename) VFS = {} for stdlib_dir in ("libs", "Lib"): for root, _dir, files in os.walk(os.path.join(main_root, stdlib_dir)): flag = False root_elts = root.split(os.sep) for exclude in exclude_dirs: if exclude in root_elts: flag = True continue if flag: continue # skip these modules if '__pycache__' in root: continue nb += 1 for _file in files: ext = os.path.splitext(_file)[1] if ext not in ('.js', '.py'): continue if re.match(r'^module\d+\..*$', _file): continue nb += 1 file_name = os.path.join(root, _file) with open(file_name, encoding='utf-8') as f: data = f.read() if ext == '.py': data = python_minifier.minify(data, preserve_lines=True) vfs_path = os.path.join(root, _file).replace(main_root, '') vfs_path = vfs_path.replace("\\", "/") if vfs_path.startswith('/libs/crypto_js/rollups/'): if _file not in ('md5.js', 'sha1.js', 'sha3.js', 'sha224.js', 'sha384.js', 'sha512.js'): continue mod_name = vfs_path[len(stdlib_dir) + 2:].replace('/', '.') mod_name, ext = os.path.splitext(mod_name) is_package = mod_name.endswith('__init__') if is_package: mod_name = mod_name[:-9] VFS[mod_name] = [ext, data, 1] else: VFS[mod_name] = [ext, data] print("adding {}".format(mod_name)) print('{} files, {} errors'.format(nb, nb_err)) with open(filename, "w") as out: out.write('__BRYTHON__.use_VFS = true;\n') out.write('__BRYTHON__.VFS={}\n\n'.format(json.dumps(VFS)))
def process(filename, exclude_dirs=[ 'unittest', 'test', ]): """Process a VFS filename for Brython.""" print("Generating {}".format(filename)) nb = 0 nb_err = 0 _main_root = os.path.dirname(filename) _VFS = {} for _mydir in ("libs", "Lib"): for _root, _dir, _files in os.walk(os.path.join(_main_root, _mydir)): #if _root.endswith('lib_migration'): _flag = False for _exclude in exclude_dirs: if _exclude in _root: #_root.endswith(_exclude): _flag = True continue if _flag: continue # skip these modules if '__pycache__' in _root: continue nb += 1 for _file in _files: _ext = os.path.splitext(_file)[1] if _ext not in ('.js', '.py'): continue nb += 1 file_name = os.path.join(_root, _file) _data = open(file_name, encoding='utf-8').read() if _ext == '.py': _data = python_minifier.minify(_data) _vfs_filename = os.path.join(_root, _file).replace(_main_root, '') _vfs_filename = _vfs_filename.replace("\\", "/") if _vfs_filename.startswith('/libs/crypto_js/rollups/'): if _file not in ('md5.js', 'sha1.js', 'sha3.js', 'sha224.js', 'sha384.js', 'sha512.js'): continue mod_name = _vfs_filename[len(_mydir) + 2:].replace('/', '.') mod_name, ext = os.path.splitext(mod_name) is_package = mod_name.endswith('__init__') if is_package: mod_name = mod_name[:-9] _VFS[mod_name] = [ext, _data, 1] else: _VFS[mod_name] = [ext, _data] print(("adding %s %s" % (mod_name, _vfs_filename))) print('%s files, %s errors' % (nb, nb_err)) with open(filename, "w") as file_to_write_VFS: file_to_write_VFS.write('__BRYTHON__.use_VFS = true;\n') file_to_write_VFS.write('__BRYTHON__.VFS=%s;\n\n' % json.dumps(_VFS))
def test_file(path): try: with open(path, 'rb') as f: source = f.read() except IOError: pytest.skip('IOError opening file') try: original_ast = ast.parse(source, path) except SyntaxError: pytest.skip('Invalid syntax in file') # Test unparsing unparse(original_ast) # Test transforms minify(source, filename=path)
def process(filename, exclude_dirs=['unittest','test','site-packages']): """Process a VFS filename for Brython.""" print("Generating {}".format(filename)) nb = 0 nb_err = 0 _main_root = os.path.dirname(filename) _VFS = {} for _mydir in ("libs", "Lib"): for _root, _dir, _files in os.walk(os.path.join(_main_root, _mydir)): #if _root.endswith('lib_migration'): _flag=False for _exclude in exclude_dirs: if _exclude in _root: #_root.endswith(_exclude): _flag=True continue if _flag: continue # skip these modules if '__pycache__' in _root: continue nb += 1 for _file in _files: _ext = os.path.splitext(_file)[1] if _ext not in ('.js', '.py'): continue if re.match(r'^module\d+\..*$', _file): continue nb += 1 file_name = os.path.join(_root, _file) _data = open(file_name, encoding='utf-8').read() if _ext == '.py': _data = python_minifier.minify(_data, preserve_lines=True) _vfs_filename = os.path.join(_root, _file).replace(_main_root, '') _vfs_filename = _vfs_filename.replace("\\", "/") if _vfs_filename.startswith('/libs/crypto_js/rollups/'): if _file not in ('md5.js', 'sha1.js', 'sha3.js', 'sha224.js', 'sha384.js', 'sha512.js'): continue mod_name = _vfs_filename[len(_mydir) + 2:].replace('/', '.') mod_name, ext = os.path.splitext(mod_name) is_package = mod_name.endswith('__init__') if is_package: mod_name = mod_name[:-9] _VFS[mod_name] = [ext, _data, 1] else: _VFS[mod_name] = [ext, _data] print(("adding %s %s" % (mod_name, _vfs_filename))) print('%s files, %s errors' % (nb, nb_err)) with open(filename, "w") as file_to_write_VFS: file_to_write_VFS.write('__BRYTHON__.use_VFS = true;\n') file_to_write_VFS.write('__BRYTHON__.VFS=%s;\n\n' % json.dumps(_VFS))
def create_vfs(path, exclude_dirs=['test', 'site-packages'], progress_func=print): ret = {} def skip(p): for ex in exclude_dirs: if ex in p.parts: return True if '__pycache__' in p.parts: return True if p.is_file(): if p.suffix not in ('.js', '.py'): return True if re.match(r'^module\d+\..*$', p.name): return True if '/'.join(p.parts[-4:-1]) == 'libs/crypto_js/rollups': if p.name not in ('md5.js', 'sha1.js', 'sha3.js', 'sha224.js', 'sha384.js', 'sha512.js'): return True return False nb = 0 for subdir in ("libs", "Lib"): for p in walk(path / subdir, skip): nb += 1 vfs_path = p.relative_to(path) data = p.read_text(encoding='utf-8') if p.suffix == '.py': tree = ast.parse(data) visitor = Visitor(path / subdir, vfs_path.pargs[1:]) visitor.visit(tree) imports = sorted(list(visitor.imports)) data = python_minifier.minify(data, preserve_lines=True) parent_mod_name = '.'.join(vfs_path.parts[1:-1]) if p.stem == '__init__': ret[parent_mod_name] = [p.suffix, data, imports, 1] progress_func("Adding package {}".format(parent_mod_name)) else: if parent_mod_name: ret[parent_mod_name + '.' + p.stem] = [p.suffix, data] if p.suffix == '.py': ret[parent_mod_name + '.' + p.stem].append(imports) progress_func("Adding {}".format(parent_mod_name + '.' + p.stem)) else: ret[p.stem] = [p.suffix, data] if p.suffix == '.py': ret[parent_mod_name + '.' + p.stem].append(imports) progress_func("Adding {}".format(p.stem)) return ret, nb
def add_state_machine_handler(template: troposphere.Template): # minify so it fits in 4096 bytes and we don't need to upload to S3 first code = python_minifier.minify(open("lambda/rds.py").read(), "lambda/rds.py", rename_globals=True, preserve_globals=["handler", "NotReady"]) return add_lambda( template, "Handler", code, HANDLER_POLICIES, Timeout=30, )
def __minify_files(self: object, files_to_minify: str) -> bool: for file in files_to_minify: try: with open(file, 'r') as data: # , rename_globals=True, rename_locals=True, hoist_literals=True minified_content = minify(data.read()) if minified_content: with open(f'build\\{file}', 'w') as data: data.write(minified_content) self.__push_msg_(f'Minified {file}') except Exception as err_obj: self.__push_msg_(f'Error: {err_obj}', type='err') return False return True
def _zip_codedir(self, home, ziph): home = home.rstrip('/') total_files = {} zip_relpaths = set() for root, _, files in os.walk(home): for file in files: filepath = join(root, file) zip_relpath = relpath(filepath, home) zip_relpaths.add(zip_relpath) total_files[zip_relpath] = filepath src_zip_relpaths = zip_relpaths.copy() ignore_patterns = self.get_ignore_patterns(home) for ignore_pattern in ignore_patterns: if ignore_pattern.startswith('!'): ignore_pattern = ignore_pattern[1:] zip_relpaths.update( fnmatch.filter(src_zip_relpaths, ignore_pattern)) continue zip_relpaths -= set(fnmatch.filter(zip_relpaths, ignore_pattern)) minify_relpaths = set() minify_patterns = self.get_minify_patterns(home) for minify_pattern in minify_patterns: if minify_pattern.startswith('!'): minify_pattern = minify_pattern[1:] minify_relpaths -= set( fnmatch.filter(minify_relpaths, minify_pattern)) continue minify_relpaths.update( fnmatch.filter(src_zip_relpaths, minify_pattern)) # 仅支持混淆 .py 文件 minify_relpaths = fnmatch.filter(minify_relpaths, '*.py') with console.status('正在打包模型源码...', spinner='dots'): for zip_relpath in sorted(zip_relpaths): filepath = total_files[zip_relpath] if zip_relpath in minify_relpaths: continue ziph.write(filepath, zip_relpath) output('\t{}'.format(zip_relpath)) for zip_relpath in sorted(minify_relpaths): filepath = total_files[zip_relpath] with open(filepath, 'r') as fp: content = fp.read() with tempfile.NamedTemporaryFile(mode='w') as temp_fp: content = python_minifier.minify(content) temp_fp.write(content) temp_fp.flush() temp_fp.seek(0) ziph.write(temp_fp.name, zip_relpath) output('\t{} [green]MINIFIED[/green]'.format(zip_relpath))
async def minify(self, ctx): """Minify a python file. You need to attach a file to this command, and it's extension needs to be `.py`. """ await ctx.trigger_typing() if not ctx.message.attachments: return await ctx.send_help() file = ctx.message.attachments[0] if not file.filename.lower().endswith(".py"): return await ctx.send("Must be a python file.") converted = io.BytesIO(minifier.minify(await file.read()).encode()) content = "Please see the attached file below, with your minimized code." await ctx.send(content=content, file=discord.File(converted, filename=file.filename.lower()))
def create_example(option): options = { 'combine_imports': False, 'hoist_literals': False, 'remove_annotations': False, 'remove_literal_statements': False, 'remove_pass': False, 'rename_globals': False, 'rename_locals': False, 'remove_object_base': False } options[option] = True with open(f'transforms/{option}.py') as source: with open(f'transforms/{option}.min.py', 'w') as minified: minified.write( minify(source.read(), filename=f'{option}.py', **options))
def minify_python(filepath): """ Minify a Python File """ try: if not os.path.isfile(filepath): return False if not filepath.endswith(".py"): return False print("-- Minifying %s" % filepath) with open(filepath, "r") as fileread: filecontent = fileread.read() fileread.close() filemini = python_minifier.minify(filecontent) with open(filepath, "w") as filewrite: filewrite.write(filemini) filewrite.close() return True except Exception: print("-- Unable to minify %s" % filepath) return False
async def minify(self, ctx): """Minify a python file. You need to attach a file to this command, and it's extension needs to be `.py`. """ await ctx.trigger_typing() if not ctx.message.attachments: return await ctx.send_help() file = ctx.message.attachments[0] file_name = file.filename.lower() if not file_name.endswith((".py", ".python")): return await ctx.send("Must be a python file.") with contextlib.suppress(UnicodeDecodeError, UnicodeEncodeError): file = await file.read() converted = io.BytesIO(minifier.minify(file).encode(encoding="utf-8")) content = "Please see the attached file below, with your minimized code." return await ctx.send( content=content, file=discord.File(converted, filename=file_name) ) return await ctx.send("The file provided was in an unsupported format.")
async def minify(self, ctx): """Minify a python file. You need to attach a file to this command, and it's extension needs to be `.py`. """ await ctx.trigger_typing() if not ctx.message.attachments: return await ctx.send_help() file = ctx.message.attachments[0] file_name = file.filename.lower() if not file_name.endswith((".py", ".python")): return await ctx.send("Must be a python file.") try: file = await file.read() except UnicodeDecodeError: return await ctx.send("Something went wrong when trying to decode this file.") converted = io.BytesIO(minifier.minify(file).encode(encoding="utf-8")) return await ctx.send( content="Please see the attached file below, with your minified code.", file=discord.File(converted, filename=file_name), )
def minify_python(s): """ Minify python code. @param s: python code to minify @type s: L{str} @return: the minified python code @rtype: L{str} """ # not working with brython :( return s if python_minifier is None: raise NotImplementedError( "Dependency 'python-minifier' required, but not found!") return python_minifier.minify( s, remove_annotations=True, remove_pass=True, # remove_literal_statements= combine_imports=True, hoist_literals=True, rename_locals=True, rename_globals=True, )
def process(filename, exclude_dirs=['test','site-packages']): """Process a VFS filename for Brython.""" print("Generating {}".format(filename)) nb = 0 nb_err = 0 main_root = os.path.dirname(filename) VFS = {} for stdlib_dir in ("libs", "Lib"): lib_path = os.path.join(main_root, stdlib_dir) for root, _dir, files in os.walk(lib_path): flag = False root_elts = root.split(os.sep) for exclude in exclude_dirs: if exclude in root_elts: flag = True continue if flag: continue # skip these modules if '__pycache__' in _dir: _dir.remove("__pycache__") nb += 1 if stdlib_dir == "Lib": if root == lib_path: package = [] else: package = root[len(lib_path) + 1:].split(os.sep) for _file in files: ext = os.path.splitext(_file)[1] if ext not in ('.js', '.py'): continue if re.match(r'^module\d+\..*$', _file): continue if not git.in_index(_file): continue nb += 1 file_name = os.path.join(root, _file) with open(file_name, encoding='utf-8') as f: data = f.read() if ext == '.py': data = python_minifier.minify(data, preserve_lines=True) path_elts = package[:] if os.path.basename(filename) != "__init__.py": path_elts.append(os.path.basename(file_name)[:-3]) fqname = ".".join(path_elts) with open(os.path.join(root, file_name), encoding="utf-8") as f: tree = ast.parse(f.read()) visitor = Visitor(lib_path, package) visitor.visit(tree) imports = sorted(list(visitor.imports)) vfs_path = os.path.join(root, _file).replace(main_root, '') vfs_path = vfs_path.replace("\\", "/") if vfs_path.startswith('/libs/crypto_js/rollups/'): if _file not in ('md5.js', 'sha1.js', 'sha3.js', 'sha224.js', 'sha256.js', 'sha384.js', 'sha512.js'): continue mod_name = vfs_path[len(stdlib_dir) + 2:].replace('/', '.') mod_name, ext = os.path.splitext(mod_name) is_package = mod_name.endswith('__init__') if ext == ".py": if is_package: mod_name = mod_name[:-9] VFS[mod_name] = [ext, data, imports, 1] else: VFS[mod_name] = [ext, data, imports] else: VFS[mod_name] = [ext, data] print("adding {}".format(mod_name)) print('{} files, {} errors'.format(nb, nb_err)) with open(filename, "w") as out: out.write('__BRYTHON__.use_VFS = true;\n') out.write('__BRYTHON__.VFS={}\n\n'.format(json.dumps(VFS)))
def make(package_name, package_path, exclude_dirs=None): if not package_name: raise ValueError("package name is not specified") print("Generating package {}".format(package_name)) VFS = {"$timestamp": int(1000 * time.time())} nb = 0 if exclude_dirs is None: exclude_dirs = [] # If package_path folder has __init__.py, all scripts will be in the # package named package_name in_package = "__init__.py" in os.listdir(package_path) for dirpath, dirnames, filenames in os.walk(package_path): flag = False root_elts = dirpath.split(os.sep) for exclude in exclude_dirs: if exclude in root_elts: continue if '__pycache__' in dirnames: dirnames.remove("__pycache__") if dirpath == package_path: package = [] else: package = dirpath[len(package_path) + 1:].split(os.sep) if in_package: package = [package_name] + package has_init = "__init__.py" in filenames has_py = False for filename in filenames: name, ext = os.path.splitext(filename) if ext != '.py': continue has_py = True is_package = name.endswith('__init__') if is_package: mod_name = '.'.join(package) else: mod_name = '.'.join(package + [name]) nb += 1 absname = os.path.join(dirpath, filename) with open(absname, encoding='utf-8') as f: data = f.read() data = python_minifier.minify(data, preserve_lines=True) path_elts = package[:] if os.path.basename(filename) != "__init__.py": path_elts.append(os.path.basename(filename)[:-3]) fqname = ".".join(path_elts) with open(absname, encoding="utf-8") as f: tree = ast.parse(f.read()) visitor = Visitor(package_path, package) visitor.visit(tree) imports = sorted(list(visitor.imports)) if is_package: VFS[mod_name] = [ext, data, imports, 1] else: VFS[mod_name] = [ext, data, imports] print("adding {} from {}".format(mod_name, filename)) if has_py and not has_init: # Add empty package file mod_name = '.'.join(package) VFS[mod_name] = [".py", "", [], 1] print("adding empty {}".format(mod_name)) if nb == 0: print("No Python file found in current directory") else: print('{} files'.format(nb)) with open(os.path.join(package_path, package_name + ".brython.js"), "w", encoding="utf-8") as out: out.write('__BRYTHON__.use_VFS = true;\n') out.write('var scripts = {}\n'.format(json.dumps(VFS, indent=4))) out.write('__BRYTHON__.update_VFS(scripts)\n')
def process_unittest(filename): """Process a VFS filename for Brython.""" print("Generating {}".format(filename)) nb = 0 nb_err = 0 _main_root = os.path.dirname(filename) _VFS = {} for _mydir in ("Lib",): for _root, _dir, _files in os.walk(os.path.join(_main_root, _mydir)): if 'unittest' not in _root: if 'test' not in _root: continue if '__pycache__' in _root: continue for _file in _files: _ext = os.path.splitext(_file)[1] if _ext not in ('.py'): continue nb += 1 file_name = os.path.join(_root, _file) encoding = "utf-8" try: src = open(file_name, encoding=encoding).read() except: encoding = "iso-8859-1" src = open(file_name, encoding=encoding).read() if _ext.lower() == '.py': try: _data = python_minifier.minify(src) except Exception as error: print(error) nb_err += 1 _vfs_filename = os.path.join(_root, _file).replace(_main_root, '') _vfs_filename = _vfs_filename.replace("\\", "/") mod_name = _vfs_filename[len(_mydir) + 2:].replace('/', '.') mod_name, ext = os.path.splitext(mod_name) is_package = mod_name.endswith('__init__') if is_package: mod_name = mod_name[:-9] _VFS[mod_name] = [_data, 1] else: _VFS[mod_name] = [_data] print(("Adding %s %s" % (mod_name, _vfs_filename))) print('%s files, %s errors' % (nb, nb_err)) with open(filename, "w") as file_to_write_VFS: file_to_write_VFS.write('__BRYTHON__.libs = __BRYTHON__.libs || {};\n') file_to_write_VFS.write("__BRYTHON__.=libs['unittest']=%s;\n\n" % json.dumps(_VFS)) file_to_write_VFS.write(""" __BRYTHON__.import_from_unittest function(mod_name){ var stored = __BRYTHON__.libs['unittest'][mod_name] if(stored!==undefined){ var module_contents = stored[0] var $is_package = stored[1] var path = 'py_unittest' var module = {name:mod_name,__class__:$B.$ModuleDict,$is_package:$is_package} if($is_package){var package=mod_name} else{ var elts = mod_name.split('.') elts.pop() var package = elts.join('.') } $B.modules[mod_name].$package = $is_package $B.modules[mod_name].__package__ = package run_py(module,path,module_contents) return true } return null } // add this import function to brython by doing the following: // <body onload="brython({custom_import_funcs:[__BRYTHON__.import_from_unittest]})"> // this will allow us to import unittest modules. """)
def main(): parser = argparse.ArgumentParser(description='Minify Python source') parser.add_argument( 'path', type=str, help='The source file to minify. Use "-" to read from stdin') parser.add_argument( '--no-combine-imports', action='store_false', help='Disable combining adjacent import statements', dest='combine_imports', ) parser.add_argument( '--no-remove-pass', action='store_false', default=True, help='Disable removing Pass statements', dest='remove_pass', ) parser.add_argument( '--remove-literal-statements', action='store_true', help= 'Enable removing statements that are just a literal (including docstrings)', dest='remove_literal_statements', ) parser.add_argument( '--no-remove-annotations', action='store_false', help='Disable removing function and variable annotations', dest='remove_annotations', ) parser.add_argument( '--no-hoist-literals', action='store_false', help='Disable replacing string and bytes literals with variables', dest='hoist_literals', ) parser.add_argument('--no-rename-locals', action='store_false', help='Disable shortening of local names', dest='rename_locals') parser.add_argument( '--preserve-locals', type=str, action='append', help='Comma separated list of local names that will not be shortened', dest='preserve_locals') parser.add_argument('--rename-globals', action='store_true', help='Enable shortening of global names', dest='rename_globals') parser.add_argument( '--preserve-globals', type=str, action='append', help='Comma separated list of global names that will not be shortened', dest='preserve_globals') parser.add_argument( '--no-remove-object-base', action='store_false', help='Disable removing object from base class list', dest='remove_object_base', ) parser.add_argument('-v', '--version', action='version', version=version) args = parser.parse_args() if args.path == '-': source = sys.stdin.read() else: with open(args.path, 'rb') as f: source = f.read() preserve_globals = [] if args.preserve_globals: for arg in args.preserve_globals: names = [name.strip() for name in arg.split(',') if name] preserve_globals.extend(names) preserve_locals = [] if args.preserve_locals: for arg in args.preserve_locals: names = [name.strip() for name in arg.split(',') if name] preserve_locals.extend(names) sys.stdout.write( minify(source, filename=args.path, combine_imports=args.combine_imports, remove_pass=args.remove_pass, remove_literal_statements=args.remove_literal_statements, hoist_literals=args.hoist_literals, rename_locals=args.rename_locals, preserve_locals=preserve_locals, rename_globals=args.rename_globals, preserve_globals=preserve_globals, remove_object_base=args.remove_object_base))
TEXTE = python_minifier.minify( TEXTE, filename=None, remove_annotations=True, remove_pass=True, remove_literal_statements=True, combine_imports=True, hoist_literals=True, rename_locals=True, preserve_locals=None, rename_globals=True, preserve_globals=[ # Objets 'Neutron', 'Proton', 'Electron', 'Atome', 'Molecule', 'Ion', 'IonMonoAtomique', 'IonPolyAtomique', 'Noyau', 'Quark', 'QUp', 'QDown', 'Equation', 'Reaction', # Globales 'params', 'u', 'mol', # Exceptions 'ValeurIncorrecte', 'Instable', 'Incompatible' ], remove_object_base=True, convert_posargs_to_args=True)
for dir_path, _, file_names in os.walk('jerryc05-original'): if 'cache' in dir_path: continue new_dir_path = dir_path.replace('jerryc05-original', 'jerryc05') if not os.path.isdir(new_dir_path): os.mkdir(new_dir_path) for file_name in file_names: with open(f'{dir_path}/{file_name}', 'r', encoding='utf-8') as f1, \ open(f'{new_dir_path}/{file_name}', 'w+', encoding='utf-8') as f2: f2.write( python_minifier.minify(f1.read(), remove_literal_statements=True, rename_globals=True, preserve_globals=[ 'main', 'station_name', 'ticket_count', 'colored_text', ]) if '.py' == file_name[-3:] and file_name not in exclude else f1.read()) print('Shrinking finished!!') rmtree = shutil.rmtree mp.Process(target=rmtree, args=('dist', True)).start() mp.Process(target=rmtree, args=('build', True)).start() argv = sys.argv argv.append('bdist_wheel') # do not delete import setup import setup
unicode_error = 0 errdict = defaultdict(int) syntax_error_total = 0 kind_dict = defaultdict(int) for kind in p.iterdir(): kind_length = 0 if not kind.is_dir(): continue for i in kind.iterdir(): kind_length += 1 if '.py' in i.suffixes and i.is_file() and '.clean' not in i.suffixes: try: # print (i) text = i.open().read() text = python_minifier.minify(text, remove_literal_statements=True, rename_globals=True) with open(i.parent / (i.stem + '.clean' + i.suffix), "w") as newfile: newfile.write(text) except UnicodeDecodeError: # print ("UnicodeDecodeError",i) unicode_error += 1 except SyntaxError: i.unlink() syntax_error_total += 1 errdict[kind.name] += 1 print(f"removed {kind.name}", i) finally: dir_length_total += 1 kind_length += 1
def process(filename, exclude_dirs=['test','site-packages']): """Process a VFS filename for Brython.""" print("Generating {}".format(filename)) nb = 0 nb_err = 0 main_root = os.path.dirname(filename) VFS = {} for stdlib_dir in ("libs", "Lib"): lib_path = os.path.join(main_root, stdlib_dir) for root, _dir, files in os.walk(lib_path): flag = False root_elts = root.split(os.sep) for exclude in exclude_dirs: if exclude in root_elts: flag = True continue if flag: continue # skip these modules if '__pycache__' in _dir: _dir.remove("__pycache__") nb += 1 if stdlib_dir == "Lib": if root == lib_path: package = [] else: package = root[len(lib_path) + 1:].split(os.sep) for _file in files: ext = os.path.splitext(_file)[1] if ext not in ('.js', '.py'): continue if re.match(r'^module\d+\..*$', _file): continue if not git.in_index(_file): print(_file, "not in git index") continue nb += 1 file_name = os.path.join(root, _file) with open(file_name, encoding='utf-8') as f: data = f.read() if ext == '.py': data = python_minifier.minify(data, preserve_lines=True) path_elts = package[:] if os.path.basename(filename) != "__init__.py": path_elts.append(os.path.basename(file_name)[:-3]) fqname = ".".join(path_elts) with open(os.path.join(root, file_name), encoding="utf-8") as f: tree = ast.parse(f.read()) visitor = Visitor(lib_path, package) visitor.visit(tree) imports = sorted(list(visitor.imports)) vfs_path = os.path.join(root, _file).replace(main_root, '') vfs_path = vfs_path.replace("\\", "/") if vfs_path.startswith('/libs/crypto_js/rollups/'): if _file not in ('md5.js', 'sha1.js', 'sha3.js', 'sha224.js', 'sha256.js', 'sha384.js', 'sha512.js'): continue mod_name = vfs_path[len(stdlib_dir) + 2:].replace('/', '.') mod_name, ext = os.path.splitext(mod_name) is_package = mod_name.endswith('__init__') if ext == ".py": if is_package: mod_name = mod_name[:-9] VFS[mod_name] = [ext, data, imports, 1] else: VFS[mod_name] = [ext, data, imports] else: VFS[mod_name] = [ext, data] print("adding {}".format(mod_name)) print('{} files, {} errors'.format(nb, nb_err)) with open(filename, "w") as out: out.write('__BRYTHON__.use_VFS = true;\n') out.write('var scripts = {}\n'.format(json.dumps(VFS))) out.write('__BRYTHON__.update_VFS(scripts)\n')
parser.add_argument("file", type=str, help="target file for obfuscation") g_output.add_argument("-o", "--output", type=str, help="output destination") args = parser.parse_args() if __name__ == "__main__": filename = args.file output = args.output if args.output else filename.split( ".")[0] + "_BFUSCATED.py" start = time.time() with open(filename) as f: code = f.read().replace("\"", "'") minified = python_minifier.minify(code) if args.method == "lambda": RESULT = functions.lambda_bfuscate(minified, args) elif args.method == "len": RESULT = functions.len_bfuscate(minified, args) else: print(f"[×] Unknown obfuscation method '{args.method}'") sys.exit() with open(output, "w+") as f: f.write(RESULT) print(f"[✔] Successfully obfuscated: {filename}") print(f"Before\t\t{len(code)} characters") print(f"Minified\t{len(minified)} characters")
def make(package_name, package_path, exclude_dirs=None): print("Generating package {}".format(package_name)) VFS = {} nb = 0 if exclude_dirs is None: exclude_dirs = [] for dirpath, dirnames, filenames in os.walk(package_path): flag = False root_elts = dirpath.split(os.sep) for exclude in exclude_dirs: if exclude in root_elts: continue if '__pycache__' in dirnames: dirnames.remove("__pycache__") if dirpath == package_path: package = [] else: package = dirpath[len(package_path) + 1:].split(os.sep) for filename in filenames: ext = os.path.splitext(filename)[1] if ext not in ('.js', '.py'): continue if filename.endswith(".brython.js"): # no recursion continue nb += 1 absname = os.path.join(dirpath, filename) with open(absname, encoding='utf-8') as f: data = f.read() if ext == '.py': data = python_minifier.minify(data, preserve_lines=True) path_elts = package[:] if os.path.basename(filename) != "__init__.py": path_elts.append(os.path.basename(filename)[:-3]) fqname = ".".join(path_elts) with open(absname, encoding="utf-8") as f: tree = ast.parse(f.read()) visitor = Visitor(package_path, package) visitor.visit(tree) imports = sorted(list(visitor.imports)) mod_name = filename.replace(os.sep, '.') if package_name: mod_name = package_name + "." + mod_name mod_name, ext = os.path.splitext(mod_name) is_package = mod_name.endswith('__init__') if ext == ".py": if is_package: mod_name = mod_name[:-9] VFS[mod_name] = [ext, data, imports, 1] else: VFS[mod_name] = [ext, data, imports] else: VFS[mod_name] = [ext, data] print("adding {}".format(mod_name)) print('{} files'.format(nb)) with open(os.path.join(package_path, package_name + ".brython.js"), "w", encoding="utf-8") as out: out.write('__BRYTHON__.use_VFS = true;\n') out.write('var scripts = {}\n'.format(json.dumps(VFS))) out.write('__BRYTHON__.update_VFS(scripts)\n')
def process_unittest(filename): """Process a VFS filename for Brython.""" print("Generating {}".format(filename)) nb = 0 nb_err = 0 _main_root = os.path.dirname(filename) _VFS = {} for _mydir in ("Lib", ): for _root, _dir, _files in os.walk(os.path.join(_main_root, _mydir)): if 'unittest' not in _root: if 'test' not in _root: continue if '__pycache__' in _root: continue for _file in _files: _ext = os.path.splitext(_file)[1] if _ext not in ('.py'): continue nb += 1 file_name = os.path.join(_root, _file) encoding = "utf-8" try: src = open(file_name, encoding=encoding).read() except: encoding = "iso-8859-1" src = open(file_name, encoding=encoding).read() if _ext.lower() == '.py': try: _data = python_minifier.minify(src) except Exception as error: print(error) nb_err += 1 _vfs_filename = os.path.join(_root, _file).replace(_main_root, '') _vfs_filename = _vfs_filename.replace("\\", "/") mod_name = _vfs_filename[len(_mydir) + 2:].replace('/', '.') mod_name, ext = os.path.splitext(mod_name) is_package = mod_name.endswith('__init__') if is_package: mod_name = mod_name[:-9] _VFS[mod_name] = [_data, 1] else: _VFS[mod_name] = [_data] print(("Adding %s %s" % (mod_name, _vfs_filename))) print('%s files, %s errors' % (nb, nb_err)) with open(filename, "w") as file_to_write_VFS: file_to_write_VFS.write('__BRYTHON__.libs = __BRYTHON__.libs || {};\n') file_to_write_VFS.write("__BRYTHON__.=libs['unittest']=%s;\n\n" % json.dumps(_VFS)) file_to_write_VFS.write(""" __BRYTHON__.import_from_unittest function(mod_name){ var stored = __BRYTHON__.libs['unittest'][mod_name] if(stored!==undefined){ var module_contents = stored[0] var $is_package = stored[1] var path = 'py_unittest' var module = {name:mod_name,__class__:$B.$ModuleDict,$is_package:$is_package} if($is_package){var package=mod_name} else{ var elts = mod_name.split('.') elts.pop() var package = elts.join('.') } $B.modules[mod_name].$package = $is_package $B.modules[mod_name].__package__ = package run_py(module,path,module_contents) return true } return null } // add this import function to brython by doing the following: // <body onload="brython({custom_import_funcs:[__BRYTHON__.import_from_unittest]})"> // this will allow us to import unittest modules. """)
def minify(input_file): import python_minifier with open(input_file, 'r') as handler: return python_minifier.minify(handler.read())