def get_module_meta_path(module_description):
"""Returns the finder to be appended to sys.meta_path
module_description is a tuple of 2 elements:
format: either 'zip', 'tar', 'tar:gz', 'tar:bz' or a string to be used as module name
content: a base64 encoded string of a zip archive, a tar(gz/bz2) archive or a plain python module
"""
raw_format = module_description[0].split(':')
if raw_format[0] in ('zip', 'tar'):
f = BytesIO()
f.write(decodestring(module_description[1]))
f.seek(0)
if raw_format[0] == 'zip':
zipf = PyZipFile(f)
module_dict = dict((splitext(z.filename)[0].replace('/', '.'),
zipf.open(z.filename).read())
for z in zipf.infolist()
if splitext(z.filename)[1] == ".py")
elif raw_format[0] == 'tar':
compression = raw_format[1] if len(raw_format) > 1 else ''
tarf = taropen(fileobj=f, mode="r:" + compression)
module_dict = dict((splitext(t.name)[0].replace('/', '.'),
tarf.extractfile(t.name).read())
for t in tarf.getmembers()
if splitext(t.name)[1] == ".py")
else:
module_dict = {
module_description[0]: decodestring(module_description[1])
}
return Finder(module_dict)
def load_special_tools(self, var, ban=[]):
"""
Loads third-party extensions modules for certain programming languages
by trying to list certain files in the extras/ directory. This method
is typically called once for a programming language group, see for
example :py:mod:`waflib.Tools.compiler_c`
:param var: glob expression, for example 'cxx\_\*.py'
:type var: string
:param ban: list of exact file names to exclude
:type ban: list of string
"""
if os.path.isdir(waf_dir):
lst = self.root.find_node(waf_dir).find_node('waflib/extras').ant_glob(var)
for x in lst:
if not x.name in ban:
load_tool(x.name.replace('.py', ''))
else:
from zipfile import PyZipFile
waflibs = PyZipFile(waf_dir)
lst = waflibs.namelist()
for x in lst:
if not re.match('waflib/extras/%s' % var.replace('*', '.*'), var):
continue
f = os.path.basename(x)
doban = False
for b in ban:
r = b.replace('*', '.*')
if re.match(r, f):
doban = True
if not doban:
f = f.replace('.py', '')
load_tool(f)
def load_special_tools(self, var, ban=[]):
global waf_dir
if os.path.isdir(waf_dir):
lst = self.root.find_node(waf_dir).find_node(
'waflib/extras').ant_glob(var)
for x in lst:
if not x.name in ban:
load_tool(x.name.replace('.py', ''))
else:
from zipfile import PyZipFile
waflibs = PyZipFile(waf_dir)
lst = waflibs.namelist()
for x in lst:
if not re.match("waflib/extras/%s" % var.replace("*", ".*"),
var):
continue
f = os.path.basename(x)
doban = False
for b in ban:
r = b.replace("*", ".*")
if re.match(b, f):
doban = True
if not doban:
f = f.replace('.py', '')
load_tool(f)
def get_module_as_zip_from_module_directory(module_directory, b64enc=True, src=True):
"""Compress a module directory
@param module_directory: The module directory
@param base64enc: if True the function will encode the zip file with base64
@param src: Integrate the source files
@return: a stream to store in a file-like object
"""
RE_exclude = re.compile('(?:^\..+\.swp$)|(?:\.py[oc]$)|(?:\.bak$)|(?:\.~.~$)', re.I)
def _zippy(archive, path, src=True):
path = os.path.abspath(path)
base = os.path.basename(path)
for f in tools.osutil.listdir(path, True):
bf = os.path.basename(f)
if not RE_exclude.search(bf) and (src or bf == '__terp__.py' or not bf.endswith('.py')):
archive.write(os.path.join(path, f), os.path.join(base, f))
archname = StringIO()
archive = PyZipFile(archname, "w", ZIP_DEFLATED)
archive.writepy(module_directory)
_zippy(archive, module_directory, src=src)
archive.close()
val = archname.getvalue()
archname.close()
if b64enc:
val = base64.encodestring(val)
return val
def packaged_code(base_path: Path) -> bytes:
"""
Package code into a zip file, and return the raw zip bytes
Args:
base_path (pathlib.Path): Base path of the project
Returns:
(bytes): Raw bytes of the packaged code
"""
with TemporaryFile("w+b") as fp:
if base_path.is_dir():
with PyZipFile(fp, "w") as pzf:
pzf.writepy(base_path / "__main__.py") # type: ignore
pzf.writepy(base_path / "utils") # type: ignore
elif base_path.is_file():
with TemporaryDirectory() as new_path_str:
new_path = Path(new_path_str)
zipped = ZipFile(base_path)
zipped.extractall(new_path)
with PyZipFile(fp, "w") as pzf:
for script_path in _walk_path(new_path):
pzf.write(script_path,
script_path.relative_to(new_path))
else:
raise OSError(f"{base_path} is not a directory or a file")
fp.seek(0)
return fp.read()
def GenerateMission(filename):
# Create XML parser
parser = make_parser()
generator = MissionGenerator()
parser.setContentHandler(generator)
# Parse KMZ file
kmz = PyZipFile(filename, 'r')
kml = kmz.open('doc.kml', 'r')
parser.parse(kml)
kmz.close()
# Arrange the XML items into useful variables
items = {}
items['base_location'] = ParseCoordinates(generator.mapping, 'Base')
items['landing_site'] = ParseCoordinates(generator.mapping, 'Fake reported location')
items['path'] = ParseCoordinates(generator.mapping, 'Sample Nominal Path')
items['return_path']= items['path'].reverse()
items['base_geofence'] = GenerateGeofences(ParseCoordinates(generator.mapping, 'Base geofence'))
items['landing_site_geofence'] = GenerateGeofences(ParseCoordinates(generator.mapping, 'Sample remote landing site'))
items['mission_geofence'] = GenerateGeofences(ParseCoordinates(generator.mapping, 'Sample full geofence'))
return items
def load_special_tools(self, var, ban=[]):
"""
Loads third-party extensions modules for certain programming languages
by trying to list certain files in the extras/ directory. This method
is typically called once for a programming language group, see for
example :py:mod:`waflib.Tools.compiler_c`
:param var: glob expression, for example 'cxx\_\*.py'
:type var: string
:param ban: list of exact file names to exclude
:type ban: list of string
"""
if os.path.isdir(waf_dir):
lst = self.root.find_node(waf_dir).find_node('waflib/extras').ant_glob(var)
for x in lst:
if not x.name in ban:
load_tool(x.name.replace('.py', ''))
else:
from zipfile import PyZipFile
waflibs = PyZipFile(waf_dir)
lst = waflibs.namelist()
for x in lst:
if not re.match('waflib/extras/%s' % var.replace('*', '.*'), var):
continue
f = os.path.basename(x)
doban = False
for b in ban:
r = b.replace('*', '.*')
if re.match(r, f):
doban = True
if not doban:
f = f.replace('.py', '')
load_tool(f)
def compile_src(creator_name: str,
src_dir: str,
build_dir: str,
mods_dir: str,
mod_name: str = "Untitled") -> None:
"""
Packages your mod into a proper mod file. It creates 2 mod files, a full mod file which contains all the files
in the source folder unchanged along with the compiled python versions next to uncompiled ones and a slim mod-file
which contains only the compiled versions.
Modified from andrew's code.
https://sims4studio.com/thread/15145/started-python-scripting
:param creator_name: The creators name
:param src_dir: Source dir for the mod files
:param build_dir: Place to put the mod files
:param mods_dir: Place to an extra copy of the slim mod file for testing
:param mod_name: Name to call the mod
:return: Nothing
"""
# Prepend creator name to mod name
mod_name = creator_name + '_' + mod_name
mods_sub_dir = os.path.join(mods_dir, mod_name)
# Create ts4script paths
ts4script_full_build_path = os.path.join(build_dir,
mod_name + '.ts4script')
ts4script_mod_path = os.path.join(mods_sub_dir, mod_name + '.ts4script')
print("Clearing out old builds...")
# Delete and re-create build and sub-folder in Mods
is_devmode = symlink_exists_win("", mods_dir, mod_name)
symlink_remove_win("", mods_dir, mod_name)
if is_devmode:
print("Exiting Dev Mode...")
remove_dir(build_dir)
ensure_path_created(build_dir)
ensure_path_created(mods_sub_dir)
print("Re-building mod...")
# Compile the mod
zf = PyZipFile(ts4script_full_build_path,
mode='w',
compression=ZIP_STORED,
allowZip64=True,
optimize=2)
compile_full(src_dir, zf)
zf.close()
# Copy it over to the mods folder
shutil.copyfile(ts4script_full_build_path, ts4script_mod_path)
print("----------")
print("Complete")
def extract_subfolder(root, filename, ea_folder):
src = os.path.join(root, filename)
dst = os.path.join(ea_folder, filename)
if src != dst:
shutil.copyfile(src, dst)
zip = PyZipFile(dst)
out_folder = os.path.join(ea_folder, os.path.splitext(filename)[0])
zip.extractall(out_folder)
decompile_dir(out_folder)
pass
def build_zip(dest):
print "Writing", dest
from zipfile import PyZipFile
f = PyZipFile(dest, "w")
f.writepy("src/singleshot")
f.writepy("lib")
f.writepy("lib/simpletal")
f.close()
def fill_queue(self, curr_folder):
for root, dirs, files in os.walk(curr_folder):
for ext_filter in self.script_package_types:
for filename in fnmatch.filter(files, ext_filter):
src = os.path.join(root, filename)
dst = os.path.join(self.ea_folder, filename)
if src != dst:
shutil.copyfile(src, dst)
zip = PyZipFile(dst)
out_folder = os.path.join(self.ea_folder,
os.path.splitext(filename)[0])
zip.extractall(out_folder)
self.q.put(out_folder)
def build_zip(module_dir):
# This can fail at writepy if there is something wrong with the files
# in xframes. Go ahead anyway, but things will probably fail if this job is
# distributed
try:
tf = NamedTemporaryFile(suffix=".zip", delete=False)
z = PyZipFile(tf, "w")
z.writepy(module_dir)
z.close()
return tf.name
except:
logging.warn("Zip file distribution failed -- workers will not get xframes code.")
logging.warn("Check for unexpected files in xframes directory.")
return None
def unpack_catapult(bindir):
path = os.path.join(bindir, 'catapult', 'tracing')
if os.path.exists(path):
message('info', 'Catapult is taken from: ' + path)
return path
zip_path = os.path.join(bindir, 'catapult.zip')
if os.path.exists(zip_path):
print("Extracting catapult...")
from zipfile import PyZipFile
pzf = PyZipFile(zip_path)
pzf.extractall(bindir)
return path
else:
print("Error: Catapult is not found at", zip_path)
return None
def get_catapult_path(args):
if 'INTEL_SEA_CATAPULT' in os.environ and os.path.exists(os.environ['INTEL_SEA_CATAPULT']):
return os.environ['INTEL_SEA_CATAPULT']
else:
path = os.path.join(args.bindir, 'catapult', 'tracing')
if os.path.exists(path):
return path
zip_path = os.path.join(args.bindir, 'catapult.zip')
if os.path.exists(zip_path):
print "Extracting catapult..."
from zipfile import PyZipFile
pzf = PyZipFile(zip_path)
pzf.extractall(args.bindir)
return path
return None
def _extract_sub_folder(cls,
root: str,
filename: str,
ea_folder: str,
decompile_files: bool = True):
src = os.path.join(root, filename)
dst = os.path.join(ea_folder, filename)
if src != dst:
shutil.copyfile(src, dst)
py_zip = PyZipFile(dst)
out_folder = os.path.join(ea_folder, os.path.splitext(filename)[0])
py_zip.extractall(out_folder)
if decompile_files:
cls.decompile_folder(out_folder)
pass
def infoID() -> Tuple[Dict[str, List[str]], Dict[str, str]]:
with PyZipFile(os.path.abspath(file), "r") as zipFile:
with zipFile.open("name.json", "r") as f:
name = json.loads(f.read().decode())
with zipFile.open("area.json", "r") as f:
area = json.loads(f.read().decode())
return name, area
def test_build(self):
builder = Builder('test.zip')
builder._runtime = RUNTIME_NODE_JS
ok_(hasattr(builder.build(), 'read'))
with PyZipFile(builder._zippath, 'r', compression=ZIP_DEFLATED) as zipfile:
ok_('lambda_function.pyc' in zipfile.namelist())
ok_('.lamvery_secret.json' in zipfile.namelist())
def test_iterative_read():
with tempfile.TemporaryFile() as tf:
with PyZipFile(tf, 'w') as zf:
with zf.open('sample.txt', 'w') as out:
# 10MB sample file.
out.write(b'P' * 0xA00000)
tf.seek(0)
zf = ZipFile(tf)
assert len(zf) == 1
it = zf.read_iter(b'sample.txt', max_chunk_size=0x100000)
chunk_count = 0
complete_size = 0
for chunk in it:
chunk_count += 1
complete_size += len(chunk)
for char in chunk:
assert char == 0x50
# We're reading a 10MB file in 1MB chunks, we should always have
# 10 iterations. Note that miniz guarantees returning once the
# max_chunk_size is hit or the file is consumed.
assert chunk_count == 10
assert complete_size == 0xA00000
def build(self):
if self._clean_build:
self._prepare_clean_build()
self._run_hooks(self._hooks.get('pre', []))
with PyZipFile(self._zippath, 'w',
compression=ZIP_DEFLATED) as zipfile:
for p in self._get_paths():
if os.path.isdir(p):
self._archive_dir(zipfile, p)
else:
self._archive_file(zipfile, p)
if not self._single_file:
secret_path = os.path.join(self._tmpdir,
lamvery.secret.SECRET_FILE_NAME)
env_path = os.path.join(self._tmpdir,
lamvery.env.ENV_FILE_NAME)
self._generate_json(secret_path, self._secret)
self._generate_json(env_path, self._env)
self._archive_file(zipfile, secret_path)
self._archive_file(zipfile, env_path)
if self._runtime == lamvery.config.RUNTIME_NODE_JS:
self._archive_dist(zipfile, 'lamvery.js')
self._run_hooks(self._hooks.get('post', []))
return open(self._zippath, 'rb')
def _update_software(self, data):
logging.info("Starting software upgrade")
with open('update.zip', 'wb') as update_file:
update_file.write(data)
zip = PyZipFile('update.zip')
zip.extractall()
logging.info("Update extracted")
try:
with open('version.txt', 'r') as version_file:
version = version_file.readline()
logging.info("Upgrade software to version {0}".format(version))
self._config["VERSIONS"]["DeviceProgramVersion"] = version
with open('settings.ini', 'w') as configfile:
self._config.write(configfile)
except Exception as e:
logging.error("Upgrade failed: ERROR MESSAGE: {0}".format(repr(e)))
def test_create_zipfile(self):
archive = Archive('test.zip')
ok_(hasattr(archive.create_zipfile(), 'read'))
with PyZipFile(archive._zippath, 'r',
compression=ZIP_DEFLATED) as zipfile:
ok_('lambda_function.pyc' in zipfile.namelist())
ok_('.lamvery_secret.json' in zipfile.namelist())
def _loadMainData() -> Tuple[Dict[str, str], Dict[str, List[str]]]:
with PyZipFile(_DATA_BINARY_DIR, "r") as zipFile:
with zipFile.open("name.json", "r") as f:
nameLoad = json.loads(f.read().decode())
with zipFile.open("area.json", "r") as f:
areaLoad = json.loads(f.read().decode())
return areaLoad, nameLoad
def build_zip():
if 'XPATTERNS_HOME' not in os.environ:
return None
# This can fail at writepy if there is something wrong with the files
# in xpatterns. Go ahead anyway, but things will probably fail of this job is
# distributed
try:
tf = NamedTemporaryFile(suffix='.zip', delete=False)
z = PyZipFile(tf, 'w')
z.writepy(os.path.join(os.environ['XPATTERNS_HOME'], 'xpatterns'))
z.close()
return tf.name
except:
print 'Zip file distribution failed -- workers will not get xpatterns code.'
print 'Check for unexpected files in XPATTERNS_HOME/xpatterns.'
return None
def __init__(self, filename):
zipf = PyZipFile(filename)
# test this with corrupt zipfile
zipf.testzip()
size = self._get_extracted_zipsize(zipf)
# if size > 1gb
if size > 1000000000:
raise BufferError('filesize exceeds 1GB')
friends = self._extract_friends(zipf)
albums = self._extract_albums(zipf)
messages = self._extract_messages(zipf)
posts = self._extract_posts(zipf)
self.profile = profile_import(friends, albums, messages, posts)
def test_open_fd():
"""Ensure we can open file-like objects implementing fileno()."""
with tempfile.TemporaryFile() as tf:
# Generate an empty ZipFile.
with PyZipFile(tf, 'w'):
pass
tf.seek(0)
ZipFile(tf)
def __init__(self, fobj):
self.zf = fobj if isinstance(fobj, PyZipFile) else PyZipFile(fobj)
# for info in self._zf.infolist():
# print(info)
self.node_tree = self.read_node('OFD.xml')
# parse node
self.document_node = self.read_node(self.node_tree['DocBody']['DocRoot'].text)
self.document = OFDDocument(self.zf, self.document_node)
def test_create_zipfile_with_single_file(self):
archive = Archive('test.zip',
function_filename='lambda_function.py',
single_file=True)
archive.create_zipfile()
with PyZipFile(archive._zippath, 'r',
compression=ZIP_DEFLATED) as zipfile:
ok_('lambda_function.py' in zipfile.namelist())
ok_(not ('.lamvery_secret.json' in zipfile.namelist()))
def test_open_by_path():
"""Ensure we can open a file given a filesystem path."""
with tempfile.NamedTemporaryFile() as tf:
# Generate an empty ZipFile.
with PyZipFile(tf, 'w'):
pass
tf.seek(0)
# `tf.name` being the full filesystem path.
ZipFile(tf.name.encode('UTF-8'))
def _load(cls, path):
"""
:type path: str
"""
bytesIO = BytesIO()
# PyZipFile compiles .py files on the fly, filters out any non-Python files and
# distinguishes between packages and simple directories.
with PyZipFile(file=bytesIO, mode='w') as zipFile:
zipFile.writepy(path)
bytesIO.seek(0)
return bytesIO
def create_zipfile(self):
lamvery.secret.generate(self._secretpath, self._secret)
with PyZipFile(self._zippath, 'w',
compression=ZIP_DEFLATED) as zipfile:
for p in self._get_paths():
if os.path.isdir(p):
self._archive_dir(zipfile, p)
else:
self._archive_file(zipfile, p)
if not self._single_file:
self._archive_file(zipfile, self._secretpath)
return open(self._zippath, 'rb')
def get_module_meta_path(module_description):
"""Returns the finder to be appended to sys.meta_path
module_description is a tuple of 2 elements:
format: either 'zip', 'tar', 'tar:gz', 'tar:bz' or a string to be used as module name
content: a base64 encoded string of a zip archive, a tar(gz/bz2) archive or a plain python module
"""
raw_format = module_description[0].split(':')
if raw_format[0] in ('zip', 'tar'):
f = BytesIO()
f.write(decodestring(module_description[1]))
f.seek(0)
if raw_format[0] == 'zip':
zipf = PyZipFile(f)
module_dict = dict((splitext(z.filename)[0].replace('/', '.'), zipf.open(z.filename).read()) for z in zipf.infolist() if splitext(z.filename)[1] == ".py")
elif raw_format[0] == 'tar':
compression = raw_format[1] if len(raw_format) > 1 else ''
tarf = taropen(fileobj=f, mode="r:" + compression)
module_dict = dict((splitext(t.name)[0].replace('/', '.'), tarf.extractfile(t.name).read()) for t in tarf.getmembers() if splitext(t.name)[1] == ".py")
else:
module_dict = {module_description[0]: decodestring(module_description[1])}
return Finder(module_dict)
def dump_func(func):
mod = inspect.getmodule(func)
mod_parts = []
mod_parts.append(mod.__package__ if mod.__package__ else '')
if mod.__name__ == '__main__':
srcfile = Path(os.path.abspath(inspect.getsourcefile(mod)))
mod_parts.append(srcfile.stem)
else:
mod_parts.append(mod.__name__)
mod_parts.append(func.__qualname__)
func_bin = '|'.join(mod_parts)
if _check_func_from_zip(mod):
m = _pk_re.search(mod.__file__)
if not m:
raise RuntimeError(
'Modele from zip import, can not find aws package')
pkfile = mod.__file__[:m.end()]
with open(pkfile, 'rb') as infile:
func_content = infile.read()
elif mod.__package__ and (mod.__package__ == 'awsfrwk'
or mod.__package__.startswith('awsfrwk.')):
# be called in local
func_content = None
else:
# package file and dir
srcfile = Path(os.path.abspath(inspect.getsourcefile(mod)))
curdir = Path(os.path.abspath('.'))
try:
reldir = srcfile.relative_to(curdir)
except ValueError:
raise RuntimeError('Function %s: source file must in current dir' %
func.__name__)
targetpath = curdir.joinpath(reldir.parts[0])
def _enum_dir(p, zipf):
for sp in p.iterdir():
if sp.is_dir():
_enum_dir(sp, zipf)
continue
suffix = sp.suffix.lower()
if suffix != '.py' and suffix != '.pyc' and suffix != '.pyo':
relp = sp.relative_to(targetpath.parent)
zipf.write(str(sp), str(relp))
with TemporaryFile(dir=runtime_dir) as tmpf:
with PyZipFile(tmpf, "w") as zipfp:
zipfp.writepy(str(targetpath))
if targetpath.is_dir():
_enum_dir(targetpath, zipfp)
tmpf.seek(0)
func_content = tmpf.read()
return func_bin, func_content
def debug_install_mod(mod_src: str, mods_dir: str, mod_name: str,
mod_folder_name: str) -> None:
"""
Compiles and installs the mod which adds a cheat code so the user can setup the debugger in-game
:param mod_src: Path to the script which does this
:param mods_dir: Path to the users mod folder
:param mod_name: Name of the mod
:param mod_folder_name: Name of mod Subfolder
:return: Nothing
"""
print("Compiling and installing the cheatcode mod...")
# Get destination file path
mods_sub_dir = os.path.join(mods_dir, mod_folder_name)
mod_path = os.path.join(mods_sub_dir, mod_name + '.ts4script')
ensure_path_created(mods_sub_dir)
# Get compiled path and compile mod
mod_src_pyc = replace_extension(mod_src, "pyc")
py_compile.compile(mod_src, mod_src_pyc)
# Create mod at destination and add compiled file to it
zf = PyZipFile(mod_path,
mode='w',
compression=ZIP_STORED,
allowZip64=True,
optimize=2)
zf.write(mod_src_pyc, mod_name + ".pyc")
zf.close()
def load_special_tools(self, var, ban=[]):
global waf_dir
if os.path.isdir(waf_dir):
lst = self.root.find_node(waf_dir).find_node('waflib/extras').ant_glob(var)
for x in lst:
if not x.name in ban:
load_tool(x.name.replace('.py', ''))
else:
from zipfile import PyZipFile
waflibs = PyZipFile(waf_dir)
lst = waflibs.namelist()
for x in lst:
if not re.match("waflib/extras/%s" % var.replace("*", ".*"), var):
continue
f = os.path.basename(x)
doban = False
for b in ban:
r = b.replace("*", ".*")
if re.match(b, f):
doban = True
if not doban:
f = f.replace('.py', '')
load_tool(f)
def compile_full(src_dir: str, zf: PyZipFile) -> None:
"""
Compiles a full mod (Contains all files in source including python files which it then compiles
Modified from andrew's code.
https://sims4studio.com/thread/15145/started-python-scripting
:param src_dir: source folder
:param zf: Zip File Handle
:return: Nothing
"""
for folder, subs, files in os.walk(src_dir):
for filename in fnmatch.filter(files, '*.py'):
file_path_py = folder + os.sep + filename
file_path_pyc = replace_extension(file_path_py, "pyc")
rel_path_pyc = get_rel_path(file_path_pyc, src_dir)
py_compile.compile(file_path_py, file_path_pyc)
zf.write(file_path_pyc, rel_path_pyc)
remove_file(file_path_pyc)
for filename in fnmatch.filter(files, '*[!p][!y][!c]'):
rel_path = get_rel_path(folder + os.sep + filename, src_dir)
zf.write(folder + os.sep + filename, rel_path)
def get_zip_from_directory(directory, b64enc=True):
RE_exclude = re.compile('(?:^\..+\.swp$)|(?:\.py[oc]$)|(?:\.bak$)|(?:\.~.~$)', re.I)
def _zippy(archive, path):
path = os.path.abspath(path)
base = os.path.basename(path)
for f in tools.osutil.listdir(path, True):
bf = os.path.basename(f)
if not RE_exclude.search(bf):
archive.write(os.path.join(path, f), os.path.join(base, f))
archname = StringIO()
archive = PyZipFile(archname, "w", ZIP_DEFLATED)
archive.writepy(directory)
_zippy(archive, directory)
archive.close()
val = archname.getvalue()
archname.close()
if b64enc:
val = base64.encodestring(val)
return val
def update_library(obfdist, libzip):
'''Update compressed library generated by py2exe or cx_Freeze, replace
the original scripts with obfuscated ones.
'''
# # It's simple ,but there are duplicated .pyc files
# with PyZipFile(libzip, 'a') as f:
# f.writepy(obfdist)
filelist = []
for root, dirs, files in os.walk(obfdist):
filelist.extend([os.path.join(root, s) for s in files])
with PyZipFile(libzip, 'r') as f:
namelist = f.namelist()
f.extractall(obfdist)
for s in filelist:
if s.lower().endswith('.py'):
compile_file(s, s + 'c')
with PyZipFile(libzip, 'w') as f:
for name in namelist:
f.write(os.path.join(obfdist, name), name)
def compile_slim(src_dir: str, zf: PyZipFile) -> None:
"""
Compiles a slim mod (Contains only the pyc files)
Modified from andrew's code.
https://sims4studio.com/thread/15145/started-python-scripting
It is not reccomended to use this function because it's not reccomended to only have pyc files in your project
as it makes your project less flexible. Going forward this will nto be called by default.
:param src_dir: source folder
:param zf: Zip File Handle
:return: Nothing
"""
for folder, subs, files in os.walk(src_dir):
for filename in fnmatch.filter(files, '*.py'):
file_path_py = folder + os.sep + filename
file_path_pyc = replace_extension(file_path_py, "pyc")
rel_path_pyc = get_rel_path(file_path_pyc, src_dir)
py_compile.compile(file_path_py, file_path_pyc)
zf.write(file_path_pyc, rel_path_pyc)
remove_file(file_path_pyc)
def _load(cls, path):
sitePackages = path
assert os.path.basename(sitePackages) == 'site-packages'
bytesIO = BytesIO()
with PyZipFile(file=bytesIO, mode='w') as zipFile:
# This adds the .py files but omits subdirectories since site-packages is not a package
zipFile.writepy(sitePackages)
# Now add the missing packages
for name in os.listdir(sitePackages):
path = os.path.join(sitePackages, name)
if os.path.isdir(path) and os.path.isfile(os.path.join(path, '__init__.py')):
zipFile.writepy(path)
bytesIO.seek(0)
return bytesIO
def zip_directory(directory, b64enc=True, src=True):
"""Compress a directory
@param directory: The directory to compress
@param base64enc: if True the function will encode the zip file with base64
@param src: Integrate the source files
@return: a string containing the zip file
"""
RE_exclude = re.compile('(?:^\..+\.swp$)|(?:\.py[oc]$)|(?:\.bak$)|(?:\.~.~$)', re.I)
def _zippy(archive, path, src=True):
path = os.path.abspath(path)
base = os.path.basename(path)
for f in tools.osutil.listdir(path, True):
bf = os.path.basename(f)
if not RE_exclude.search(bf) and (src or bf in ('__openerp__.py', '__terp__.py') or not bf.endswith('.py')):
archive.write(os.path.join(path, f), os.path.join(base, f))
archname = StringIO()
archive = PyZipFile(archname, "w", ZIP_DEFLATED)
# for Python 2.5, ZipFile.write() still expects 8-bit strings (2.6 converts to utf-8)
directory = tools.ustr(directory).encode('utf-8')
archive.writepy(directory)
_zippy(archive, directory, src=src)
archive.close()
archive_data = archname.getvalue()
archname.close()
if b64enc:
return base64.encodestring(archive_data)
return archive_data
class EPub(object):
epub_file = None
epub_zip = None
title = None
def __init__(self, epub_file):
if not epub_file:
raise ValueError('Invalid epub file path')
self.epub_file = epub_file
self.epub_zip = PyZipFile(epub_file, 'r')
def table_of_contents(self):
basedir = None
# find opf file
soup = BeautifulSoup(self.epub_zip.read('META-INF/container.xml'))
opf = dict(soup.find('rootfile').attrs)['full-path']
basedir = os.path.dirname(opf)
if basedir:
basedir = '{0}/'.format(basedir)
soup = BeautifulSoup(self.epub_zip.read(opf), "html.parser")
# title
self.title = soup.find('dc:title').text
# all files, not in order
x, ncx = {}, None
for item in soup.find('manifest').findAll('item'):
d = dict(item.attrs)
x[d['id']] = '{0}{1}'.format(basedir, d['href'])
if d['media-type'] == 'application/x-dtbncx+xml':
ncx = '{0}{1}'.format(basedir, d['href'])
book_sections = []
for item in soup.find('spine').findAll('itemref'):
book_sections.append(x[dict(item.attrs)['idref']])
nav_labels = {}
if ncx:
soup = BeautifulSoup(self.epub_zip.read(ncx), "html.parser")
for navpoint in soup('navpoint'):
k = navpoint.content.get('src', None)
# strip off any anchor text
k = k.split('#')[0]
if k:
nav_labels[k] = navpoint.navlabel.text
toc = []
for section in book_sections:
if section in nav_labels:
toc.append(nav_labels[section])
else:
toc.append(section)
return toc
def chapter_to_text(self, chapter):
soup = BeautifulSoup(self.epub_zip.read(chapter), "html.parser")
return soup.find('body').getText()
def book_parts(self):
toc = self.table_of_contents()
book = []
for chapter in toc:
book.append((chapter, self.chapter_to_text(chapter)))
return book
class BuilderTestCase(TestCase):
def setUp(self):
tmp = tempfile.mkstemp(prefix=__name__)
self.zipfile_path = tmp[1]
self.zipfile = PyZipFile(self.zipfile_path, 'w')
self.pj_root = os.path.abspath(
os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
def tearDown(self):
os.remove(self.zipfile_path)
try:
os.remove('test.json')
except:
pass
def test_build(self):
builder = Builder('test.zip')
builder._runtime = RUNTIME_NODE_JS
ok_(hasattr(builder.build(), 'read'))
with PyZipFile(builder._zippath, 'r', compression=ZIP_DEFLATED) as zipfile:
ok_('lambda_function.pyc' in zipfile.namelist())
ok_('.lamvery_secret.json' in zipfile.namelist())
def test_build_with_single_file(self):
builder = Builder('test.zip', function_filename='lambda_function.py', single_file=True)
builder.build()
with PyZipFile(builder._zippath, 'r', compression=ZIP_DEFLATED) as zipfile:
ok_('lambda_function.py' in zipfile.namelist())
ok_(not ('.lamvery_secret.json' in zipfile.namelist()))
def test_generate_json(self):
builder = Builder('test.zip')
builder._generate_json(
JSON_FILE_NAME, {'foo': 2, 'bar': 3})
data = json.load(open(JSON_FILE_NAME, 'r'))
eq_(data.get('foo'), 2)
def test_archive_dir(self):
builder = Builder('test.zip')
builder._archive_dir(self.zipfile, self.pj_root)
ok_(isinstance(self.zipfile.getinfo('setup.pyc'), zipfile.ZipInfo))
def test_archive_file(self):
builder = Builder('test.zip')
builder._archive_file(
self.zipfile, os.path.join(self.pj_root, 'setup.py'))
ok_(isinstance(self.zipfile.getinfo('setup.pyc'), zipfile.ZipInfo))
builder._archive_file(
self.zipfile, os.path.join(self.pj_root, 'README.md'))
ok_(isinstance(self.zipfile.getinfo('README.md'), zipfile.ZipInfo))
def test_archive_dist(self):
builder = Builder('test.zip')
builder._archive_dist(self.zipfile, 'lamvery.js')
ok_(isinstance(self.zipfile.getinfo('lamvery.js'), zipfile.ZipInfo))
@raises(KeyError)
def test_archive_single_file_key_error(self):
self._single_file = True
builder = Builder('test.zip', single_file=True)
builder._archive_file(
self.zipfile, os.path.join(self.pj_root, 'setup.py'))
ok_(isinstance(self.zipfile.getinfo('setup.pyc'), zipfile.ZipInfo))
def test_archive_single_file(self):
self._single_file = True
builder = Builder('test.zip', single_file=True)
builder._archive_file(
self.zipfile, os.path.join(self.pj_root, 'setup.py'))
ok_(isinstance(self.zipfile.getinfo('setup.py'), zipfile.ZipInfo))
def test_is_exclude(self):
builder = Builder('test.zip', exclude=['^\.lamvery\.yml$'])
eq_(builder.is_exclude('foo.txt'), False)
eq_(builder.is_exclude('.lamvery.yml'), True)
def test_is_exclude_file(self):
builder = Builder('test.zip')
eq_(builder.is_exclude_file('test.zip'), True)
eq_(builder.is_exclude_file('foo.txt'), False)
builder.is_exclude = Mock(return_value=True)
eq_(builder.is_exclude_file('foo.txt'), True)
def test_is_exclude_dir(self):
builder = Builder('test.zip')
eq_(builder.is_exclude_dir('.git'), True)
eq_(builder.is_exclude_dir('foo'), False)
builder.is_exclude = Mock(return_value=True)
eq_(builder.is_exclude_file('foo'), True)
def test_is_source_file(self):
builder = Builder('test.zip')
eq_(builder.is_source_file('foo.py'), True)
eq_(builder.is_source_file('foo.pyc'), True)
eq_(builder.is_source_file('foo.php'), False)
def test_get_paths(self):
builder = Builder('test.zip')
paths = builder._get_paths()
ok_(os.path.join(self.pj_root, 'lamvery') in paths)
builder = Builder('test.zip', no_libs=True)
paths = builder._get_paths()
ok_(os.path.join(self.pj_root, 'lamvery') in paths)
ok_(os.path.join(self.pj_root, 'lambda_function.py') in paths)
ok_(os.path.join(self.pj_root, 'lambda_function.pyc') in paths)
ok_(os.path.join(self.pj_root, '.lamvery.yml') in paths)
builder = Builder('test.zip', function_filename='test.py', single_file=True)
paths = builder._get_paths()
ok_(os.path.join(self.pj_root, 'test.py') in paths)
ok_(not os.path.join(self.pj_root, 'lambda_function.pyc') in paths)
ok_(not os.path.join(self.pj_root, '.lamvery.yml') in paths)
builder = Builder('test.zip', clean_build=True)
paths = builder._get_paths()
ok_(os.path.join(self.pj_root, 'lamvery') not in paths)
del os.environ['VIRTUAL_ENV']
builder = Builder('test.zip')
paths = builder._get_paths()
ok_(os.path.join(self.pj_root, 'lamvery') in paths)
def test_get_size(self):
builder = Builder('test.zip')
builder.build()
ok_(isinstance(builder.get_size(), int))
class ArchiveTestCase(TestCase):
def setUp(self):
tmp = tempfile.mkstemp(prefix=__name__)
self.zipfile_path = tmp[1]
self.zipfile = PyZipFile(self.zipfile_path, 'w')
self.pj_root = os.path.abspath(
os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
def tearDown(self):
os.remove(self.zipfile_path)
def test_create_zipfile(self):
archive = Archive('test.zip')
ok_(hasattr(archive.create_zipfile(), 'read'))
with PyZipFile(archive._zippath, 'r', compression=ZIP_DEFLATED) as zipfile:
ok_('lambda_function.pyc' in zipfile.namelist())
ok_('.lamvery_secret.json' in zipfile.namelist())
def test_create_zipfile_with_single_file(self):
archive = Archive('test.zip', function_filename='lambda_function.py', single_file=True)
archive.create_zipfile()
with PyZipFile(archive._zippath, 'r', compression=ZIP_DEFLATED) as zipfile:
ok_('lambda_function.py' in zipfile.namelist())
ok_(not ('.lamvery_secret.json' in zipfile.namelist()))
def test_archive_dir(self):
archive = Archive('test.zip')
archive._archive_dir(self.zipfile, self.pj_root)
ok_(isinstance(self.zipfile.getinfo('setup.pyc'), zipfile.ZipInfo))
def test_archive_file(self):
archive = Archive('test.zip')
archive._archive_file(
self.zipfile, os.path.join(self.pj_root, 'setup.py'))
ok_(isinstance(self.zipfile.getinfo('setup.pyc'), zipfile.ZipInfo))
archive._archive_file(
self.zipfile, os.path.join(self.pj_root, 'README.md'))
ok_(isinstance(self.zipfile.getinfo('README.md'), zipfile.ZipInfo))
@raises(KeyError)
def test_archive_single_file_key_error(self):
self._single_file = True
archive = Archive('test.zip', single_file=True)
archive._archive_file(
self.zipfile, os.path.join(self.pj_root, 'setup.py'))
ok_(isinstance(self.zipfile.getinfo('setup.pyc'), zipfile.ZipInfo))
def test_archive_single_file(self):
self._single_file = True
archive = Archive('test.zip', single_file=True)
archive._archive_file(
self.zipfile, os.path.join(self.pj_root, 'setup.py'))
ok_(isinstance(self.zipfile.getinfo('setup.py'), zipfile.ZipInfo))
def test_is_exclude(self):
archive = Archive('test.zip', exclude=['^\.lamvery\.yml$'])
eq_(archive.is_exclude('foo.txt'), False)
eq_(archive.is_exclude('.lamvery.yml'), True)
def test_is_exclude_file(self):
archive = Archive('test.zip')
eq_(archive.is_exclude_file('test.zip'), True)
eq_(archive.is_exclude_file('foo.txt'), False)
archive.is_exclude = Mock(return_value=True)
eq_(archive.is_exclude_file('foo.txt'), True)
def test_is_exclude_dir(self):
archive = Archive('test.zip')
eq_(archive.is_exclude_dir('.git'), True)
eq_(archive.is_exclude_dir('foo'), False)
archive.is_exclude = Mock(return_value=True)
eq_(archive.is_exclude_file('foo'), True)
def test_is_source_file(self):
archive = Archive('test.zip')
eq_(archive.is_source_file('foo.py'), True)
eq_(archive.is_source_file('foo.pyc'), True)
eq_(archive.is_source_file('foo.php'), False)
def test_get_paths(self):
archive = Archive('test.zip')
paths = archive._get_paths()
ok_(os.path.join(self.pj_root, 'lamvery') in paths)
archive = Archive('test.zip', no_libs=True)
paths = archive._get_paths()
ok_(os.path.join(self.pj_root, 'lamvery') in paths)
ok_(os.path.join(self.pj_root, 'lambda_function.py') in paths)
ok_(os.path.join(self.pj_root, 'lambda_function.pyc') in paths)
ok_(os.path.join(self.pj_root, '.lamvery.yml') in paths)
archive = Archive('test.zip', function_filename='test.py', single_file=True)
paths = archive._get_paths()
ok_(os.path.join(self.pj_root, 'test.py') in paths)
ok_(not os.path.join(self.pj_root, 'lambda_function.pyc') in paths)
ok_(not os.path.join(self.pj_root, '.lamvery.yml') in paths)
del os.environ['VIRTUAL_ENV']
archive = Archive('test.zip')
paths = archive._get_paths()
ok_(os.path.join(self.pj_root, 'lamvery') in paths)
def test_get_size(self):
archive = Archive('test.zip')
archive.create_zipfile()
ok_(isinstance(archive.get_size(), int))
#!/usr/bin/env python
#
# Packs Python standard library into zip file python$(ver).zip
#
import os, os.path, shutil, sys
from zipfile import PyZipFile
name = "python%i%i.zip" % (sys.version_info[0], sys.version_info[1])
print "creating %s..." % name
# delete tests, we don't need them:
for root, dirs, files in os.walk("Lib", topdown=False):
if "test" in dirs:
shutil.rmtree(os.path.join(root, "test"))
# pack Lib to a zipfile:
zip = PyZipFile(name, mode="w")
for f in os.listdir("Lib"):
fn = os.path.join("Lib", f)
if os.path.isdir(fn) or fn.endswith(".py"):
zip.writepy(fn)
else:
print "warning: ignoring file %s" % f
zip.close()
def setUp(self):
tmp = tempfile.mkstemp(prefix=__name__)
self.zipfile_path = tmp[1]
self.zipfile = PyZipFile(self.zipfile_path, 'w')
self.pj_root = os.path.abspath(
os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
def __init__(self, epub_file):
if not epub_file:
raise ValueError('Invalid epub file path')
self.epub_file = epub_file
self.epub_zip = PyZipFile(epub_file, 'r')
