def testWritePyfile(self):
zipfp = zipfile.PyZipFile(TemporaryFile(), "w")
fn = __file__
if fn.endswith('.pyc') or fn.endswith('.pyo'):
fn = fn[:-1]
elif fn.endswith('$py.class'):
fn = fn[:-9] + '.py'
zipfp.writepy(fn)
bn = os.path.basename(fn)
self.assertTrue(bn not in zipfp.namelist())
if not is_jython:
self.assertTrue(bn + 'o' in zipfp.namelist() or bn + 'c' in zipfp.namelist())
else:
self.assertTrue(bn[:-3] + '$py.class' in zipfp.namelist())
zipfp.close()
zipfp = zipfile.PyZipFile(TemporaryFile(), "w")
fn = __file__
if fn.endswith('.pyc') or fn.endswith('.pyo'):
fn = fn[:-1]
elif fn.endswith('$py.class'):
fn = fn[:-9] + '.py'
zipfp.writepy(fn, "testpackage")
bn = "%s/%s"%("testpackage", os.path.basename(fn))
self.assertTrue(bn not in zipfp.namelist())
if not is_jython:
self.assertTrue(bn + 'o' in zipfp.namelist() or bn + 'c' in zipfp.namelist())
else:
self.assertTrue(bn[:-3] + '$py.class' in zipfp.namelist())
zipfp.close()
def testWritePyfile(self):
zipfp = zipfile.PyZipFile(TemporaryFile(), "w")
fn = __file__
if fn.endswith('.pyc') or fn.endswith('.pyo'):
fn = fn[:-1]
zipfp.writepy(fn)
bn = os.path.basename(fn)
self.assert_(bn not in zipfp.namelist())
self.assert_(bn + 'o' in zipfp.namelist()
or bn + 'c' in zipfp.namelist())
zipfp.close()
zipfp = zipfile.PyZipFile(TemporaryFile(), "w")
fn = __file__
if fn.endswith('.pyc') or fn.endswith('.pyo'):
fn = fn[:-1]
zipfp.writepy(fn, "testpackage")
bn = "%s/%s" % ("testpackage", os.path.basename(fn))
self.assert_(bn not in zipfp.namelist())
self.assert_(bn + 'o' in zipfp.namelist()
or bn + 'c' in zipfp.namelist())
zipfp.close()
def test_pyzipfile():
myzip = zipfile.PyZipFile("spam.zip", "w") # [consider-using-with]
with zipfile.PyZipFile("spam.zip", "w"): # must not trigger
pass
_ = myzip.open("eggs.txt") # [consider-using-with]
with myzip.open("eggs.txt"): # must not trigger
pass
def setUp(self):
# Create small zipfile with just a few libraries
lib = "libsmall.zip"
lib = os.path.abspath(os.path.join(os.path.dirname(__file__), lib))
if os.path.exists(lib):
os.unlink(lib)
zf = zipfile.PyZipFile(lib, "w", compression=zipfile.ZIP_DEFLATED)
zf.writepy(os.path.dirname(zipimportx.__file__))
zf.writepy(os.path.dirname(distutils.__file__))
zf.writepy(os.path.dirname(logging.__file__))
zf.writepy(os.path.dirname(email.__file__))
zf.writepy(os.path.dirname(sqlite3.__file__))
zf.writepy(os.path.dirname(ctypes.__file__))
zf.close()
# Create medium zipfile with everything in top-level stdlib
lib = "libmedium.zip"
lib = os.path.abspath(os.path.join(os.path.dirname(__file__), lib))
if os.path.exists(lib):
os.unlink(lib)
zf = zipfile.PyZipFile(lib, "w")
zf.writepy(os.path.dirname(zipimportx.__file__))
zf.writepy(LIBHOME)
zf.close()
# Create large zipfile with everything we can find
lib = "liblarge.zip"
lib = os.path.abspath(os.path.join(os.path.dirname(__file__), lib))
if os.path.exists(lib):
os.unlink(lib)
zf = zipfile.PyZipFile(lib, "w")
zf.writepy(os.path.dirname(zipimportx.__file__))
zf.writepy(LIBHOME)
for (dirnm, subdirs, files) in os.walk(LIBHOME):
if os.path.basename(dirnm) in (
"lib-python",
"test",
):
del subdirs[:]
continue
if "__init__.pyc" in files:
del subdirs[:]
try:
zf.writepy(dirnm)
except (
EnvironmentError,
SyntaxError,
):
pass
zf.close()
def create_spark(session_name,
spark=SparkDefault.MASTER_ADDRESS,
spark_local_dir=SparkDefault.LOCAL_DIR,
config=SparkDefault.CONFIG,
packages=SparkDefault.PACKAGES,
spark_log_level=SparkDefault.LOG_LEVEL,
dep_zip=False):
log = logging.getLogger("spark")
log.info("Starting %s on %s", session_name, spark)
builder = SparkSession.builder.master(spark).appName(session_name)
builder = builder.config("spark.jars.packages", ",".join(packages))
builder = builder.config("spark.local.dir", spark_local_dir)
for cfg in config:
builder = builder.config(*cfg.split("=", 1))
session = builder.getOrCreate()
session.sparkContext.setLogLevel(spark_log_level)
# Hide py4j verbose logging (It appears in travis mostly)
logging.getLogger("py4j").setLevel(logging.WARNING)
if dep_zip:
zip_path = os.path.expanduser("~/.cache/sourced/ml/sourced.zip")
if not os.path.exists(zip_path):
os.makedirs(zip_path.split("sourced.zip")[0])
working_dir = os.getcwd()
os.chdir(pkg_resources.working_set.by_key["sourced-ml"].location)
with zipfile.PyZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
zf.write("sourced")
zf.writepy("sourced/ml", "sourced")
zf.writepy("sourced/engine", "sourced")
os.chdir(working_dir)
session.sparkContext.addPyFile(zip_path)
return session
def main():
os.chdir(os.path.dirname(__file__))
with zipfile.PyZipFile('mcafterlife.pyz', 'w') as fd:
for dirpath, dirnames, filenames in os.walk('src'):
fd.writepy(dirpath)
for name in os.listdir('cache'):
fd.write(os.path.join('cache', name))
def zip_app():
"""Remove the old zip archive and create the new one for APP_DIR."""
if not compile_app_directory():
sys.exit(0)
print "Removing old zip archive..."
print
try:
os.remove(app_zip_file())
except OSError:
pass
print "Creating new zip archive at:"
print app_zip_file()
zf = zipfile.PyZipFile(app_zip_file(), mode='w')
print '******', project_dir()
os.chdir(project_dir())
try:
zf.writepy(app_dir(), 'app')
root_len = len(project_dir())
for root, dirs, files in os.walk(app_dir()):
dir_path_from_root = root[root_len:]
for f in files:
if not (f.endswith('.py') or f.endswith('.pyc')):
fullpath = os.path.join(root, f)
archive_name = os.path.join(dir_path_from_root, f)
zf.write(fullpath, archive_name)
finally:
zf.close()
print "Done."
def create_zipped_application_packages(basedir):
basedir = basedir.replace('$LOGINSPECT_HOME', homing.LOGINSPECT_HOME)
if os.path.exists(basedir):
shutil.rmtree(basedir)
disk.prepare_path(basedir)
applications = []
apps_path = homing.home_join('storage/col/fileinspect_applications/')
for path in os.listdir(apps_path):
if os.path.isdir(os.path.join(apps_path, path)):
applications.append(path)
for dirname, subdirs, files in os.walk(apps_path):
for f in files:
if f.endswith(".pyc"):
os.unlink(os.path.join(dirname, f))
for app in applications:
outfilename = os.path.join(basedir, '%s.fi' % app)
try:
zf = zipfile.PyZipFile(outfilename, mode='w')
zf.writepy(os.path.join(apps_path, app))
finally:
zf.close()
return
def build_zip(self, include_src=False):
"""Build the zip file (with name `self.zip_name`) in the
current directory.
Parameters
----------
- include_src : `bool` (default: False)
If True, also includes all the ``*.py`` files in the ZIP file.
By default only the ``*.pyc`` files are included.
"""
cwd = os.getcwd()
d = os.path.dirname(self.out_dir)
os.chdir(d)
z = zipfile.PyZipFile(self.zip_name, 'w', zipfile.ZIP_DEFLATED)
if include_src:
l = glob.glob(os.path.join('tvtk_classes', '*.py'))
for x in l:
fname = os.path.basename(x)
z.write(x, 'tvtk_classes/%s' % fname)
z.writepy('tvtk_classes')
z.close()
if os.path.exists(cwd + "/" + self.zip_name):
os.unlink(cwd + "/" + self.zip_name)
shutil.move(self.zip_name, cwd)
os.chdir(cwd)
def create_zipfile():
tmp = "tmp" + ZIP
zf = zipfile.PyZipFile(tmp, mode='w')
zf.write(FILE)
zf.close()
f = open(tmp, 'rb')
data = f.read()
idx = data.index(p32(0x02014B50))
f.seek(0)
data = f.read(idx)
data += f.read(10) # magic & dummy & flags
data += "\x01\x00" # compress
f.read(2) # compress
data += f.read(2 + 2 + 4)
data += "\xff\xff\xff\xff"
f.read(4) # data_size
data += f.read() # remainings
with open(ZIP, 'wb') as f:
f.write(data)
f.close()
os.unlink(tmp)
def make_bigzip():
zip_path = "libbig.zip"
if os.path.exists(zip_path):
os.unlink(zip_path)
#zip_obj = zipfile.PyZipFile(zip_path,"w",compression=zipfile.ZIP_DEFLATED,optimize=-1)
zip_obj = zipfile.PyZipFile(zip_path,
"w",
compression=zipfile.ZIP_DEFLATED)
zip_obj.writepy(lib_path)
for (root, dirs, files) in os.walk(lib_path):
if os.path.basename(root) in (
"idlelib",
"test",
"tkinter",
"turtledemo",
):
del dirs[:]
continue
if "__init__.py" in files:
del dirs[:]
try:
#print(root)
zip_obj.writepy(root)
except (
EnvironmentError,
SyntaxError,
):
pass
zip_obj.close()
def generate_multi_set_examples(options, test_sets):
"""Generate examples for test sets.
Args:
options: Options containing information to generate examples.
test_sets: List of the name of test sets to generate examples.
"""
_prepare_dir(options)
multi_gen_state = MultiGenState()
options.multi_gen_state = multi_gen_state
zip_path = os.path.join(options.output_path, options.zip_to_output)
with zipfile.PyZipFile(zip_path, "w") as archive:
multi_gen_state.archive = archive
for test_name in test_sets:
# Some generation function can change the value of the options object.
# To keep the original options for each run, we use shallow copy.
new_options = copy.copy(options)
# Remove suffix and set test_name to run proper test generation function.
multi_gen_state.test_name = re.sub(
r"(_(|toco-flex|forward-compat))?$", "", test_name, count=1)
# Set label base path to write test data files with proper path.
multi_gen_state.label_base_path = os.path.join(
os.path.dirname(zip_path), test_name + ".zip")
generate_examples(new_options)
archive.writestr("manifest.txt", "".join(multi_gen_state.zip_manifest),
zipfile.ZIP_DEFLATED)
def _handle_zip(self, filename):
log(" zip? %s", filename)
if self._python_lib_zip is None:
pyth = 'Python-%s.%s.%s' % sys.version_info[:3]
zipname = '%s-Lib.zip' % pyth
if ST3:
zipname = cached_resource('Packages/Plugin Debugger/' +
zipname)
else:
zipname = os.path.join(plugin_debugger_dir(), zipname)
import zipfile
self._python_lib_zip = zipfile.PyZipFile(zipname)
if ST3:
m = re.match(r'^(.*python3.3.zip)/(.*)$', filename)
else:
m = re.match(r'^(.*python26.zip)/(.*)$', filename)
path, fn = m.groups()
fn = re.sub(r'^\./', '', fn)
fn = ('Python-%s.%s.%s/Lib/' % sys.version_info[:3]) + fn
return self._python_lib_zip.read(fn)
def make_zip(self, zipname, results):
written = 0
failed = []
with open(zipname, "wb") as final_zip:
with zipfile.PyZipFile(final_zip, "w", zipfile.ZIP_DEFLATED) as zf:
for path in results:
basename = os.path.basename(path)
try:
f = default_storage.open(path)
except Exception as e:
self.stderr.write("Cannot open %r: %s" % (path, e))
failed.append(path)
continue
zf.writestr(basename, f.read())
self.stdout.write(path)
written += 1
self.stdout.write("Written %i objects to %r" %
(written, final_zip.name))
if failed:
self.stderr.write("The following files failed to open:")
for path in failed:
self.stderr.write(" %s" % (path))
exit(8)
def testWritePythonDirectory(self):
os.mkdir(TESTFN2)
try:
fp = open(os.path.join(TESTFN2, "mod1.py"), "w")
fp.write("print 42\n")
fp.close()
fp = open(os.path.join(TESTFN2, "mod2.py"), "w")
fp.write("print 42 * 42\n")
fp.close()
fp = open(os.path.join(TESTFN2, "mod2.txt"), "w")
fp.write("bla bla bla\n")
fp.close()
zipfp = zipfile.PyZipFile(TemporaryFile(), "w")
zipfp.writepy(TESTFN2)
names = zipfp.namelist()
if not is_jython:
self.assert_('mod1.pyc' in names or 'mod1.pyo' in names)
self.assert_('mod2.pyc' in names or 'mod2.pyo' in names)
else:
self.assert_('mod1$py.class' in names)
self.assert_('mod2$py.class' in names)
self.assert_('mod2.txt' not in names)
finally:
shutil.rmtree(TESTFN2)
def export(self, mod, target, export_source=False):
raise NotImplemetedError, 'This feature does not work yet'
z = zipfile.PyZipFile(target, 'w')
config_filename = cbpos.res.installer('modconfig/%s.cfg' %
(mod.name, ))
config_file = open(config_filename, 'w')
config = ConfigParser.SafeConfigParser()
config.add_section('info')
config.set('info', 'base_name', unicode(mod.name))
config.set('info', 'name', unicode(mod.loader.name))
config.set('info', 'version', unicode(mod.loader.version))
config.write(config_file)
config_file.close()
z.write(config_filename, 'install.cfg')
if export_source:
os.path.walk(
mod.path,
lambda *args: self.__zipdirectory(*args, ignore_hidden=True),
z)
else:
z.writepy(mod.path, 'mod')
os.path.walk(mod.res_path(), self.__zipdirectory, z)
z.close()
def CreateZipFile(zipFilePath, inputFiles):
msgString = 'Nasty creating zipfile %s' % zipFilePath
print msgString
log.general.Log(msgString, log.LGINFO)
with tempfile.TemporaryFile() as tmp:
with CleanPyZipFile(tmp, 'w') as tmpzip:
for pair in inputFiles:
msgString = "Adding '%s' to zipfile " % pair[0]
print msgString
log.general.Log(msgString, log.LGINFO)
inputPathOS = blue.paths.ResolvePathForWriting(
pair[0]).rstrip('/')
if not os.path.exists(inputPathOS):
errString = "'%s' doesn't exist!" % inputPathOS
log.general.Log(errString, log.LGERR)
tmpzip.writepy(inputPathOS.encode('cp1252'), pair[1])
tmpzip.close()
with zipfile.PyZipFile(tmp, 'r') as tmpzip:
infos = tmpzip.infolist()
zipFilePathOS = blue.paths.ResolvePathForWriting(zipFilePath)
zf = file(zipFilePathOS, 'wb')
with zipfile.ZipFile(zf, 'w') as pyzip:
for i in infos:
data = tmpzip.read(i.filename)
if i.filename.endswith('.pyc') or i.filename.endswith(
'.pyo'):
i.filename = i.filename[:-1] + 'j'
data = nasty.JumbleString(data, True)
pyzip.writestr(i, data)
def unzip_with_progress(zip_path: Path,
output_dir: Path,
desc: Optional[str] = None) -> None:
try:
with zipfile.PyZipFile(str(zip_path)) as zf:
total = sum(zi.file_size for zi in zf.infolist())
progress = tqdm(desc=desc,
total=total,
unit="bytes",
unit_scale=True)
for zi in zf.infolist():
output_path = output_dir / zi.filename
if not zi.is_dir():
output_path.parent.mkdir(parents=True, exist_ok=True)
with output_path.open("wb") as output_fileobj:
with zf.open(zi, "r") as input_fileobj:
while True:
buf = input_fileobj.read(BLOCKSIZE)
if not buf:
break
progress.update(len(buf))
output_fileobj.write(buf)
except BaseException as e:
print("Exception is caught")
if output_dir.exists():
shutil.rmtree(output_dir)
raise e
def export(v):
if not os.path.exists(TMPDIR):
try:
os.mkdir(TMPDIR)
except OSError:
pass
shutil.copytree(PATHSDIR, TPATHSDIR)
shutil.copy2(PATHSX, TPATHSX)
vstr = repr_version(v)
with open(TVERFILE, 'w') as f:
f.write(vstr)
dec = v.release_level == 'DEC'
rev = v.release_level == 'REV'
catdir = DECDIR if dec else DEVDIR
if rev:
catdir = REVDIR
tdyndir = os.path.join(TPATHSDIR, 'dyn')
tscptdir = os.path.join(TPATHSDIR, 'scripts')
tpydir = os.path.join(TPATHSDIR, 'py')
tpyzip = os.path.join(TPATHSDIR, 'py.zip')
import zipfile
zf = zipfile.PyZipFile(tpyzip, 'w', zipfile.ZIP_DEFLATED)
zf.writepy(tpydir)
shutil.rmtree(tdyndir)
shutil.rmtree(tscptdir)
shutil.rmtree(tpydir)
arname = os.path.join(catdir, vstr)
shutil.make_archive(arname, 'zip', TMPDIR)
if rev:
shutil.rmtree(TMPDIR)
def zip_package(package_path, zip_path):
path_offset = len(os.path.dirname(package_path)) + 1
with zipfile.PyZipFile(zip_path, 'w') as writer:
for root, _, files in os.walk(package_path):
for file in files:
full_path = os.path.join(root, file)
archive_path = full_path[path_offset:]
writer.write(full_path, archive_path)
def setUp(self):
self.L = Registry()
# Create module
import zipfile
z = zipfile.PyZipFile("plugins.zip", 'w')
z.writepy("../plugins", "")
z.close()
def _python(self):
Build._python(self)
trim(join(self.pythonext_python_dir, "share"))
self._zip_python(zipfile.PyZipFile(join(self.pythonext_python_dir, "lib", "python%s.zip" % py_ver), "w"))
os.mkdir(self.pythonext_python_lib_dir)
python_lib_path = join(py_library_path, "python%s" % py_ver_dotted)
for dirpath in ("lib-dynload", "config", "plat-linux2"):
shutil.copytree(join(python_lib_path, dirpath), join(self.pythonext_python_lib_dir, dirpath))
def zipdir(directory, no_pyc=False):
log = logging.getLogger("zipdir")
if not os.path.isdir(directory):
raise OSError(20, "Not a directory", directory)
msg = "Zipping directory '%s'"
if no_pyc:
msg += " (without pre-compilation)"
log.info(msg, directory)
filename = os.path.realpath(directory + ".zip")
# Open the file in read an update mode
if os.path.exists(filename):
zip_file = open(filename, "r+b")
else:
# If the file does not exist, we need to create it.
# "append mode" ensures that, in case of two processes trying to
# create the file, they do not truncate each other file
zip_file = open(filename, "ab")
locker.lock(zip_file)
try:
if zipfile.is_zipfile(filename):
infolist = zipfile.ZipFile(filename).infolist()
else:
infolist = []
(added, modified, untouched,
removed) = _zipChanges(directory, infolist)
if added or modified or removed:
temp_buf = StringIO()
z = zipfile.PyZipFile(temp_buf, "w", zipfile.ZIP_DEFLATED)
for f in added + modified + untouched:
src = os.path.join(directory, f)
checkEncoding(open(src, 'rb'))
if no_pyc:
log.debug("adding '%s'", f)
z.write(src, f)
else:
# Remove the .pyc file to always force a re-compilation
if os.path.exists(src + 'c'):
log.debug("removing old .pyc for '%s'", f)
os.remove(src + 'c')
log.debug("adding '%s'", f)
z.writepy(src, os.path.dirname(f))
z.close()
zip_file.seek(0)
zip_file.write(temp_buf.getvalue())
zip_file.truncate()
log.info("File '%s' closed", filename)
else:
log.info("Nothing to do on '%s'", filename)
except UnicodeDecodeError as x:
log.error("Wrong encoding in file '%s':", src)
log.error(" %s", x)
log.error("Probably you forgot the line '# -*- coding: utf-8 -*-'")
sys.exit(1)
finally:
locker.unlock(zip_file)
zip_file.close()
def replace_code(self,code):
code = self._obj2code(code)
zf = zipfile.PyZipFile(self.path,"a")
try:
bcode = imp.get_magic() + struct.pack("<i",time.time())
bcode += marshal.dumps(compile(code,self.INITMOD+".py","exec"))
zf.writestr(self.INITMOD+".pyc",bcode)
finally:
zf.close()
def __parseXmlZipFile(self):
"""
Uncompress an odf file and parse the "content.xml" file.
"""
try:
self.__zipFile = zipfile.PyZipFile(self.Document.getData(),'r')
except zipfile.BadZipfile, e:
#Log.error("File is not a zip file")
return None, None
def make_mediumzip():
zip_path = "libmedium.zip"
if os.path.exists(zip_path):
os.unlink(zip_path)
zip_obj = zipfile.PyZipFile(zip_path,
"w",
compression=zipfile.ZIP_DEFLATED)
zip_obj.writepy(lib_path)
zip_obj.close()
def prepare_zip_file():
clean_pyc_files()
tmpd = tempfile.mkdtemp()
egg = os.path.join(tmpd, "testpkg1.egg")
eggzip = zipfile.PyZipFile(egg, "w", zipfile.ZIP_DEFLATED)
eggzip.writepy(os.path.join(samples_dir, "testmod1.py"))
eggzip.writepy(os.path.join(samples_dir, "testpkg1"))
eggzip.close()
return egg
def main():
global ZF
comments = []
ZF = zipfile.PyZipFile(get_zip_file())
next_num = extract_num()
while next_num:
comments.append(ZF.getinfo('{}.txt'.format(next_num)).comment)
print(next_num)
next_num = extract_num('{}.txt'.format(next_num))
print(''.join(comment.decode('utf8') for comment in comments))
def check_jar(test, jar, files):
"""
Check the contents of the pom. Make sure the groupId, artifactId, and version are properly set.
:param files: List of entries expected in the jar
"""
jar_file = zipfile.PyZipFile(jar, 'r')
entries = jar_file.namelist()
for expected in files:
test.assertTrue(expected in entries)
jar_file.close()
def ziplib(fname):
#libpath = os.path.dirname(fname) # this should point to your packages directory
zf = zipfile.PyZipFile(zippath, mode='w')
try:
#zf.debug = 0 # making it verbose, good for debugging
zf.writepy(fname)
#zf.writepy(libpath)
return zippath # return path to generated zip archive
finally:
zf.close()
