def _diff_dotzip_file(self,source,target):
s_zf = self._open_and_check_zipfile(source)
if s_zf is None:
self._diff_binary_file(source,target)
else:
t_zf = self._open_and_check_zipfile(target)
if t_zf is None:
s_zf.close()
self._diff_binary_file(source,target)
else:
try:
self._write_command(PF_REC_ZIP)
with _tempdir() as workdir:
# Write commands to transform source metadata file
# into target metadata file.
s_meta = os.path.join(workdir,"s_meta")
with open(s_meta,"wb") as f:
_write_zipfile_metadata(f,s_zf)
t_meta = os.path.join(workdir,"t_meta")
with open(t_meta,"wb") as f:
_write_zipfile_metadata(f,t_zf)
self._diff_binary_file(s_meta,t_meta)
self._write_command(END)
# Write commands to transform source contents
# directory into target contents directory.
s_workdir = os.path.join(workdir,"source")
t_workdir = os.path.join(workdir,"target")
extract_zipfile(source,s_workdir)
extract_zipfile(target,t_workdir)
self._diff(s_workdir,t_workdir)
self._write_command(END)
finally:
t_zf.close()
s_zf.close()
def _diff_dotzip_file(self,source,target):
s_zf = self._open_and_check_zipfile(source)
if s_zf is None:
self._diff_binary_file(source,target)
else:
t_zf = self._open_and_check_zipfile(target)
if t_zf is None:
s_zf.close()
self._diff_binary_file(source,target)
else:
try:
self._write_command(PF_REC_ZIP)
with _tempdir() as workdir:
# Write commands to transform source metadata file
# into target metadata file.
s_meta = os.path.join(workdir,"s_meta")
with open(s_meta,"wb") as f:
_write_zipfile_metadata(f,s_zf)
t_meta = os.path.join(workdir,"t_meta")
with open(t_meta,"wb") as f:
_write_zipfile_metadata(f,t_zf)
self._diff_binary_file(s_meta,t_meta)
self._write_command(END)
# Write commands to transform source contents
# directory into target contents directory.
s_workdir = os.path.join(workdir,"source")
t_workdir = os.path.join(workdir,"target")
extract_zipfile(source,s_workdir)
extract_zipfile(target,t_workdir)
self._diff(s_workdir,t_workdir)
self._write_command(END)
finally:
t_zf.close()
s_zf.close()
def _do_PF_REC_ZIP(self):
"""Execute the PF_REC_ZIP command.
This patches the current target by treating it as a zipfile and
recursing into it. It extracts the source file to a temp directory,
then reads commands and applies them to that directory.
This command expects two END-terminated blocks of sub-commands. The
first block patches the zipfile metadata, and the second patches the
actual contents of the zipfile.
"""
self._check_begin_patch()
if not self.dry_run:
workdir = os.path.join(self._workdir,str(len(self._context_stack)))
os.mkdir(workdir)
t_temp = os.path.join(workdir,"contents")
m_temp = os.path.join(workdir,"meta")
z_temp = os.path.join(workdir,"result.zip")
cur_state = self._blank_state()
zfmeta = [None] # stupid lack of mutable closure variables...
# First we process a set of commands to generate the zipfile metadata.
def end_metadata():
if not self.dry_run:
zfmeta[0] = _read_zipfile_metadata(m_temp)
self.target = t_temp
# Then we process a set of commands to patch the actual contents.
def end_contents():
self._restore_state(cur_state)
if not self.dry_run:
create_zipfile(t_temp,z_temp,members=zfmeta[0].infolist())
with open(z_temp,"rb") as f:
data = f.read(1024*16)
while data:
self.outfile.write(data)
data = f.read(1024*16)
zfmeta[0].close()
really_rmtree(workdir)
self._context_stack.append(end_contents)
self._context_stack.append(end_metadata)
if not self.dry_run:
# Begin by writing the current zipfile metadata to a temp file.
# This will be patched, then end_metadata() will be called.
with open(m_temp,"wb") as f:
zf = zipfile.ZipFile(self.target)
try:
_write_zipfile_metadata(f,zf)
finally:
zf.close()
extract_zipfile(self.target,t_temp)
self.root_dir = workdir
self.target = m_temp
def _do_PF_REC_ZIP(self):
"""Execute the PF_REC_ZIP command.
This patches the current target by treating it as a zipfile and
recursing into it. It extracts the source file to a temp directory,
then reads commands and applies them to that directory.
This command expects two END-terminated blocks of sub-commands. The
first block patches the zipfile metadata, and the second patches the
actual contents of the zipfile.
"""
self._check_begin_patch()
if not self.dry_run:
workdir = os.path.join(self._workdir,str(len(self._context_stack)))
os.mkdir(workdir)
t_temp = os.path.join(workdir,"contents")
m_temp = os.path.join(workdir,"meta")
z_temp = os.path.join(workdir,"result.zip")
cur_state = self._blank_state()
zfmeta = [None] # stupid lack of mutable closure variables...
# First we process a set of commands to generate the zipfile metadata.
def end_metadata():
if not self.dry_run:
zfmeta[0] = _read_zipfile_metadata(m_temp)
self.target = t_temp
# Then we process a set of commands to patch the actual contents.
def end_contents():
self._restore_state(cur_state)
if not self.dry_run:
create_zipfile(t_temp,z_temp,members=zfmeta[0].infolist())
with open(z_temp,"rb") as f:
data = f.read(1024*16)
while data:
self.outfile.write(data)
data = f.read(1024*16)
zfmeta[0].close()
really_rmtree(workdir)
self._context_stack.append(end_contents)
self._context_stack.append(end_metadata)
if not self.dry_run:
# Begin by writing the current zipfile metadata to a temp file.
# This will be patched, then end_metadata() will be called.
with open(m_temp,"wb") as f:
zf = zipfile.ZipFile(self.target)
try:
_write_zipfile_metadata(f,zf)
finally:
zf.close()
extract_zipfile(self.target,t_temp)
self.root_dir = workdir
self.target = m_temp
def _run_eskytester(self, options):
"""Build and run the eskytester app using the given distutils options.
The "eskytester" application can be found next to this file, and the
sequence of tests performed range across "script1.py" to "script3.py".
"""
olddir = os.path.abspath(os.curdir)
# tdir = os.path.join(os.path.dirname(__file__),"DIST")
# if os.path.exists(tdir):
# really_rmtree(tdir)
# os.mkdir(tdir)
tdir = tempfile.mkdtemp()
server = None
script2 = None
try:
options.setdefault("build", {})["build_base"] = os.path.join(tdir, "build")
options.setdefault("bdist", {})["dist_dir"] = os.path.join(tdir, "dist")
# Set some callbacks to test that they work correctly
options.setdefault("bdist_esky", {}).setdefault(
"pre_freeze_callback", "esky.tests.test_esky.assert_freezedir_exists"
)
options.setdefault("bdist_esky", {}).setdefault("pre_zip_callback", assert_freezedir_exists)
platform = get_platform()
deploydir = "deploy.%s" % (platform,)
esky_root = dirname(dirname(dirname(__file__)))
os.chdir(tdir)
shutil.copytree(os.path.join(esky_root, "esky", "tests", "eskytester"), "eskytester")
dir_util._path_created.clear()
# Build three increasing versions of the test package.
# Version 0.2 will include a bundled MSVCRT on win32.
# Version 0.3 will be distributed as a patch.
metadata = dict(
name="eskytester",
packages=["eskytester"],
author="rfk",
description="the esky test package",
data_files=[("data", ["eskytester/datafile.txt"])],
package_data={"eskytester": ["pkgdata.txt"]},
)
options2 = options.copy()
options2["bdist_esky"] = options["bdist_esky"].copy()
options2["bdist_esky"]["bundle_msvcrt"] = True
script1 = "eskytester/script1.py"
script2 = Executable([None, open("eskytester/script2.py")], name="script2")
script3 = "eskytester/script3.py"
dist_setup(version="0.1", scripts=[script1], options=options, script_args=["bdist_esky"], **metadata)
dist_setup(
version="0.2", scripts=[script1, script2], options=options2, script_args=["bdist_esky"], **metadata
)
dist_setup(
version="0.3", scripts=[script2, script3], options=options, script_args=["bdist_esky_patch"], **metadata
)
os.unlink(os.path.join(tdir, "dist", "eskytester-0.3.%s.zip" % (platform,)))
# Check that the patches apply cleanly
uzdir = os.path.join(tdir, "unzip")
deep_extract_zipfile(os.path.join(tdir, "dist", "eskytester-0.1.%s.zip" % (platform,)), uzdir)
with open(os.path.join(tdir, "dist", "eskytester-0.3.%s.from-0.1.patch" % (platform,)), "rb") as f:
esky.patch.apply_patch(uzdir, f)
shutil.rmtree(uzdir)
deep_extract_zipfile(os.path.join(tdir, "dist", "eskytester-0.2.%s.zip" % (platform,)), uzdir)
with open(os.path.join(tdir, "dist", "eskytester-0.3.%s.from-0.2.patch" % (platform,)), "rb") as f:
esky.patch.apply_patch(uzdir, f)
shutil.rmtree(uzdir)
# Serve the updates at http://localhost:8000/dist/
print "running local update server"
server = HTTPServer(("localhost", 8000), SimpleHTTPRequestHandler)
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
# Set up the deployed esky environment for the initial version
zfname = os.path.join(tdir, "dist", "eskytester-0.1.%s.zip" % (platform,))
os.mkdir(deploydir)
extract_zipfile(zfname, deploydir)
# Run the scripts in order.
if options["bdist_esky"]["freezer_module"] == "py2app":
appdir = os.path.join(deploydir, os.listdir(deploydir)[0])
cmd1 = os.path.join(appdir, "Contents", "MacOS", "script1")
cmd2 = os.path.join(appdir, "Contents", "MacOS", "script2")
cmd3 = os.path.join(appdir, "Contents", "MacOS", "script3")
else:
appdir = deploydir
if sys.platform == "win32":
cmd1 = os.path.join(deploydir, "script1.exe")
cmd2 = os.path.join(deploydir, "script2.exe")
cmd3 = os.path.join(deploydir, "script3.exe")
else:
cmd1 = os.path.join(deploydir, "script1")
cmd2 = os.path.join(deploydir, "script2")
cmd3 = os.path.join(deploydir, "script3")
print "spawning eskytester script1", options["bdist_esky"]["freezer_module"]
os.unlink(os.path.join(tdir, "dist", "eskytester-0.1.%s.zip" % (platform,)))
p = subprocess.Popen(cmd1)
assert p.wait() == 0
os.unlink(os.path.join(appdir, "tests-completed"))
print "spawning eskytester script2"
os.unlink(os.path.join(tdir, "dist", "eskytester-0.2.%s.zip" % (platform,)))
p = subprocess.Popen(cmd2)
assert p.wait() == 0
os.unlink(os.path.join(appdir, "tests-completed"))
print "spawning eskytester script3"
p = subprocess.Popen(cmd3)
assert p.wait() == 0
os.unlink(os.path.join(appdir, "tests-completed"))
finally:
if script2:
script2.script[1].close()
os.chdir(olddir)
if sys.platform == "win32":
# wait for the cleanup-at-exit pocess to finish
time.sleep(4)
really_rmtree(tdir)
if server:
server.shutdown()
def _run_eskytester(self, options):
"""Build and run the eskytester app using the given distutils options.
The "eskytester" application can be found next to this file, and the
sequence of tests performed range across "script1.py" to "script3.py".
"""
olddir = os.path.abspath(os.curdir)
# tdir = os.path.join(os.path.dirname(__file__),"DIST")
# if os.path.exists(tdir):
# really_rmtree(tdir)
# os.mkdir(tdir)
tdir = tempfile.mkdtemp()
server = None
script2 = None
try:
options.setdefault("build",
{})["build_base"] = os.path.join(tdir, "build")
options.setdefault("bdist",
{})["dist_dir"] = os.path.join(tdir, "dist")
# Set some callbacks to test that they work correctly
options.setdefault("bdist_esky", {}).setdefault(
"pre_freeze_callback",
"esky.tests.test_esky.assert_freezedir_exists")
options.setdefault("bdist_esky",
{}).setdefault("pre_zip_callback",
assert_freezedir_exists)
options["bdist_esky"].setdefault("excludes",
[]).extend(["Tkinter", "tkinter"])
platform = get_platform()
deploydir = "deploy.%s" % (platform, )
esky_root = dirname(dirname(dirname(__file__)))
os.chdir(tdir)
shutil.copytree(
os.path.join(esky_root, "esky", "tests", "eskytester"),
"eskytester")
dir_util._path_created.clear()
# Build three increasing versions of the test package.
# Version 0.2 will include a bundled MSVCRT on win32.
# Version 0.3 will be distributed as a patch.
metadata = dict(
name="eskytester",
packages=["eskytester"],
author="rfk",
description="the esky test package",
data_files=[("data", ["eskytester/datafile.txt"])],
package_data={"eskytester": ["pkgdata.txt"]},
)
options2 = options.copy()
options2["bdist_esky"] = options["bdist_esky"].copy()
options2["bdist_esky"]["bundle_msvcrt"] = True
script1 = "eskytester/script1.py"
script2 = Executable([None, open("eskytester/script2.py")],
name="script2")
script3 = "eskytester/script3.py"
dist_setup(version="0.1",
scripts=[script1],
options=options,
script_args=["bdist_esky"],
**metadata)
dist_setup(version="0.2",
scripts=[script1, script2],
options=options2,
script_args=["bdist_esky"],
**metadata)
dist_setup(version="0.3",
scripts=[script2, script3],
options=options,
script_args=["bdist_esky_patch"],
**metadata)
os.unlink(
os.path.join(tdir, "dist",
"eskytester-0.3.%s.zip" % (platform, )))
# Check that the patches apply cleanly
uzdir = os.path.join(tdir, "unzip")
deep_extract_zipfile(
os.path.join(tdir, "dist",
"eskytester-0.1.%s.zip" % (platform, )), uzdir)
with open(
os.path.join(
tdir, "dist",
"eskytester-0.3.%s.from-0.1.patch" % (platform, )),
"rb") as f:
esky.patch.apply_patch(uzdir, f)
really_rmtree(uzdir)
deep_extract_zipfile(
os.path.join(tdir, "dist",
"eskytester-0.2.%s.zip" % (platform, )), uzdir)
with open(
os.path.join(
tdir, "dist",
"eskytester-0.3.%s.from-0.2.patch" % (platform, )),
"rb") as f:
esky.patch.apply_patch(uzdir, f)
really_rmtree(uzdir)
# Serve the updates at LOCAL_HTTP_PORT set in esky.util
print "running local update server"
try:
server = HTTPServer(("localhost", LOCAL_HTTP_PORT),
SimpleHTTPRequestHandler)
except Exception:
# in travis ci we start our own server
pass
else:
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
# Set up the deployed esky environment for the initial version
zfname = os.path.join(tdir, "dist",
"eskytester-0.1.%s.zip" % (platform, ))
os.mkdir(deploydir)
extract_zipfile(zfname, deploydir)
# Run the scripts in order.
if options["bdist_esky"]["freezer_module"] == "py2app":
appdir = os.path.join(deploydir, os.listdir(deploydir)[0])
cmd1 = os.path.join(appdir, "Contents", "MacOS", "script1")
cmd2 = os.path.join(appdir, "Contents", "MacOS", "script2")
cmd3 = os.path.join(appdir, "Contents", "MacOS", "script3")
else:
appdir = deploydir
if sys.platform == "win32":
cmd1 = os.path.join(deploydir, "script1.exe")
cmd2 = os.path.join(deploydir, "script2.exe")
cmd3 = os.path.join(deploydir, "script3.exe")
else:
cmd1 = os.path.join(deploydir, "script1")
cmd2 = os.path.join(deploydir, "script2")
cmd3 = os.path.join(deploydir, "script3")
print "spawning eskytester script1", options["bdist_esky"][
"freezer_module"]
os.unlink(
os.path.join(tdir, "dist",
"eskytester-0.1.%s.zip" % (platform, )))
p = subprocess.Popen(cmd1)
assert p.wait() == 0
os.unlink(os.path.join(appdir, "tests-completed"))
print "spawning eskytester script2"
os.unlink(
os.path.join(tdir, "dist",
"eskytester-0.2.%s.zip" % (platform, )))
p = subprocess.Popen(cmd2)
assert p.wait() == 0
os.unlink(os.path.join(appdir, "tests-completed"))
print "spawning eskytester script3"
p = subprocess.Popen(cmd3)
assert p.wait() == 0
os.unlink(os.path.join(appdir, "tests-completed"))
finally:
if script2:
script2.script[1].close()
os.chdir(olddir)
if sys.platform == "win32":
# wait for the cleanup-at-exit pocess to finish
time.sleep(4)
really_rmtree(tdir)
if server:
server.shutdown()
def main(args):
"""Command-line diffing and patching for esky."""
parser = optparse.OptionParser()
parser.add_option("-z","--zipped",action="store_true",dest="zipped",
help="work with zipped source/target dirs")
parser.add_option("-Z","--deep-zipped",action="store_true",
dest="deep_zipped",
help="work with deep zipped source/target dirs")
parser.add_option("","--diff-window",dest="diff_window",metavar="N",
help="set the window size for diffing files")
parser.add_option("","--dry-run",dest="dry_run",action="store_true",
help="print commands instead of executing them")
(opts,args) = parser.parse_args(args)
if opts.deep_zipped:
opts.zipped = True
if opts.zipped:
workdir = tempfile.mkdtemp()
if opts.diff_window:
scale = 1
if opts.diff_window[-1].lower() == "k":
scale = 1024
opts.diff_window = opts.diff_window[:-1]
elif opts.diff_window[-1].lower() == "m":
scale = 1024 * 1024
opts.diff_window = opts.diff_window[:-1]
elif opts.diff_window[-1].lower() == "g":
scale = 1024 * 1024 * 1024
opts.diff_window = opts.diff_window[:-1]
opts.diff_window = int(float(opts.diff_window)*scale)
stream = None
try:
cmd = args[0]
if cmd == "diff":
# Generate a diff between two files/directories.
# If --zipped is specified, the source and/or target is unzipped
# to a temporary directory before processing.
source = args[1]
target = args[2]
if len(args) > 3:
stream = open(args[3],"wb")
else:
stream = sys.stdout
if opts.zipped:
if os.path.isfile(source):
source_zip = source
source = os.path.join(workdir,"source")
if opts.deep_zipped:
deep_extract_zipfile(source_zip,source)
else:
extract_zipfile(source_zip,source)
if os.path.isfile(target):
target_zip = target
target = os.path.join(workdir,"target")
if opts.deep_zipped:
deep_extract_zipfile(target_zip,target)
else:
extract_zipfile(target_zip,target)
write_patch(source,target,stream,diff_window_size=opts.diff_window)
elif cmd == "patch":
# Patch a file or directory.
# If --zipped is specified, the target is unzipped to a temporary
# directory before processing, then overwritten with a zipfile
# containing the new directory contents.
target = args[1]
if len(args) > 2:
stream = open(args[2],"rb")
else:
stream = sys.stdin
target_zip = None
if opts.zipped:
if os.path.isfile(target):
target_zip = target
target = os.path.join(workdir,"target")
if opts.deep_zipped:
deep_extract_zipfile(target_zip,target)
else:
extract_zipfile(target_zip,target)
apply_patch(target,stream,dry_run=opts.dry_run)
if opts.zipped and target_zip is not None:
target_dir = os.path.dirname(target_zip)
(fd,target_temp) = tempfile.mkstemp(dir=target_dir)
os.close(fd)
if opts.deep_zipped:
prefix = zipfile_common_prefix_dir(target_zip)
def name_filter(nm):
return prefix + nm
create_zipfile(target,target_temp,name_filter)
else:
create_zipfile(target,target_temp)
if sys.platform == "win32":
os.unlink(target_zip)
time.sleep(0.01)
really_rename(target_temp,target_zip)
else:
raise ValueError("invalid command: " + cmd)
finally:
if stream is not None:
if stream not in (sys.stdin,sys.stdout,):
stream.close()
if opts.zipped:
really_rmtree(workdir)
def main(args):
"""Command-line diffing and patching for esky."""
parser = optparse.OptionParser()
parser.add_option("-z","--zipped",action="store_true",dest="zipped",
help="work with zipped source/target dirs")
parser.add_option("-Z","--deep-zipped",action="store_true",
dest="deep_zipped",
help="work with deep zipped source/target dirs")
parser.add_option("","--diff-window",dest="diff_window",metavar="N",
help="set the window size for diffing files")
parser.add_option("","--dry-run",dest="dry_run",action="store_true",
help="print commands instead of executing them")
(opts,args) = parser.parse_args(args)
if opts.deep_zipped:
opts.zipped = True
if opts.zipped:
workdir = tempfile.mkdtemp()
if opts.diff_window:
scale = 1
if opts.diff_window[-1].lower() == "k":
scale = 1024
opts.diff_window = opts.diff_window[:-1]
elif opts.diff_window[-1].lower() == "m":
scale = 1024 * 1024
opts.diff_window = opts.diff_window[:-1]
elif opts.diff_window[-1].lower() == "g":
scale = 1024 * 1024 * 1024
opts.diff_window = opts.diff_window[:-1]
opts.diff_window = int(float(opts.diff_window)*scale)
stream = None
try:
cmd = args[0]
if cmd == "diff":
# Generate a diff between two files/directories.
# If --zipped is specified, the source and/or target is unzipped
# to a temporary directory before processing.
source = args[1]
target = args[2]
if len(args) > 3:
stream = open(args[3],"wb")
else:
stream = sys.stdout
if opts.zipped:
if os.path.isfile(source):
source_zip = source
source = os.path.join(workdir,"source")
if opts.deep_zipped:
deep_extract_zipfile(source_zip,source)
else:
extract_zipfile(source_zip,source)
if os.path.isfile(target):
target_zip = target
target = os.path.join(workdir,"target")
if opts.deep_zipped:
deep_extract_zipfile(target_zip,target)
else:
extract_zipfile(target_zip,target)
write_patch(source,target,stream,diff_window_size=opts.diff_window)
elif cmd == "patch":
# Patch a file or directory.
# If --zipped is specified, the target is unzipped to a temporary
# directory before processing, then overwritten with a zipfile
# containing the new directory contents.
target = args[1]
if len(args) > 2:
stream = open(args[2],"rb")
else:
stream = sys.stdin
target_zip = None
if opts.zipped:
if os.path.isfile(target):
target_zip = target
target = os.path.join(workdir,"target")
if opts.deep_zipped:
deep_extract_zipfile(target_zip,target)
else:
extract_zipfile(target_zip,target)
apply_patch(target,stream,dry_run=opts.dry_run)
if opts.zipped and target_zip is not None:
target_dir = os.path.dirname(target_zip)
(fd,target_temp) = tempfile.mkstemp(dir=target_dir)
os.close(fd)
if opts.deep_zipped:
prefix = zipfile_common_prefix_dir(target_zip)
def name_filter(nm):
return prefix + nm
create_zipfile(target,target_temp,name_filter)
else:
create_zipfile(target,target_temp)
if sys.platform == "win32":
os.unlink(target_zip)
time.sleep(0.01)
really_rename(target_temp,target_zip)
else:
raise ValueError("invalid command: " + cmd)
finally:
if stream is not None:
if stream not in (sys.stdin,sys.stdout,):
stream.close()
if opts.zipped:
really_rmtree(workdir)