def __del__(self):
if self.infile:
self.infile.close()
if self.outfile:
self.outfile.close()
if self._workdir and shutil:
really_rmtree(self._workdir)
def __del__(self):
if self.infile:
self.infile.close()
if self.outfile:
self.outfile.close()
if self._workdir and shutil:
really_rmtree(self._workdir)
def _extract(self, filename, dest):
dest = os.path.join(self.workdir, dest)
if os.path.exists(dest):
really_rmtree(dest)
f = tarfile.open(os.path.join(self.tfdir, filename), "r:gz")
try:
f.extractall(dest)
finally:
f.close()
return dest
def _extract(self, filename, dest):
dest = os.path.join(self.workdir, dest)
if os.path.exists(dest):
really_rmtree(dest)
f = tarfile.open(os.path.join(self.tfdir, filename), "r:gz")
try:
f.extractall(dest)
finally:
f.close()
return dest
def _do_REMOVE(self):
"""Execute the REMOVE command.
This forcibly removes the file or directory at the current target path.
"""
self._check_end_patch()
if not self.dry_run:
if os.path.isdir(self.target):
really_rmtree(self.target)
elif os.path.exists(self.target):
os.unlink(self.target)
def end_contents():
self._restore_state(cur_state)
if not self.dry_run:
create_zipfile(t_temp,z_temp,members=zfmeta[0].infolist())
with open(z_temp,"rb") as f:
data = f.read(1024*16)
while data:
self.outfile.write(data)
data = f.read(1024*16)
zfmeta[0].close()
really_rmtree(workdir)
def _do_REMOVE(self):
"""Execute the REMOVE command.
This forcibly removes the file or directory at the current target path.
"""
self._check_end_patch()
if not self.dry_run:
if os.path.isdir(self.target):
really_rmtree(self.target)
elif os.path.exists(self.target):
os.unlink(self.target)
def cleanup(self,app):
# TODO: hang onto the latest downloaded version
dldir = self._workdir(app,"downloads")
for nm in os.listdir(dldir):
os.unlink(os.path.join(dldir,nm))
updir = self._workdir(app,"unpack")
for nm in os.listdir(updir):
really_rmtree(os.path.join(updir,nm))
rddir = self._workdir(app,"ready")
for nm in os.listdir(rddir):
really_rmtree(os.path.join(rddir,nm))
def end_contents():
self._restore_state(cur_state)
if not self.dry_run:
create_zipfile(t_temp,z_temp,members=zfmeta[0].infolist())
with open(z_temp,"rb") as f:
data = f.read(1024*16)
while data:
self.outfile.write(data)
data = f.read(1024*16)
zfmeta[0].close()
really_rmtree(workdir)
def _run_create_zipfile(self):
"""Zip up the final distribution."""
print "zipping up the esky"
fullname = self.distribution.get_fullname()
platform = get_platform()
zfname = os.path.join(self.dist_dir,"%s.%s.zip"%(fullname,platform,))
if hasattr(self.freezer_module,"zipit"):
self.freezer_module.zipit(self,self.bootstrap_dir,zfname)
else:
create_zipfile(self.bootstrap_dir,zfname,compress=True)
really_rmtree(self.bootstrap_dir)
def _run_initialise_dirs(self):
"""Create the dirs into which to freeze the app."""
fullname = self.distribution.get_fullname()
platform = get_platform()
self.bootstrap_dir = os.path.join(self.dist_dir, "%s.%s" % (fullname, platform))
if self.enable_appdata_dir:
self.freeze_dir = os.path.join(self.bootstrap_dir, ESKY_APPDATA_DIR, "%s.%s" % (fullname, platform))
else:
self.freeze_dir = os.path.join(self.bootstrap_dir, "%s.%s" % (fullname, platform))
if os.path.exists(self.bootstrap_dir):
really_rmtree(self.bootstrap_dir)
os.makedirs(self.freeze_dir)
def _do_MAKEDIR(self):
"""Execute the MAKEDIR command.
This makes a directory at the current target path. It automatically
removes any existing entry at that path, as well as creating any
intermediate directories.
"""
self._check_end_patch()
if not self.dry_run:
if os.path.isdir(self.target):
really_rmtree(self.target)
elif os.path.exists(self.target):
os.unlink(self.target)
os.makedirs(self.target)
def _do_MAKEDIR(self):
"""Execute the MAKEDIR command.
This makes a directory at the current target path. It automatically
removes any existing entry at that path, as well as creating any
intermediate directories.
"""
self._check_end_patch()
if not self.dry_run:
if os.path.isdir(self.target):
really_rmtree(self.target)
elif os.path.exists(self.target):
os.unlink(self.target)
os.makedirs(self.target)
def _extract(self, source, target):
'''extracts two tar gz files into a source and target dir which are returned'''
if os.path.exists(self.src_dir):
really_rmtree(self.src_dir)
if os.path.exists(self.tgt_dir):
really_rmtree(self.tgt_dir)
f_source = tarfile.open(os.path.join(self.tfdir, source), "r:gz")
f_target = tarfile.open(os.path.join(self.tfdir, target), "r:gz")
try:
f_source.extractall(self.src_dir)
f_target.extractall(self.tgt_dir)
finally:
f_source.close()
f_target.close()
return self.src_dir, self.tgt_dir
def _extract(self,source, target):
'''extracts two tar gz files into a source and target dir which are returned'''
if os.path.exists(self.src_dir):
really_rmtree(self.src_dir)
if os.path.exists(self.tgt_dir):
really_rmtree(self.tgt_dir)
f_source = tarfile.open(os.path.join(self.tfdir,source),"r:gz")
f_target = tarfile.open(os.path.join(self.tfdir,target),"r:gz")
try:
f_source.extractall(self.src_dir)
f_target.extractall(self.tgt_dir)
finally:
f_source.close()
f_target.close()
return self.src_dir, self.tgt_dir
def _run_initialise_dirs(self):
"""Create the dirs into which to freeze the app."""
fullname = self.distribution.get_fullname()
platform = get_platform()
self.bootstrap_dir = os.path.join(self.dist_dir,
"%s.%s"%(fullname,platform,))
if self.enable_appdata_dir:
self.freeze_dir = os.path.join(self.bootstrap_dir,ESKY_APPDATA_DIR,
"%s.%s"%(fullname,platform,))
else:
self.freeze_dir = os.path.join(self.bootstrap_dir,
"%s.%s"%(fullname,platform,))
if os.path.exists(self.bootstrap_dir):
really_rmtree(self.bootstrap_dir)
os.makedirs(self.freeze_dir)
def _run_create_zipfile(self):
"""Zip up the final distribution."""
print "zipping up the esky"
fullname = self.distribution.get_fullname()
platform = get_platform()
zfname = os.path.join(self.dist_dir,
"%s.%s.zip" % (fullname, platform, ))
if hasattr(self.freezer_module, "zipit"):
self.freezer_module.zipit(self, self.bootstrap_dir, zfname)
else:
create_zipfile(self.bootstrap_dir, zfname, compress=True)
# Only remove bootstrap dir if option is passed
if (self.rm_freeze_dir_after_zipping
and self.rm_freeze_dir_after_zipping != 'False'):
really_rmtree(self.bootstrap_dir)
def _run_create_zipfile(self):
"""Zip up the final distribution."""
print "zipping up the esky"
fullname = self.distribution.get_fullname()
platform = get_platform()
zfname = os.path.join(self.dist_dir,
"%s.%s.zip" % (fullname, platform, ))
if hasattr(self.freezer_module, "zipit"):
self.freezer_module.zipit(self, self.bootstrap_dir, zfname)
else:
create_zipfile(self.bootstrap_dir, zfname, compress=True)
# Only remove bootstrap dir if option is passed
if (self.rm_freeze_dir_after_zipping
and self.rm_freeze_dir_after_zipping != 'False'):
really_rmtree(self.bootstrap_dir)
def _check_begin_patch(self):
"""Begin patching the current file, if not already.
This method is called by all file-patching commands; if there is
no file open for patching then the current target is opened.
"""
if not self.outfile and not self.dry_run:
if os.path.exists(self.target) and not os.path.isfile(self.target):
really_rmtree(self.target)
self.new_target = self.target + ".new"
while os.path.exists(self.new_target):
self.new_target += ".new"
if os.path.exists(self.target):
self.infile = open(self.target,"rb")
else:
self.infile = BytesIO("".encode("ascii"))
self.outfile = open(self.new_target,"wb")
def _run_create_zipfile(self):
"""Zip up the final distribution."""
if self.compress:
fullname = self.distribution.get_fullname()
platform = get_platform()
zfname = os.path.join(self.dist_dir,"%s.%s.zip"%(fullname,platform,))
if hasattr(self.freezer_module,"zipit"):
self.freezer_module.zipit(self,self.bootstrap_dir,zfname)
else:
if self.compress == 'zip':
print "zipping up the esky with compression"
create_zipfile(self.bootstrap_dir,zfname,compress=True)
really_rmtree(self.bootstrap_dir)
elif self.compress == 'ZIP':
print "zipping up the esky without compression"
create_zipfile(self.bootstrap_dir,zfname,compress=False)
really_rmtree(self.bootstrap_dir)
else:
print("To zip the esky use compress or c set to ZIP or zip")
def _do_MOVE_FROM(self):
"""Execute the MOVE_FROM command.
This reads a path from the command stream, and moves whatever is
at that path to the current target path. The source path is
interpreted relative to the directory containing the current path;
this caters for the common case of moving a file within the same
directory.
"""
self._check_end_patch()
source_path = os.path.join(os.path.dirname(self.target),self._read_path())
self._check_path(source_path)
if not self.dry_run:
if os.path.exists(self.target):
if os.path.isdir(self.target):
really_rmtree(self.target)
else:
os.unlink(self.target)
if sys.platform == "win32":
time.sleep(0.01)
really_rename(source_path,self.target)
def test_patch_bigfile(self):
tdir = tempfile.mkdtemp()
try:
data = [os.urandom(100)*10 for i in xrange(6)]
for nm in ("source","target"):
with open(os.path.join(tdir,nm),"wb") as f:
for i in xrange(1000):
for chunk in data:
f.write(chunk)
data[2],data[3] = data[3],data[2]
with open(os.path.join(tdir,"patch"),"wb") as f:
esky.patch.write_patch(os.path.join(tdir,"source"),os.path.join(tdir,"target"),f)
dgst1 = esky.patch.calculate_digest(os.path.join(tdir,"target"))
dgst2 = esky.patch.calculate_digest(os.path.join(tdir,"source"))
self.assertNotEquals(dgst1,dgst2)
with open(os.path.join(tdir,"patch"),"rb") as f:
esky.patch.apply_patch(os.path.join(tdir,"source"),f)
dgst3 = esky.patch.calculate_digest(os.path.join(tdir,"source"))
self.assertEquals(dgst1,dgst3)
finally:
really_rmtree(tdir)
def _do_MOVE_FROM(self):
"""Execute the MOVE_FROM command.
This reads a path from the command stream, and moves whatever is
at that path to the current target path. The source path is
interpreted relative to the directory containing the current path;
this caters for the common case of moving a file within the same
directory.
"""
self._check_end_patch()
source_path = os.path.join(os.path.dirname(self.target),self._read_path())
self._check_path(source_path)
if not self.dry_run:
if os.path.exists(self.target):
if os.path.isdir(self.target):
really_rmtree(self.target)
else:
os.unlink(self.target)
if sys.platform == "win32":
time.sleep(0.01)
really_rename(source_path,self.target)
def setUp(self):
self.tests_root = dirname(__file__)
platform = get_platform()
self.tfdir = tfdir = os.path.join(self.tests_root,"patch-test-files")
self.workdir = workdir = os.path.join(self.tests_root,"patch-test-temp."+platform)
if not os.path.isdir(tfdir):
os.makedirs(tfdir)
if os.path.isdir(workdir):
really_rmtree(workdir)
os.makedirs(workdir)
self.src_dir = os.path.join(workdir, 'source')
self.tgt_dir = os.path.join(workdir, 'target')
# Ensure we have the expected test files.
# Download from PyPI if necessary.
for (tfname,hash) in self._TEST_FILES:
tfpath = os.path.join(tfdir,tfname)
if not os.path.exists(tfpath):
data = urllib2.urlopen(self._TEST_FILES_URL+tfname).read()
assert hashlib.md5(data).hexdigest() == hash
with open(tfpath,"wb") as f:
f.write(data)
def test_patch_bigfile(self):
tdir = tempfile.mkdtemp()
try:
data = [os.urandom(100)*10 for i in xrange(6)]
for nm in ("source","target"):
with open(os.path.join(tdir,nm),"wb") as f:
for i in xrange(1000):
for chunk in data:
f.write(chunk)
data[2],data[3] = data[3],data[2]
with open(os.path.join(tdir,"patch"),"wb") as f:
esky.patch.write_patch(os.path.join(tdir,"source"),os.path.join(tdir,"target"),f)
dgst1 = esky.patch.calculate_digest(os.path.join(tdir,"target"))
dgst2 = esky.patch.calculate_digest(os.path.join(tdir,"source"))
self.assertNotEquals(dgst1,dgst2)
with open(os.path.join(tdir,"patch"),"rb") as f:
esky.patch.apply_patch(os.path.join(tdir,"source"),f)
dgst3 = esky.patch.calculate_digest(os.path.join(tdir,"source"))
self.assertEquals(dgst1,dgst3)
finally:
really_rmtree(tdir)
def _do_COPY_FROM(self):
"""Execute the COPY_FROM command.
This reads a path from the command stream, and copies whatever is
at that path to the current target path. The source path is
interpreted relative to the directory containing the current path;
this caters for the common case of copying a file within the same
directory.
"""
self._check_end_patch()
source_path = os.path.join(os.path.dirname(self.target),self._read_path())
self._check_path(source_path)
if not self.dry_run:
if os.path.exists(self.target):
if os.path.isdir(self.target):
really_rmtree(self.target)
else:
os.unlink(self.target)
if os.path.isfile(source_path):
shutil.copy2(source_path,self.target)
else:
shutil.copytree(source_path,self.target)
def _do_COPY_FROM(self):
"""Execute the COPY_FROM command.
This reads a path from the command stream, and copies whatever is
at that path to the current target path. The source path is
interpreted relative to the directory containing the current path;
this caters for the common case of copying a file within the same
directory.
"""
self._check_end_patch()
source_path = os.path.join(os.path.dirname(self.target),self._read_path())
self._check_path(source_path)
if not self.dry_run:
if os.path.exists(self.target):
if os.path.isdir(self.target):
really_rmtree(self.target)
else:
os.unlink(self.target)
if os.path.isfile(source_path):
shutil.copy2(source_path,self.target)
else:
shutil.copytree(source_path,self.target)
def setUp(self):
self.tests_root = dirname(__file__)
platform = get_platform()
self.tfdir = tfdir = os.path.join(self.tests_root, "patch-test-files")
self.workdir = workdir = os.path.join(self.tests_root,
"patch-test-temp." + platform)
if not os.path.isdir(tfdir):
os.makedirs(tfdir)
if os.path.isdir(workdir):
really_rmtree(workdir)
os.makedirs(workdir)
self.src_dir = os.path.join(workdir, 'source')
self.tgt_dir = os.path.join(workdir, 'target')
# Ensure we have the expected test files.
# Download from PyPI if necessary.
for (tfname, hash) in self._TEST_FILES:
tfpath = os.path.join(tfdir, tfname)
if not os.path.exists(tfpath):
data = urllib2.urlopen(self._TEST_FILES_URL + tfname).read()
assert hashlib.md5(data).hexdigest() == hash
with open(tfpath, "wb") as f:
f.write(data)
def __exit__(self,*args):
really_rmtree(self.path)
# TODO: remove compatability hooks for ESKY_APPDATA_DIR="".
# If a patch fails to apply because we've put an appdata dir
# where it doesn't expect one, try again with old layout.
for _ in xrange(2):
# Apply any patches in turn.
for (patchfile,patchurl) in patches:
try:
try:
with open(patchfile,"rb") as f:
apply_patch(uppath,f)
except EnvironmentError, e:
if e.errno not in (errno.ENOENT,):
raise
if not path[0][0].endswith(".patch"):
raise
really_rmtree(uppath)
os.mkdir(uppath)
self._copy_best_version(app,uppath,False)
break
except (PatchError,EnvironmentError):
self.version_graph.remove_all_links(patchurl)
try:
os.unlink(patchfile)
except EnvironmentError:
pass
raise
else:
break
# Find the actual version dir that we're unpacking.
# TODO: remove compatability hooks for ESKY_APPDATA_DIR=""
vdir = join_app_version(app.name,version,app.platform)
def main(args):
"""Command-line diffing and patching for esky."""
parser = optparse.OptionParser()
parser.add_option("-z","--zipped",action="store_true",dest="zipped",
help="work with zipped source/target dirs")
parser.add_option("-Z","--deep-zipped",action="store_true",
dest="deep_zipped",
help="work with deep zipped source/target dirs")
parser.add_option("","--diff-window",dest="diff_window",metavar="N",
help="set the window size for diffing files")
parser.add_option("","--dry-run",dest="dry_run",action="store_true",
help="print commands instead of executing them")
(opts,args) = parser.parse_args(args)
if opts.deep_zipped:
opts.zipped = True
if opts.zipped:
workdir = tempfile.mkdtemp()
if opts.diff_window:
scale = 1
if opts.diff_window[-1].lower() == "k":
scale = 1024
opts.diff_window = opts.diff_window[:-1]
elif opts.diff_window[-1].lower() == "m":
scale = 1024 * 1024
opts.diff_window = opts.diff_window[:-1]
elif opts.diff_window[-1].lower() == "g":
scale = 1024 * 1024 * 1024
opts.diff_window = opts.diff_window[:-1]
opts.diff_window = int(float(opts.diff_window)*scale)
stream = None
try:
cmd = args[0]
if cmd == "diff":
# Generate a diff between two files/directories.
# If --zipped is specified, the source and/or target is unzipped
# to a temporary directory before processing.
source = args[1]
target = args[2]
if len(args) > 3:
stream = open(args[3],"wb")
else:
stream = sys.stdout
if opts.zipped:
if os.path.isfile(source):
source_zip = source
source = os.path.join(workdir,"source")
if opts.deep_zipped:
deep_extract_zipfile(source_zip,source)
else:
extract_zipfile(source_zip,source)
if os.path.isfile(target):
target_zip = target
target = os.path.join(workdir,"target")
if opts.deep_zipped:
deep_extract_zipfile(target_zip,target)
else:
extract_zipfile(target_zip,target)
write_patch(source,target,stream,diff_window_size=opts.diff_window)
elif cmd == "patch":
# Patch a file or directory.
# If --zipped is specified, the target is unzipped to a temporary
# directory before processing, then overwritten with a zipfile
# containing the new directory contents.
target = args[1]
if len(args) > 2:
stream = open(args[2],"rb")
else:
stream = sys.stdin
target_zip = None
if opts.zipped:
if os.path.isfile(target):
target_zip = target
target = os.path.join(workdir,"target")
if opts.deep_zipped:
deep_extract_zipfile(target_zip,target)
else:
extract_zipfile(target_zip,target)
apply_patch(target,stream,dry_run=opts.dry_run)
if opts.zipped and target_zip is not None:
target_dir = os.path.dirname(target_zip)
(fd,target_temp) = tempfile.mkstemp(dir=target_dir)
os.close(fd)
if opts.deep_zipped:
prefix = zipfile_common_prefix_dir(target_zip)
def name_filter(nm):
return prefix + nm
create_zipfile(target,target_temp,name_filter)
else:
create_zipfile(target,target_temp)
if sys.platform == "win32":
os.unlink(target_zip)
time.sleep(0.01)
really_rename(target_temp,target_zip)
else:
raise ValueError("invalid command: " + cmd)
finally:
if stream is not None:
if stream not in (sys.stdin,sys.stdout,):
stream.close()
if opts.zipped:
really_rmtree(workdir)
def run(self):
self.tempdir = tempfile.mkdtemp()
try:
self._run()
finally:
really_rmtree(self.tempdir)
def tearDown(self):
really_rmtree(self.tdir)
t2.wait()
return runme
t1 = threading.Thread(target=runit(e1, trigger1, trigger2))
t2 = threading.Thread(target=runit(e2, trigger2, trigger1))
t1.start()
t2.start()
t1.join()
t2.join()
assert len(locked) == 1
assert (e1 in locked or e2 in locked)
assert len(errors) == 1
assert isinstance(errors[0], esky.EskyLockedError)
finally:
really_rmtree(appdir)
def test_esky_lock_breaking(self):
"""Test that breaking the lock on an Esky works correctly."""
appdir = tempfile.mkdtemp()
try:
os.makedirs(
os.path.join(appdir, ESKY_APPDATA_DIR, "testapp-0.1",
ESKY_CONTROL_DIR))
open(
os.path.join(appdir, ESKY_APPDATA_DIR, "testapp-0.1",
ESKY_CONTROL_DIR, "bootstrap-manifest.txt"),
"wb").close()
e1 = esky.Esky(appdir, "http://example.com/downloads/")
e2 = esky.Esky(appdir, "http://example.com/downloads/")
trigger1 = threading.Event()
def _run_eskytester(self, options):
"""Build and run the eskytester app using the given distutils options.
The "eskytester" application can be found next to this file, and the
sequence of tests performed range across "script1.py" to "script3.py".
"""
olddir = os.path.abspath(os.curdir)
# tdir = os.path.join(os.path.dirname(__file__),"DIST")
# if os.path.exists(tdir):
# really_rmtree(tdir)
# os.mkdir(tdir)
tdir = tempfile.mkdtemp()
server = None
script2 = None
try:
options.setdefault("build",
{})["build_base"] = os.path.join(tdir, "build")
options.setdefault("bdist",
{})["dist_dir"] = os.path.join(tdir, "dist")
# Set some callbacks to test that they work correctly
options.setdefault("bdist_esky", {}).setdefault(
"pre_freeze_callback",
"esky.tests.test_esky.assert_freezedir_exists")
options.setdefault("bdist_esky",
{}).setdefault("pre_zip_callback",
assert_freezedir_exists)
options["bdist_esky"].setdefault("excludes",
[]).extend(["Tkinter", "tkinter"])
platform = get_platform()
deploydir = "deploy.%s" % (platform, )
esky_root = dirname(dirname(dirname(__file__)))
os.chdir(tdir)
shutil.copytree(
os.path.join(esky_root, "esky", "tests", "eskytester"),
"eskytester")
dir_util._path_created.clear()
# Build three increasing versions of the test package.
# Version 0.2 will include a bundled MSVCRT on win32.
# Version 0.3 will be distributed as a patch.
metadata = dict(
name="eskytester",
packages=["eskytester"],
author="rfk",
description="the esky test package",
data_files=[("data", ["eskytester/datafile.txt"])],
package_data={"eskytester": ["pkgdata.txt"]},
)
options2 = options.copy()
options2["bdist_esky"] = options["bdist_esky"].copy()
options2["bdist_esky"]["bundle_msvcrt"] = True
script1 = "eskytester/script1.py"
script2 = Executable([None, open("eskytester/script2.py")],
name="script2")
script3 = "eskytester/script3.py"
dist_setup(version="0.1",
scripts=[script1],
options=options,
script_args=["bdist_esky"],
**metadata)
dist_setup(version="0.2",
scripts=[script1, script2],
options=options2,
script_args=["bdist_esky"],
**metadata)
dist_setup(version="0.3",
scripts=[script2, script3],
options=options,
script_args=["bdist_esky_patch"],
**metadata)
os.unlink(
os.path.join(tdir, "dist",
"eskytester-0.3.%s.zip" % (platform, )))
# Check that the patches apply cleanly
uzdir = os.path.join(tdir, "unzip")
deep_extract_zipfile(
os.path.join(tdir, "dist",
"eskytester-0.1.%s.zip" % (platform, )), uzdir)
with open(
os.path.join(
tdir, "dist",
"eskytester-0.3.%s.from-0.1.patch" % (platform, )),
"rb") as f:
esky.patch.apply_patch(uzdir, f)
really_rmtree(uzdir)
deep_extract_zipfile(
os.path.join(tdir, "dist",
"eskytester-0.2.%s.zip" % (platform, )), uzdir)
with open(
os.path.join(
tdir, "dist",
"eskytester-0.3.%s.from-0.2.patch" % (platform, )),
"rb") as f:
esky.patch.apply_patch(uzdir, f)
really_rmtree(uzdir)
# Serve the updates at LOCAL_HTTP_PORT set in esky.util
print "running local update server"
try:
server = HTTPServer(("localhost", LOCAL_HTTP_PORT),
SimpleHTTPRequestHandler)
except Exception:
# in travis ci we start our own server
pass
else:
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
# Set up the deployed esky environment for the initial version
zfname = os.path.join(tdir, "dist",
"eskytester-0.1.%s.zip" % (platform, ))
os.mkdir(deploydir)
extract_zipfile(zfname, deploydir)
# Run the scripts in order.
if options["bdist_esky"]["freezer_module"] == "py2app":
appdir = os.path.join(deploydir, os.listdir(deploydir)[0])
cmd1 = os.path.join(appdir, "Contents", "MacOS", "script1")
cmd2 = os.path.join(appdir, "Contents", "MacOS", "script2")
cmd3 = os.path.join(appdir, "Contents", "MacOS", "script3")
else:
appdir = deploydir
if sys.platform == "win32":
cmd1 = os.path.join(deploydir, "script1.exe")
cmd2 = os.path.join(deploydir, "script2.exe")
cmd3 = os.path.join(deploydir, "script3.exe")
else:
cmd1 = os.path.join(deploydir, "script1")
cmd2 = os.path.join(deploydir, "script2")
cmd3 = os.path.join(deploydir, "script3")
print "spawning eskytester script1", options["bdist_esky"][
"freezer_module"]
os.unlink(
os.path.join(tdir, "dist",
"eskytester-0.1.%s.zip" % (platform, )))
p = subprocess.Popen(cmd1)
assert p.wait() == 0
os.unlink(os.path.join(appdir, "tests-completed"))
print "spawning eskytester script2"
os.unlink(
os.path.join(tdir, "dist",
"eskytester-0.2.%s.zip" % (platform, )))
p = subprocess.Popen(cmd2)
assert p.wait() == 0
os.unlink(os.path.join(appdir, "tests-completed"))
print "spawning eskytester script3"
p = subprocess.Popen(cmd3)
assert p.wait() == 0
os.unlink(os.path.join(appdir, "tests-completed"))
finally:
if script2:
script2.script[1].close()
os.chdir(olddir)
if sys.platform == "win32":
# wait for the cleanup-at-exit pocess to finish
time.sleep(4)
really_rmtree(tdir)
if server:
server.shutdown()
def tearDown(self):
really_rmtree(self.workdir)
locked.append(e)
t1.set()
t2.wait()
return runme
t1 = threading.Thread(target=runit(e1,trigger1,trigger2))
t2 = threading.Thread(target=runit(e2,trigger2,trigger1))
t1.start()
t2.start()
t1.join()
t2.join()
assert len(locked) == 1
assert (e1 in locked or e2 in locked)
assert len(errors) == 1
assert isinstance(errors[0],esky.EskyLockedError)
finally:
really_rmtree(appdir)
def test_esky_lock_breaking(self):
"""Test that breaking the lock on an Esky works correctly."""
appdir = tempfile.mkdtemp()
try:
os.makedirs(os.path.join(appdir,ESKY_APPDATA_DIR,"testapp-0.1",ESKY_CONTROL_DIR))
open(os.path.join(appdir,ESKY_APPDATA_DIR,"testapp-0.1",ESKY_CONTROL_DIR,"bootstrap-manifest.txt"),"wb").close()
e1 = esky.Esky(appdir,"http://example.com/downloads/")
e2 = esky.Esky(appdir,"http://example.com/downloads/")
trigger1 = threading.Event(); trigger2 = threading.Event()
errors = []
def run1():
try:
e1.lock()
def main(args):
"""Command-line diffing and patching for esky."""
parser = optparse.OptionParser()
parser.add_option("-z","--zipped",action="store_true",dest="zipped",
help="work with zipped source/target dirs")
parser.add_option("-Z","--deep-zipped",action="store_true",
dest="deep_zipped",
help="work with deep zipped source/target dirs")
parser.add_option("","--diff-window",dest="diff_window",metavar="N",
help="set the window size for diffing files")
parser.add_option("","--dry-run",dest="dry_run",action="store_true",
help="print commands instead of executing them")
(opts,args) = parser.parse_args(args)
if opts.deep_zipped:
opts.zipped = True
if opts.zipped:
workdir = tempfile.mkdtemp()
if opts.diff_window:
scale = 1
if opts.diff_window[-1].lower() == "k":
scale = 1024
opts.diff_window = opts.diff_window[:-1]
elif opts.diff_window[-1].lower() == "m":
scale = 1024 * 1024
opts.diff_window = opts.diff_window[:-1]
elif opts.diff_window[-1].lower() == "g":
scale = 1024 * 1024 * 1024
opts.diff_window = opts.diff_window[:-1]
opts.diff_window = int(float(opts.diff_window)*scale)
stream = None
try:
cmd = args[0]
if cmd == "diff":
# Generate a diff between two files/directories.
# If --zipped is specified, the source and/or target is unzipped
# to a temporary directory before processing.
source = args[1]
target = args[2]
if len(args) > 3:
stream = open(args[3],"wb")
else:
stream = sys.stdout
if opts.zipped:
if os.path.isfile(source):
source_zip = source
source = os.path.join(workdir,"source")
if opts.deep_zipped:
deep_extract_zipfile(source_zip,source)
else:
extract_zipfile(source_zip,source)
if os.path.isfile(target):
target_zip = target
target = os.path.join(workdir,"target")
if opts.deep_zipped:
deep_extract_zipfile(target_zip,target)
else:
extract_zipfile(target_zip,target)
write_patch(source,target,stream,diff_window_size=opts.diff_window)
elif cmd == "patch":
# Patch a file or directory.
# If --zipped is specified, the target is unzipped to a temporary
# directory before processing, then overwritten with a zipfile
# containing the new directory contents.
target = args[1]
if len(args) > 2:
stream = open(args[2],"rb")
else:
stream = sys.stdin
target_zip = None
if opts.zipped:
if os.path.isfile(target):
target_zip = target
target = os.path.join(workdir,"target")
if opts.deep_zipped:
deep_extract_zipfile(target_zip,target)
else:
extract_zipfile(target_zip,target)
apply_patch(target,stream,dry_run=opts.dry_run)
if opts.zipped and target_zip is not None:
target_dir = os.path.dirname(target_zip)
(fd,target_temp) = tempfile.mkstemp(dir=target_dir)
os.close(fd)
if opts.deep_zipped:
prefix = zipfile_common_prefix_dir(target_zip)
def name_filter(nm):
return prefix + nm
create_zipfile(target,target_temp,name_filter)
else:
create_zipfile(target,target_temp)
if sys.platform == "win32":
os.unlink(target_zip)
time.sleep(0.01)
really_rename(target_temp,target_zip)
else:
raise ValueError("invalid command: " + cmd)
finally:
if stream is not None:
if stream not in (sys.stdin,sys.stdout,):
stream.close()
if opts.zipped:
really_rmtree(workdir)
def tearDown(self):
really_rmtree(self.workdir)
def _run_eskytester(self, options):
"""Build and run the eskytester app using the given distutils options.
The "eskytester" application can be found next to this file, and the
sequence of tests performed range across "script1.py" to "script3.py".
"""
olddir = os.path.abspath(os.curdir)
# tdir = os.path.join(os.path.dirname(__file__),"DIST")
# if os.path.exists(tdir):
# really_rmtree(tdir)
# os.mkdir(tdir)
tdir = tempfile.mkdtemp()
server = None
script2 = None
try:
options.setdefault("build", {})["build_base"] = os.path.join(tdir, "build")
options.setdefault("bdist", {})["dist_dir"] = os.path.join(tdir, "dist")
# Set some callbacks to test that they work correctly
options.setdefault("bdist_esky", {}).setdefault(
"pre_freeze_callback", "esky.tests.test_esky.assert_freezedir_exists"
)
options.setdefault("bdist_esky", {}).setdefault("pre_zip_callback", assert_freezedir_exists)
platform = get_platform()
deploydir = "deploy.%s" % (platform,)
esky_root = dirname(dirname(dirname(__file__)))
os.chdir(tdir)
shutil.copytree(os.path.join(esky_root, "esky", "tests", "eskytester"), "eskytester")
dir_util._path_created.clear()
# Build three increasing versions of the test package.
# Version 0.2 will include a bundled MSVCRT on win32.
# Version 0.3 will be distributed as a patch.
metadata = dict(
name="eskytester",
packages=["eskytester"],
author="rfk",
description="the esky test package",
data_files=[("data", ["eskytester/datafile.txt"])],
package_data={"eskytester": ["pkgdata.txt"]},
)
options2 = options.copy()
options2["bdist_esky"] = options["bdist_esky"].copy()
options2["bdist_esky"]["bundle_msvcrt"] = True
script1 = "eskytester/script1.py"
script2 = Executable([None, open("eskytester/script2.py")], name="script2")
script3 = "eskytester/script3.py"
dist_setup(version="0.1", scripts=[script1], options=options, script_args=["bdist_esky"], **metadata)
dist_setup(
version="0.2", scripts=[script1, script2], options=options2, script_args=["bdist_esky"], **metadata
)
dist_setup(
version="0.3", scripts=[script2, script3], options=options, script_args=["bdist_esky_patch"], **metadata
)
os.unlink(os.path.join(tdir, "dist", "eskytester-0.3.%s.zip" % (platform,)))
# Check that the patches apply cleanly
uzdir = os.path.join(tdir, "unzip")
deep_extract_zipfile(os.path.join(tdir, "dist", "eskytester-0.1.%s.zip" % (platform,)), uzdir)
with open(os.path.join(tdir, "dist", "eskytester-0.3.%s.from-0.1.patch" % (platform,)), "rb") as f:
esky.patch.apply_patch(uzdir, f)
shutil.rmtree(uzdir)
deep_extract_zipfile(os.path.join(tdir, "dist", "eskytester-0.2.%s.zip" % (platform,)), uzdir)
with open(os.path.join(tdir, "dist", "eskytester-0.3.%s.from-0.2.patch" % (platform,)), "rb") as f:
esky.patch.apply_patch(uzdir, f)
shutil.rmtree(uzdir)
# Serve the updates at http://localhost:8000/dist/
print "running local update server"
server = HTTPServer(("localhost", 8000), SimpleHTTPRequestHandler)
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
# Set up the deployed esky environment for the initial version
zfname = os.path.join(tdir, "dist", "eskytester-0.1.%s.zip" % (platform,))
os.mkdir(deploydir)
extract_zipfile(zfname, deploydir)
# Run the scripts in order.
if options["bdist_esky"]["freezer_module"] == "py2app":
appdir = os.path.join(deploydir, os.listdir(deploydir)[0])
cmd1 = os.path.join(appdir, "Contents", "MacOS", "script1")
cmd2 = os.path.join(appdir, "Contents", "MacOS", "script2")
cmd3 = os.path.join(appdir, "Contents", "MacOS", "script3")
else:
appdir = deploydir
if sys.platform == "win32":
cmd1 = os.path.join(deploydir, "script1.exe")
cmd2 = os.path.join(deploydir, "script2.exe")
cmd3 = os.path.join(deploydir, "script3.exe")
else:
cmd1 = os.path.join(deploydir, "script1")
cmd2 = os.path.join(deploydir, "script2")
cmd3 = os.path.join(deploydir, "script3")
print "spawning eskytester script1", options["bdist_esky"]["freezer_module"]
os.unlink(os.path.join(tdir, "dist", "eskytester-0.1.%s.zip" % (platform,)))
p = subprocess.Popen(cmd1)
assert p.wait() == 0
os.unlink(os.path.join(appdir, "tests-completed"))
print "spawning eskytester script2"
os.unlink(os.path.join(tdir, "dist", "eskytester-0.2.%s.zip" % (platform,)))
p = subprocess.Popen(cmd2)
assert p.wait() == 0
os.unlink(os.path.join(appdir, "tests-completed"))
print "spawning eskytester script3"
p = subprocess.Popen(cmd3)
assert p.wait() == 0
os.unlink(os.path.join(appdir, "tests-completed"))
finally:
if script2:
script2.script[1].close()
os.chdir(olddir)
if sys.platform == "win32":
# wait for the cleanup-at-exit pocess to finish
time.sleep(4)
really_rmtree(tdir)
if server:
server.shutdown()
def __exit__(self,*args):
really_rmtree(self.path)
def lock(self, num_retries=0):
"""Lock the application directory for exclusive write access.
If the appdir is already locked by another process/thread then
EskyLockedError is raised. There is no way to perform a blocking
lock on an appdir.
Locking is achieved by creating a "locked" directory and writing the
current process/thread ID into it. os.mkdir is atomic on all platforms
that we care about.
This also has the side-effect of failing early if the user does not
have permission to modify the application directory.
"""
if self.sudo_proxy is not None:
return self.sudo_proxy.lock()
if num_retries > 5:
raise EskyLockedError
if threading:
curthread = threading.currentThread()
try:
threadid = curthread.ident
except AttributeError:
threadid = curthread.getName()
else:
threadid = "0"
myid = "%s-%s-%s" % (socket.gethostname(), os.getpid(), threadid)
lockdir = os.path.join(self.appdir, "locked")
# Do I already own the lock?
if os.path.exists(os.path.join(lockdir, myid)):
# Update file mtime to keep it safe from breakers
os.utime(os.path.join(lockdir, myid), None)
self._lock_count += 1
return True
# Try to make the "locked" directory.
try:
os.mkdir(lockdir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
# Is it stale? If so, break it and try again.
try:
newest_mtime = os.path.getmtime(lockdir)
for nm in os.listdir(lockdir):
mtime = os.path.getmtime(os.path.join(lockdir, nm))
if mtime > newest_mtime:
newest_mtime = mtime
if newest_mtime + self.lock_timeout < time.time():
really_rmtree(lockdir)
return self.lock(num_retries+1)
else:
raise EskyLockedError
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ENOTDIR,):
raise
return self.lock(num_retries+1)
else:
# Success! Record my ownership
open(os.path.join(lockdir, myid), "wb").close()
self._lock_count = 1
return True
def tearDown(self):
really_rmtree(self.testdir)
