def _fetch_file_iter(self,app,url):
nm = os.path.basename(urlparse(url).path)
outfilenm = os.path.join(self._workdir(app,"downloads"),nm)
if not os.path.exists(outfilenm):
try:
infile = self.open_url(urljoin(self.download_url,url))
outfile_size = 0
# The to determine size of download, so that we can
# detect corrupted or truncated downloads.
try:
infile_size = infile.size
except AttributeError:
try:
fh = infile.fileno()
except AttributeError:
infile_size = None
else:
infile_size = os.fstat(fh).st_size
# Read it into a temporary file, then rename into place.
try:
partfilenm = outfilenm + ".part"
partfile = open(partfilenm,"wb")
try:
data = infile.read(1024*64)
while data:
yield {"status": "downloading",
"size": infile_size,
"received": partfile.tell(),
}
partfile.write(data)
outfile_size += len(data)
data = infile.read(1024*64)
if infile_size is not None:
if outfile_size != infile_size:
err = "corrupted download: %s" % (url,)
raise IOError(err)
except Exception:
partfile.close()
os.unlink(partfilenm)
raise
else:
partfile.close()
really_rename(partfilenm,outfilenm)
finally:
infile.close()
except PermissionError:
# if we don't have access rights to write part file just let
# the exception bubble up and sudo process will handle it
raise
except Exception:
# Something went wrong. To avoid infinite looping, we
# must remove that file from the link graph.
self.version_graph.remove_all_links(url)
raise
yield {"status":"ready","path":outfilenm}
def _fetch_file_iter(self, app, url):
nm = os.path.basename(urlparse(url).path)
outfilenm = os.path.join(self._workdir(app, "downloads"), nm)
if not os.path.exists(outfilenm):
try:
infile = self.open_url(urljoin(self.download_url, url))
outfile_size = 0
# The to determine size of download, so that we can
# detect corrupted or truncated downloads.
try:
infile_size = infile.size
except AttributeError:
try:
fh = infile.fileno()
except AttributeError:
infile_size = None
else:
infile_size = os.fstat(fh).st_size
# Read it into a temporary file, then rename into place.
try:
partfilenm = outfilenm + ".part"
partfile = open(partfilenm, "wb")
try:
data = infile.read(1024 * 64)
while data:
yield {
"status": "downloading",
"size": infile_size,
"received": partfile.tell(),
}
partfile.write(data)
outfile_size += len(data)
data = infile.read(1024 * 64)
if infile_size is not None:
if outfile_size != infile_size:
err = "corrupted download: %s" % (url, )
raise IOError(err)
except Exception:
partfile.close()
os.unlink(partfilenm)
raise
else:
partfile.close()
really_rename(partfilenm, outfilenm)
finally:
infile.close()
except Exception:
# Something went wrong. To avoid infinite looping, we
# must remove that file from the link graph.
self.version_graph.remove_all_links(url)
raise
yield {"status": "ready", "path": outfilenm}
def _copy(self,source,target):
is_win32 = (sys.platform == "win32")
if is_win32 and os.path.exists(target) and target != source:
target_old = get_backup_filename(target)
really_rename(target,target_old)
try:
self._do_copy(source,target)
except:
really_rename(target_old,target)
raise
else:
try:
os.unlink(target_old)
except EnvironmentError:
pass
else:
target_old = None
if os.path.isdir(target) and os.path.isfile(source):
target_old = get_backup_filename(target)
really_rename(target,target_old)
elif os.path.isfile(target) and os.path.isdir(source):
target_old = get_backup_filename(target)
really_rename(target,target_old)
self._do_copy(source,target)
if target_old is not None:
self._remove(target_old)
def _copy(self,source,target):
is_win32 = (sys.platform == "win32")
if is_win32 and os.path.exists(target) and target != source:
target_old = get_backup_filename(target)
really_rename(target,target_old)
try:
self._do_copy(source,target)
except:
really_rename(target_old,target)
raise
else:
try:
os.unlink(target_old)
except EnvironmentError:
pass
else:
target_old = None
if os.path.isdir(target) and os.path.isfile(source):
target_old = get_backup_filename(target)
really_rename(target,target_old)
elif os.path.isfile(target) and os.path.isdir(source):
target_old = get_backup_filename(target)
really_rename(target,target_old)
self._do_copy(source,target)
if target_old is not None:
self._remove(target_old)
def _check_end_patch(self):
"""Finish patching the current file, if there is one.
This method is called by all non-file-patching commands; if there is
a file open for patching then it is closed and committed.
"""
if self.outfile and not self.dry_run:
self.infile.close()
self.infile = None
self.outfile.close()
self.outfile = None
if os.path.exists(self.target):
os.unlink(self.target)
if sys.platform == "win32":
time.sleep(0.01)
really_rename(self.new_target,self.target)
self.new_target = None
def _check_end_patch(self):
"""Finish patching the current file, if there is one.
This method is called by all non-file-patching commands; if there is
a file open for patching then it is closed and committed.
"""
if self.outfile and not self.dry_run:
self.infile.close()
self.infile = None
self.outfile.close()
self.outfile = None
if os.path.exists(self.target):
os.unlink(self.target)
if sys.platform == "win32":
time.sleep(0.01)
really_rename(self.new_target,self.target)
self.new_target = None
def install_version(self, version):
"""Install the specified version of the app.
This fetches the specified version if necessary, then makes it
available as a version directory inside the app directory. It
does not modify any other installed versions.
"""
if self.sudo_proxy is not None:
return self.sudo_proxy.install_version(version)
# Extract update then rename into position in main app directory
vsdir = self._get_versions_dir()
target = join_app_version(self.name, version, self.platform)
target = os.path.join(vsdir, target)
# Guard against malicious input (might be called with root privs)
assert os.path.dirname(target) == vsdir
if not os.path.exists(target):
self.fetch_version(version)
source = self.version_finder.has_version(self, version)
# TODO: remove compatability hooks for ESKY_APPDATA_DIR="".
# This is our chance to migrate to the new appdata dir layout,
# by installing into it.
if vsdir == self.appdir and ESKY_APPDATA_DIR:
vsdir = os.path.join(self.appdir, ESKY_APPDATA_DIR)
try:
os.mkdir(vsdir)
except EnvironmentError as e:
if e.errno not in (errno.EEXIST,):
raise
else:
copy_ownership_info(self.appdir, vsdir)
target = os.path.join(vsdir, os.path.basename(target))
self.lock()
try:
if not os.path.exists(target):
really_rename(source, target)
trn = esky.fstransact.FSTransaction(self.appdir)
try:
self._unpack_bootstrap_env(target, trn)
except Exception:
trn.abort()
raise
else:
trn.commit()
finally:
self.unlock()
def _fetch_file_iter(self,app,url):
nm = os.path.basename(urlparse(url).path)
outfilenm = os.path.join(self._workdir(app,"downloads"),nm)
if not os.path.exists(outfilenm):
infile = self.open_url(urljoin(self.download_url,url))
outfile_size = 0
try:
infile_size = infile.size
except AttributeError:
try:
fh = infile.fileno()
except AttributeError:
infile_size = None
else:
infile_size = os.fstat(fh).st_size
try:
partfilenm = outfilenm + ".part"
partfile = open(partfilenm,"wb")
try:
data = infile.read(1024*64)
while data:
yield {"status": "downloading",
"size": infile_size,
"received": partfile.tell(),
}
partfile.write(data)
outfile_size += len(data)
data = infile.read(1024*64)
if infile_size is not None:
if outfile_size != infile_size:
self.version_graph.remove_all_links(url)
err = "corrupted download: %s" % (url,)
raise IOError(err)
except Exception:
partfile.close()
os.unlink(partfilenm)
raise
else:
partfile.close()
really_rename(partfilenm,outfilenm)
finally:
infile.close()
yield {"status":"ready","path":outfilenm}
def _do_MOVE_FROM(self):
"""Execute the MOVE_FROM command.
This reads a path from the command stream, and moves whatever is
at that path to the current target path. The source path is
interpreted relative to the directory containing the current path;
this caters for the common case of moving a file within the same
directory.
"""
self._check_end_patch()
source_path = os.path.join(os.path.dirname(self.target),self._read_path())
self._check_path(source_path)
if not self.dry_run:
if os.path.exists(self.target):
if os.path.isdir(self.target):
really_rmtree(self.target)
else:
os.unlink(self.target)
if sys.platform == "win32":
time.sleep(0.01)
really_rename(source_path,self.target)
def _do_MOVE_FROM(self):
"""Execute the MOVE_FROM command.
This reads a path from the command stream, and moves whatever is
at that path to the current target path. The source path is
interpreted relative to the directory containing the current path;
this caters for the common case of moving a file within the same
directory.
"""
self._check_end_patch()
source_path = os.path.join(os.path.dirname(self.target),self._read_path())
self._check_path(source_path)
if not self.dry_run:
if os.path.exists(self.target):
if os.path.isdir(self.target):
really_rmtree(self.target)
else:
os.unlink(self.target)
if sys.platform == "win32":
time.sleep(0.01)
really_rename(source_path,self.target)
def _fetch_file_iter(self, app, url):
nm = os.path.basename(urlparse(url).path)
outfilenm = os.path.join(self._workdir(app, "downloads"), nm)
if not os.path.exists(outfilenm):
infile = self.open_url(urljoin(self.download_url, url))
try:
infile_size = infile.size
except AttributeError:
try:
fh = infile.fileno()
except AttributeError:
infile_size = None
else:
infile_size = os.fstat(fh).st_size
try:
partfilenm = outfilenm + ".part"
partfile = open(partfilenm, "wb")
try:
data = infile.read(1024 * 512)
while data:
yield {
"status": "downloading",
"size": infile_size,
"received": partfile.tell(),
}
partfile.write(data)
data = infile.read(1024 * 512)
except Exception:
partfile.close()
os.unlink(partfilenm)
raise
else:
partfile.close()
really_rename(partfilenm, outfilenm)
finally:
infile.close()
yield {"status": "ready", "path": outfilenm}
def main(args):
"""Command-line diffing and patching for esky."""
parser = optparse.OptionParser()
parser.add_option("-z","--zipped",action="store_true",dest="zipped",
help="work with zipped source/target dirs")
parser.add_option("-Z","--deep-zipped",action="store_true",
dest="deep_zipped",
help="work with deep zipped source/target dirs")
parser.add_option("","--diff-window",dest="diff_window",metavar="N",
help="set the window size for diffing files")
parser.add_option("","--dry-run",dest="dry_run",action="store_true",
help="print commands instead of executing them")
(opts,args) = parser.parse_args(args)
if opts.deep_zipped:
opts.zipped = True
if opts.zipped:
workdir = tempfile.mkdtemp()
if opts.diff_window:
scale = 1
if opts.diff_window[-1].lower() == "k":
scale = 1024
opts.diff_window = opts.diff_window[:-1]
elif opts.diff_window[-1].lower() == "m":
scale = 1024 * 1024
opts.diff_window = opts.diff_window[:-1]
elif opts.diff_window[-1].lower() == "g":
scale = 1024 * 1024 * 1024
opts.diff_window = opts.diff_window[:-1]
opts.diff_window = int(float(opts.diff_window)*scale)
stream = None
try:
cmd = args[0]
if cmd == "diff":
# Generate a diff between two files/directories.
# If --zipped is specified, the source and/or target is unzipped
# to a temporary directory before processing.
source = args[1]
target = args[2]
if len(args) > 3:
stream = open(args[3],"wb")
else:
stream = sys.stdout
if opts.zipped:
if os.path.isfile(source):
source_zip = source
source = os.path.join(workdir,"source")
if opts.deep_zipped:
deep_extract_zipfile(source_zip,source)
else:
extract_zipfile(source_zip,source)
if os.path.isfile(target):
target_zip = target
target = os.path.join(workdir,"target")
if opts.deep_zipped:
deep_extract_zipfile(target_zip,target)
else:
extract_zipfile(target_zip,target)
write_patch(source,target,stream,diff_window_size=opts.diff_window)
elif cmd == "patch":
# Patch a file or directory.
# If --zipped is specified, the target is unzipped to a temporary
# directory before processing, then overwritten with a zipfile
# containing the new directory contents.
target = args[1]
if len(args) > 2:
stream = open(args[2],"rb")
else:
stream = sys.stdin
target_zip = None
if opts.zipped:
if os.path.isfile(target):
target_zip = target
target = os.path.join(workdir,"target")
if opts.deep_zipped:
deep_extract_zipfile(target_zip,target)
else:
extract_zipfile(target_zip,target)
apply_patch(target,stream,dry_run=opts.dry_run)
if opts.zipped and target_zip is not None:
target_dir = os.path.dirname(target_zip)
(fd,target_temp) = tempfile.mkstemp(dir=target_dir)
os.close(fd)
if opts.deep_zipped:
prefix = zipfile_common_prefix_dir(target_zip)
def name_filter(nm):
return prefix + nm
create_zipfile(target,target_temp,name_filter)
else:
create_zipfile(target,target_temp)
if sys.platform == "win32":
os.unlink(target_zip)
time.sleep(0.01)
really_rename(target_temp,target_zip)
else:
raise ValueError("invalid command: " + cmd)
finally:
if stream is not None:
if stream not in (sys.stdin,sys.stdout,):
stream.close()
if opts.zipped:
really_rmtree(workdir)
def main(args):
"""Command-line diffing and patching for esky."""
parser = optparse.OptionParser()
parser.add_option("-z","--zipped",action="store_true",dest="zipped",
help="work with zipped source/target dirs")
parser.add_option("-Z","--deep-zipped",action="store_true",
dest="deep_zipped",
help="work with deep zipped source/target dirs")
parser.add_option("","--diff-window",dest="diff_window",metavar="N",
help="set the window size for diffing files")
parser.add_option("","--dry-run",dest="dry_run",action="store_true",
help="print commands instead of executing them")
(opts,args) = parser.parse_args(args)
if opts.deep_zipped:
opts.zipped = True
if opts.zipped:
workdir = tempfile.mkdtemp()
if opts.diff_window:
scale = 1
if opts.diff_window[-1].lower() == "k":
scale = 1024
opts.diff_window = opts.diff_window[:-1]
elif opts.diff_window[-1].lower() == "m":
scale = 1024 * 1024
opts.diff_window = opts.diff_window[:-1]
elif opts.diff_window[-1].lower() == "g":
scale = 1024 * 1024 * 1024
opts.diff_window = opts.diff_window[:-1]
opts.diff_window = int(float(opts.diff_window)*scale)
stream = None
try:
cmd = args[0]
if cmd == "diff":
# Generate a diff between two files/directories.
# If --zipped is specified, the source and/or target is unzipped
# to a temporary directory before processing.
source = args[1]
target = args[2]
if len(args) > 3:
stream = open(args[3],"wb")
else:
stream = sys.stdout
if opts.zipped:
if os.path.isfile(source):
source_zip = source
source = os.path.join(workdir,"source")
if opts.deep_zipped:
deep_extract_zipfile(source_zip,source)
else:
extract_zipfile(source_zip,source)
if os.path.isfile(target):
target_zip = target
target = os.path.join(workdir,"target")
if opts.deep_zipped:
deep_extract_zipfile(target_zip,target)
else:
extract_zipfile(target_zip,target)
write_patch(source,target,stream,diff_window_size=opts.diff_window)
elif cmd == "patch":
# Patch a file or directory.
# If --zipped is specified, the target is unzipped to a temporary
# directory before processing, then overwritten with a zipfile
# containing the new directory contents.
target = args[1]
if len(args) > 2:
stream = open(args[2],"rb")
else:
stream = sys.stdin
target_zip = None
if opts.zipped:
if os.path.isfile(target):
target_zip = target
target = os.path.join(workdir,"target")
if opts.deep_zipped:
deep_extract_zipfile(target_zip,target)
else:
extract_zipfile(target_zip,target)
apply_patch(target,stream,dry_run=opts.dry_run)
if opts.zipped and target_zip is not None:
target_dir = os.path.dirname(target_zip)
(fd,target_temp) = tempfile.mkstemp(dir=target_dir)
os.close(fd)
if opts.deep_zipped:
prefix = zipfile_common_prefix_dir(target_zip)
def name_filter(nm):
return prefix + nm
create_zipfile(target,target_temp,name_filter)
else:
create_zipfile(target,target_temp)
if sys.platform == "win32":
os.unlink(target_zip)
time.sleep(0.01)
really_rename(target_temp,target_zip)
else:
raise ValueError("invalid command: " + cmd)
finally:
if stream is not None:
if stream not in (sys.stdin,sys.stdout,):
stream.close()
if opts.zipped:
really_rmtree(workdir)
def _move(self,source,target):
if sys.platform == "win32" and os.path.exists(target):
# os.rename won't overwite an existing file on win32.
# We also want to use this on files that are potentially open.
# Renaming the target out of the way is the best we can do :-(
target_old = target + ".old"
while os.path.exists(target_old):
target_old = target_old + ".old"
really_rename(target,target_old)
try:
really_rename(source,target)
except:
really_rename(target_old,target)
raise
else:
try:
self._remove(target_old)
except EnvironmentError:
pass
else:
target_old = None
if os.path.isdir(target) and os.path.isfile(source):
target_old = target + ".old"
while os.path.exists(target_old):
target_old = target_old + ".old"
really_rename(target,target_old)
elif os.path.isfile(target) and os.path.isdir(source):
target_old = target + ".old"
while os.path.exists(target_old):
target_old = target_old + ".old"
really_rename(target,target_old)
self._create_parents(target)
really_rename(source,target)
if target_old is not None:
self._remove(target_old)
def uninstall_version(self, version):
"""Uninstall the specified version of the app."""
if self.sudo_proxy is not None:
return self.sudo_proxy.uninstall_version(version)
vsdir = self._get_versions_dir()
target_name = join_app_version(self.name, version, self.platform)
target = os.path.join(vsdir, target_name)
# Guard against malicious input (might be called with root privs)
assert os.path.dirname(target) == vsdir
# TODO: remove compatability hooks for ESKY_APPDATA_DIR="".
if ESKY_APPDATA_DIR and not os.path.exists(target):
if vsdir == self.appdir:
target = os.path.join(self.appdir, ESKY_APPDATA_DIR,
target_name)
else:
target = os.path.join(self.appdir, target_name)
lockfile = os.path.join(target, ESKY_CONTROL_DIR, "lockfile.txt")
bsfile = os.path.join(target, ESKY_CONTROL_DIR,
"bootstrap-manifest.txt")
bsfile_old = os.path.join(target, ESKY_CONTROL_DIR,
"bootstrap-manifest-old.txt")
self.lock()
try:
if not os.path.exists(target):
return
# Clean up the bootstrapping environment in a transaction.
# This might fail on windows if the version is locked.
try:
trn = esky.fstransact.FSTransaction(self.appdir)
try:
self._cleanup_bootstrap_env(version, trn)
except Exception:
trn.abort()
raise
else:
trn.commit()
except EnvironmentError:
if is_locked_version_dir(target):
raise VersionLockedError("version in use: %s" % (version,))
raise
# Disable the version by renaming its bootstrap-manifest.txt file.
# To avoid clobbering in-use version, respect locks on this file.
if sys.platform == "win32":
try:
really_rename(bsfile, bsfile_old)
except EnvironmentError:
raise VersionLockedError("version in use: %s" % (version,))
else:
try:
f = open(lockfile, "r")
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
else:
try:
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
except EnvironmentError as e:
if e.errno not in (errno.EACCES, errno.EAGAIN,):
raise
msg = "version in use: %s" % (version,)
raise VersionLockedError(msg)
else:
really_rename(bsfile, bsfile_old)
finally:
f.close()
finally:
self.unlock()
vdir = join_app_version(app.name,version,app.platform)
vdirpath = os.path.join(uppath,ESKY_APPDATA_DIR,vdir)
if not os.path.isdir(vdirpath):
vdirpath = os.path.join(uppath,vdir)
if not os.path.isdir(vdirpath):
self.version_graph.remove_all_links(path[0][1])
err = version + ": version directory does not exist"
raise EskyVersionError(err)
# Move anything that's not the version dir into "bootstrap" dir.
ctrlpath = os.path.join(vdirpath,ESKY_CONTROL_DIR)
bspath = os.path.join(ctrlpath,"bootstrap")
if not os.path.isdir(bspath):
os.makedirs(bspath)
for nm in os.listdir(uppath):
if nm != vdir and nm != ESKY_APPDATA_DIR:
really_rename(os.path.join(uppath,nm),
os.path.join(bspath,nm))
# Check that it has an esky-files/bootstrap-manifest.txt file
bsfile = os.path.join(ctrlpath,"bootstrap-manifest.txt")
if not os.path.exists(bsfile):
self.version_graph.remove_all_links(path[0][1])
err = version + ": version has no bootstrap-manifest.txt"
raise EskyVersionError(err)
# Make it available for upgrading, replacing anything
# that we previously had available.
rdpath = self._ready_name(app,version)
tmpnm = None
try:
if os.path.exists(rdpath):
tmpnm = rdpath + ".old"
while os.path.exists(tmpnm):
tmpnm = tmpnm + ".old"
def _move(self,source,target):
if sys.platform == "win32" and os.path.exists(target):
# os.rename won't overwite an existing file on win32.
# We also want to use this on files that are potentially open.
# Renaming the target out of the way is the best we can do :-(
target_old = target + ".old"
while os.path.exists(target_old):
target_old = target_old + ".old"
really_rename(target,target_old)
try:
really_rename(source,target)
except:
really_rename(target_old,target)
raise
else:
try:
self._remove(target_old)
except EnvironmentError:
pass
else:
target_old = None
if os.path.isdir(target) and os.path.isfile(source):
target_old = target + ".old"
while os.path.exists(target_old):
target_old = target_old + ".old"
really_rename(target,target_old)
elif os.path.isfile(target) and os.path.isdir(source):
target_old = target + ".old"
while os.path.exists(target_old):
target_old = target_old + ".old"
really_rename(target,target_old)
self._create_parents(target)
really_rename(source,target)
if target_old is not None:
self._remove(target_old)
