def _fetch_file_iter(self,app,url): nm = os.path.basename(urlparse(url).path) outfilenm = os.path.join(self._workdir(app,"downloads"),nm) if not os.path.exists(outfilenm): try: infile = self.open_url(urljoin(self.download_url,url)) outfile_size = 0 # The to determine size of download, so that we can # detect corrupted or truncated downloads. try: infile_size = infile.size except AttributeError: try: fh = infile.fileno() except AttributeError: infile_size = None else: infile_size = os.fstat(fh).st_size # Read it into a temporary file, then rename into place. try: partfilenm = outfilenm + ".part" partfile = open(partfilenm,"wb") try: data = infile.read(1024*64) while data: yield {"status": "downloading", "size": infile_size, "received": partfile.tell(), } partfile.write(data) outfile_size += len(data) data = infile.read(1024*64) if infile_size is not None: if outfile_size != infile_size: err = "corrupted download: %s" % (url,) raise IOError(err) except Exception: partfile.close() os.unlink(partfilenm) raise else: partfile.close() really_rename(partfilenm,outfilenm) finally: infile.close() except PermissionError: # if we don't have access rights to write part file just let # the exception bubble up and sudo process will handle it raise except Exception: # Something went wrong. To avoid infinite looping, we # must remove that file from the link graph. self.version_graph.remove_all_links(url) raise yield {"status":"ready","path":outfilenm}
def _fetch_file_iter(self, app, url): nm = os.path.basename(urlparse(url).path) outfilenm = os.path.join(self._workdir(app, "downloads"), nm) if not os.path.exists(outfilenm): try: infile = self.open_url(urljoin(self.download_url, url)) outfile_size = 0 # The to determine size of download, so that we can # detect corrupted or truncated downloads. try: infile_size = infile.size except AttributeError: try: fh = infile.fileno() except AttributeError: infile_size = None else: infile_size = os.fstat(fh).st_size # Read it into a temporary file, then rename into place. try: partfilenm = outfilenm + ".part" partfile = open(partfilenm, "wb") try: data = infile.read(1024 * 64) while data: yield { "status": "downloading", "size": infile_size, "received": partfile.tell(), } partfile.write(data) outfile_size += len(data) data = infile.read(1024 * 64) if infile_size is not None: if outfile_size != infile_size: err = "corrupted download: %s" % (url, ) raise IOError(err) except Exception: partfile.close() os.unlink(partfilenm) raise else: partfile.close() really_rename(partfilenm, outfilenm) finally: infile.close() except Exception: # Something went wrong. To avoid infinite looping, we # must remove that file from the link graph. self.version_graph.remove_all_links(url) raise yield {"status": "ready", "path": outfilenm}
def _copy(self,source,target): is_win32 = (sys.platform == "win32") if is_win32 and os.path.exists(target) and target != source: target_old = get_backup_filename(target) really_rename(target,target_old) try: self._do_copy(source,target) except: really_rename(target_old,target) raise else: try: os.unlink(target_old) except EnvironmentError: pass else: target_old = None if os.path.isdir(target) and os.path.isfile(source): target_old = get_backup_filename(target) really_rename(target,target_old) elif os.path.isfile(target) and os.path.isdir(source): target_old = get_backup_filename(target) really_rename(target,target_old) self._do_copy(source,target) if target_old is not None: self._remove(target_old)
def _check_end_patch(self): """Finish patching the current file, if there is one. This method is called by all non-file-patching commands; if there is a file open for patching then it is closed and committed. """ if self.outfile and not self.dry_run: self.infile.close() self.infile = None self.outfile.close() self.outfile = None if os.path.exists(self.target): os.unlink(self.target) if sys.platform == "win32": time.sleep(0.01) really_rename(self.new_target,self.target) self.new_target = None
def install_version(self, version): """Install the specified version of the app. This fetches the specified version if necessary, then makes it available as a version directory inside the app directory. It does not modify any other installed versions. """ if self.sudo_proxy is not None: return self.sudo_proxy.install_version(version) # Extract update then rename into position in main app directory vsdir = self._get_versions_dir() target = join_app_version(self.name, version, self.platform) target = os.path.join(vsdir, target) # Guard against malicious input (might be called with root privs) assert os.path.dirname(target) == vsdir if not os.path.exists(target): self.fetch_version(version) source = self.version_finder.has_version(self, version) # TODO: remove compatability hooks for ESKY_APPDATA_DIR="". # This is our chance to migrate to the new appdata dir layout, # by installing into it. if vsdir == self.appdir and ESKY_APPDATA_DIR: vsdir = os.path.join(self.appdir, ESKY_APPDATA_DIR) try: os.mkdir(vsdir) except EnvironmentError as e: if e.errno not in (errno.EEXIST,): raise else: copy_ownership_info(self.appdir, vsdir) target = os.path.join(vsdir, os.path.basename(target)) self.lock() try: if not os.path.exists(target): really_rename(source, target) trn = esky.fstransact.FSTransaction(self.appdir) try: self._unpack_bootstrap_env(target, trn) except Exception: trn.abort() raise else: trn.commit() finally: self.unlock()
def _fetch_file_iter(self,app,url): nm = os.path.basename(urlparse(url).path) outfilenm = os.path.join(self._workdir(app,"downloads"),nm) if not os.path.exists(outfilenm): infile = self.open_url(urljoin(self.download_url,url)) outfile_size = 0 try: infile_size = infile.size except AttributeError: try: fh = infile.fileno() except AttributeError: infile_size = None else: infile_size = os.fstat(fh).st_size try: partfilenm = outfilenm + ".part" partfile = open(partfilenm,"wb") try: data = infile.read(1024*64) while data: yield {"status": "downloading", "size": infile_size, "received": partfile.tell(), } partfile.write(data) outfile_size += len(data) data = infile.read(1024*64) if infile_size is not None: if outfile_size != infile_size: self.version_graph.remove_all_links(url) err = "corrupted download: %s" % (url,) raise IOError(err) except Exception: partfile.close() os.unlink(partfilenm) raise else: partfile.close() really_rename(partfilenm,outfilenm) finally: infile.close() yield {"status":"ready","path":outfilenm}
def _do_MOVE_FROM(self): """Execute the MOVE_FROM command. This reads a path from the command stream, and moves whatever is at that path to the current target path. The source path is interpreted relative to the directory containing the current path; this caters for the common case of moving a file within the same directory. """ self._check_end_patch() source_path = os.path.join(os.path.dirname(self.target),self._read_path()) self._check_path(source_path) if not self.dry_run: if os.path.exists(self.target): if os.path.isdir(self.target): really_rmtree(self.target) else: os.unlink(self.target) if sys.platform == "win32": time.sleep(0.01) really_rename(source_path,self.target)
def _fetch_file_iter(self, app, url): nm = os.path.basename(urlparse(url).path) outfilenm = os.path.join(self._workdir(app, "downloads"), nm) if not os.path.exists(outfilenm): infile = self.open_url(urljoin(self.download_url, url)) try: infile_size = infile.size except AttributeError: try: fh = infile.fileno() except AttributeError: infile_size = None else: infile_size = os.fstat(fh).st_size try: partfilenm = outfilenm + ".part" partfile = open(partfilenm, "wb") try: data = infile.read(1024 * 512) while data: yield { "status": "downloading", "size": infile_size, "received": partfile.tell(), } partfile.write(data) data = infile.read(1024 * 512) except Exception: partfile.close() os.unlink(partfilenm) raise else: partfile.close() really_rename(partfilenm, outfilenm) finally: infile.close() yield {"status": "ready", "path": outfilenm}
def main(args): """Command-line diffing and patching for esky.""" parser = optparse.OptionParser() parser.add_option("-z","--zipped",action="store_true",dest="zipped", help="work with zipped source/target dirs") parser.add_option("-Z","--deep-zipped",action="store_true", dest="deep_zipped", help="work with deep zipped source/target dirs") parser.add_option("","--diff-window",dest="diff_window",metavar="N", help="set the window size for diffing files") parser.add_option("","--dry-run",dest="dry_run",action="store_true", help="print commands instead of executing them") (opts,args) = parser.parse_args(args) if opts.deep_zipped: opts.zipped = True if opts.zipped: workdir = tempfile.mkdtemp() if opts.diff_window: scale = 1 if opts.diff_window[-1].lower() == "k": scale = 1024 opts.diff_window = opts.diff_window[:-1] elif opts.diff_window[-1].lower() == "m": scale = 1024 * 1024 opts.diff_window = opts.diff_window[:-1] elif opts.diff_window[-1].lower() == "g": scale = 1024 * 1024 * 1024 opts.diff_window = opts.diff_window[:-1] opts.diff_window = int(float(opts.diff_window)*scale) stream = None try: cmd = args[0] if cmd == "diff": # Generate a diff between two files/directories. # If --zipped is specified, the source and/or target is unzipped # to a temporary directory before processing. source = args[1] target = args[2] if len(args) > 3: stream = open(args[3],"wb") else: stream = sys.stdout if opts.zipped: if os.path.isfile(source): source_zip = source source = os.path.join(workdir,"source") if opts.deep_zipped: deep_extract_zipfile(source_zip,source) else: extract_zipfile(source_zip,source) if os.path.isfile(target): target_zip = target target = os.path.join(workdir,"target") if opts.deep_zipped: deep_extract_zipfile(target_zip,target) else: extract_zipfile(target_zip,target) write_patch(source,target,stream,diff_window_size=opts.diff_window) elif cmd == "patch": # Patch a file or directory. # If --zipped is specified, the target is unzipped to a temporary # directory before processing, then overwritten with a zipfile # containing the new directory contents. target = args[1] if len(args) > 2: stream = open(args[2],"rb") else: stream = sys.stdin target_zip = None if opts.zipped: if os.path.isfile(target): target_zip = target target = os.path.join(workdir,"target") if opts.deep_zipped: deep_extract_zipfile(target_zip,target) else: extract_zipfile(target_zip,target) apply_patch(target,stream,dry_run=opts.dry_run) if opts.zipped and target_zip is not None: target_dir = os.path.dirname(target_zip) (fd,target_temp) = tempfile.mkstemp(dir=target_dir) os.close(fd) if opts.deep_zipped: prefix = zipfile_common_prefix_dir(target_zip) def name_filter(nm): return prefix + nm create_zipfile(target,target_temp,name_filter) else: create_zipfile(target,target_temp) if sys.platform == "win32": os.unlink(target_zip) time.sleep(0.01) really_rename(target_temp,target_zip) else: raise ValueError("invalid command: " + cmd) finally: if stream is not None: if stream not in (sys.stdin,sys.stdout,): stream.close() if opts.zipped: really_rmtree(workdir)
def _move(self,source,target): if sys.platform == "win32" and os.path.exists(target): # os.rename won't overwite an existing file on win32. # We also want to use this on files that are potentially open. # Renaming the target out of the way is the best we can do :-( target_old = target + ".old" while os.path.exists(target_old): target_old = target_old + ".old" really_rename(target,target_old) try: really_rename(source,target) except: really_rename(target_old,target) raise else: try: self._remove(target_old) except EnvironmentError: pass else: target_old = None if os.path.isdir(target) and os.path.isfile(source): target_old = target + ".old" while os.path.exists(target_old): target_old = target_old + ".old" really_rename(target,target_old) elif os.path.isfile(target) and os.path.isdir(source): target_old = target + ".old" while os.path.exists(target_old): target_old = target_old + ".old" really_rename(target,target_old) self._create_parents(target) really_rename(source,target) if target_old is not None: self._remove(target_old)
def uninstall_version(self, version): """Uninstall the specified version of the app.""" if self.sudo_proxy is not None: return self.sudo_proxy.uninstall_version(version) vsdir = self._get_versions_dir() target_name = join_app_version(self.name, version, self.platform) target = os.path.join(vsdir, target_name) # Guard against malicious input (might be called with root privs) assert os.path.dirname(target) == vsdir # TODO: remove compatability hooks for ESKY_APPDATA_DIR="". if ESKY_APPDATA_DIR and not os.path.exists(target): if vsdir == self.appdir: target = os.path.join(self.appdir, ESKY_APPDATA_DIR, target_name) else: target = os.path.join(self.appdir, target_name) lockfile = os.path.join(target, ESKY_CONTROL_DIR, "lockfile.txt") bsfile = os.path.join(target, ESKY_CONTROL_DIR, "bootstrap-manifest.txt") bsfile_old = os.path.join(target, ESKY_CONTROL_DIR, "bootstrap-manifest-old.txt") self.lock() try: if not os.path.exists(target): return # Clean up the bootstrapping environment in a transaction. # This might fail on windows if the version is locked. try: trn = esky.fstransact.FSTransaction(self.appdir) try: self._cleanup_bootstrap_env(version, trn) except Exception: trn.abort() raise else: trn.commit() except EnvironmentError: if is_locked_version_dir(target): raise VersionLockedError("version in use: %s" % (version,)) raise # Disable the version by renaming its bootstrap-manifest.txt file. # To avoid clobbering in-use version, respect locks on this file. if sys.platform == "win32": try: really_rename(bsfile, bsfile_old) except EnvironmentError: raise VersionLockedError("version in use: %s" % (version,)) else: try: f = open(lockfile, "r") except EnvironmentError as e: if e.errno != errno.ENOENT: raise else: try: fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) except EnvironmentError as e: if e.errno not in (errno.EACCES, errno.EAGAIN,): raise msg = "version in use: %s" % (version,) raise VersionLockedError(msg) else: really_rename(bsfile, bsfile_old) finally: f.close() finally: self.unlock()
vdir = join_app_version(app.name,version,app.platform) vdirpath = os.path.join(uppath,ESKY_APPDATA_DIR,vdir) if not os.path.isdir(vdirpath): vdirpath = os.path.join(uppath,vdir) if not os.path.isdir(vdirpath): self.version_graph.remove_all_links(path[0][1]) err = version + ": version directory does not exist" raise EskyVersionError(err) # Move anything that's not the version dir into "bootstrap" dir. ctrlpath = os.path.join(vdirpath,ESKY_CONTROL_DIR) bspath = os.path.join(ctrlpath,"bootstrap") if not os.path.isdir(bspath): os.makedirs(bspath) for nm in os.listdir(uppath): if nm != vdir and nm != ESKY_APPDATA_DIR: really_rename(os.path.join(uppath,nm), os.path.join(bspath,nm)) # Check that it has an esky-files/bootstrap-manifest.txt file bsfile = os.path.join(ctrlpath,"bootstrap-manifest.txt") if not os.path.exists(bsfile): self.version_graph.remove_all_links(path[0][1]) err = version + ": version has no bootstrap-manifest.txt" raise EskyVersionError(err) # Make it available for upgrading, replacing anything # that we previously had available. rdpath = self._ready_name(app,version) tmpnm = None try: if os.path.exists(rdpath): tmpnm = rdpath + ".old" while os.path.exists(tmpnm): tmpnm = tmpnm + ".old"