def _sync(self, verbosity, output_fd): fd_pipes = {1: output_fd, 2: output_fd} opts = list(self.opts) if self.rsh: opts.append("-e") opts.append(self.rsh) opts.extend(f"--exclude={x}" for x in self.excludes) opts.extend(f"--include={x}" for x in self.includes) if verbosity < 0: opts.append("--quiet") elif verbosity > 0: opts.extend('-v' for x in range(verbosity)) # zip limits to the shortest iterable ret = None for count, ip in zip(range(self.retries), self._get_ips()): cmd = [ self.binary_path, self.uri.replace(self.hostname, ip, 1), self.basedir ] + opts ret = self._spawn(cmd, fd_pipes) if ret == 0: return True elif ret == 1: raise base.SyncError( "rsync command syntax error: {' '.join(cmd)}") elif ret == 11: raise base.SyncError("rsync ran out of disk space") # need to do something here instead of just restarting... # else: # print(ret) raise base.SyncError("all attempts failed")
def _post_download(self, path): super()._post_download(path) # create tempdir for staging decompression if not ensure_dirs( self.tempdir, mode=0o755, uid=self.uid, gid=self.gid): raise base.SyncError( f'failed creating repo update dir: {self.tempdir!r}') exts = {'gz': 'gzip', 'bz2': 'bzip2', 'xz': 'xz'} compression = exts[self.uri.rsplit('.', 1)[1]] # use tar instead of tarfile so we can easily strip leading path components # TODO: programmatically determine how man components to strip? cmd = [ 'tar', '--extract', f'--{compression}', '-f', path, '--strip-components=1', '--no-same-owner', '-C', self.tempdir ] with open(os.devnull) as f: ret = self._spawn(cmd, pipes={1: f.fileno(), 2: f.fileno()}) if ret: raise base.SyncError('failed to unpack tarball') # TODO: verify gpg data if it exists # move old repo out of the way and then move new, unpacked repo into place try: os.rename(self.basedir, self.tempdir_old) os.rename(self.tempdir, self.basedir) except OSError as e: raise base.SyncError(f'failed to update repo: {e.strerror}') from e # register old repo removal after it has been successfully replaced atexit.register( partial(shutil.rmtree, self.tempdir_old, ignore_errors=True))
def _get_ips(self): if self.use_proxy: # If we're using a proxy, name resolution is best left to the proxy. yield self.hostname return af_fam = socket.AF_INET if self.is_ipv6: af_fam = socket.AF_INET6 try: for ipaddr in socket.getaddrinfo(self.hostname, None, af_fam, socket.SOCK_STREAM): if ipaddr[0] == socket.AF_INET6: yield f"[{ipaddr[4][0]}]" else: yield ipaddr[4][0] except OSError as e: raise base.SyncError( f"DNS resolution failed for {self.hostname!r}: {e.strerror}")
def _sync(self, verbosity, output_fd, force=False, **kwargs): dest = self._pre_download() if self.uri.startswith('https://'): # default to using system ssl certs context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) else: context = None headers = {} etag_path = pjoin(self.basedir, '.etag') modified_path = pjoin(self.basedir, '.modified') if not force: # use cached ETag to check if updates exist previous_etag = None try: with open(etag_path, 'r') as f: previous_etag = f.read() except FileNotFoundError: pass if previous_etag: headers['If-None-Match'] = previous_etag # use cached modification timestamp to check if updates exist previous_modified = None try: with open(modified_path, 'r') as f: previous_modified = f.read() except FileNotFoundError: pass if previous_modified: headers['If-Modified-Since'] = previous_modified req = urllib.request.Request(self.uri, headers=headers, method='GET') # TODO: add customizable timeout try: resp = urllib.request.urlopen(req, context=context) except urllib.error.URLError as e: if e.code == 304: # TODO: raise exception to notify user the repo is up to date? return True raise base.SyncError( f'failed fetching {self.uri!r}: {e.reason}') from e # Manually check cached values ourselves since some servers appear to # ignore If-None-Match or If-Modified-Since headers. etag = resp.getheader('ETag', '') modified = resp.getheader('Last-Modified', '') if not force: if etag == previous_etag: return True if modified == previous_modified: return True try: os.makedirs(self.basedir, exist_ok=True) except OSError as e: raise base.SyncError( f'failed creating repo dir {self.basedir!r}: {e.strerror}' ) from e length = resp.getheader('content-length') if length: length = int(length) blocksize = max(4096, length // 100) else: blocksize = 1000000 try: self._download = AtomicWriteFile(dest, binary=True, perms=0o644) except OSError as e: raise base.PathError(self.basedir, e.strerror) from e # retrieve the file while providing simple progress output size = 0 while True: buf = resp.read(blocksize) if not buf: if length: sys.stdout.write('\n') break self._download.write(buf) size += len(buf) if length: sys.stdout.write('\r') progress = '=' * int(size / length * 50) percent = int(size / length * 100) sys.stdout.write("[%-50s] %d%%" % (progress, percent)) self._post_download(dest) # TODO: store this in pkgcore cache dir instead? # update cached ETag/Last-Modified values if etag: with open(etag_path, 'w') as f: f.write(etag) if modified: with open(modified_path, 'w') as f: f.write(modified) return True