def _sync(self, verbosity, output_fd):
fd_pipes = {1: output_fd, 2: output_fd}
opts = list(self.opts)
if self.rsh:
opts.append("-e")
opts.append(self.rsh)
opts.extend(f"--exclude={x}" for x in self.excludes)
opts.extend(f"--include={x}" for x in self.includes)
if verbosity < 0:
opts.append("--quiet")
elif verbosity > 0:
opts.extend('-v' for x in range(verbosity))
# zip limits to the shortest iterable
ret = None
for count, ip in zip(range(self.retries), self._get_ips()):
cmd = [
self.binary_path,
self.uri.replace(self.hostname, ip, 1), self.basedir
] + opts
ret = self._spawn(cmd, fd_pipes)
if ret == 0:
return True
elif ret == 1:
raise base.SyncError(
"rsync command syntax error: {' '.join(cmd)}")
elif ret == 11:
raise base.SyncError("rsync ran out of disk space")
# need to do something here instead of just restarting...
# else:
# print(ret)
raise base.SyncError("all attempts failed")
def _post_download(self, path):
super()._post_download(path)
# create tempdir for staging decompression
if not ensure_dirs(
self.tempdir, mode=0o755, uid=self.uid, gid=self.gid):
raise base.SyncError(
f'failed creating repo update dir: {self.tempdir!r}')
exts = {'gz': 'gzip', 'bz2': 'bzip2', 'xz': 'xz'}
compression = exts[self.uri.rsplit('.', 1)[1]]
# use tar instead of tarfile so we can easily strip leading path components
# TODO: programmatically determine how man components to strip?
cmd = [
'tar', '--extract', f'--{compression}', '-f', path,
'--strip-components=1', '--no-same-owner', '-C', self.tempdir
]
with open(os.devnull) as f:
ret = self._spawn(cmd, pipes={1: f.fileno(), 2: f.fileno()})
if ret:
raise base.SyncError('failed to unpack tarball')
# TODO: verify gpg data if it exists
# move old repo out of the way and then move new, unpacked repo into place
try:
os.rename(self.basedir, self.tempdir_old)
os.rename(self.tempdir, self.basedir)
except OSError as e:
raise base.SyncError(f'failed to update repo: {e.strerror}') from e
# register old repo removal after it has been successfully replaced
atexit.register(
partial(shutil.rmtree, self.tempdir_old, ignore_errors=True))
def _get_ips(self):
if self.use_proxy:
# If we're using a proxy, name resolution is best left to the proxy.
yield self.hostname
return
af_fam = socket.AF_INET
if self.is_ipv6:
af_fam = socket.AF_INET6
try:
for ipaddr in socket.getaddrinfo(self.hostname, None, af_fam,
socket.SOCK_STREAM):
if ipaddr[0] == socket.AF_INET6:
yield f"[{ipaddr[4][0]}]"
else:
yield ipaddr[4][0]
except OSError as e:
raise base.SyncError(
f"DNS resolution failed for {self.hostname!r}: {e.strerror}")
def _sync(self, verbosity, output_fd, force=False, **kwargs):
dest = self._pre_download()
if self.uri.startswith('https://'):
# default to using system ssl certs
context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
else:
context = None
headers = {}
etag_path = pjoin(self.basedir, '.etag')
modified_path = pjoin(self.basedir, '.modified')
if not force:
# use cached ETag to check if updates exist
previous_etag = None
try:
with open(etag_path, 'r') as f:
previous_etag = f.read()
except FileNotFoundError:
pass
if previous_etag:
headers['If-None-Match'] = previous_etag
# use cached modification timestamp to check if updates exist
previous_modified = None
try:
with open(modified_path, 'r') as f:
previous_modified = f.read()
except FileNotFoundError:
pass
if previous_modified:
headers['If-Modified-Since'] = previous_modified
req = urllib.request.Request(self.uri, headers=headers, method='GET')
# TODO: add customizable timeout
try:
resp = urllib.request.urlopen(req, context=context)
except urllib.error.URLError as e:
if e.code == 304:
# TODO: raise exception to notify user the repo is up to date?
return True
raise base.SyncError(
f'failed fetching {self.uri!r}: {e.reason}') from e
# Manually check cached values ourselves since some servers appear to
# ignore If-None-Match or If-Modified-Since headers.
etag = resp.getheader('ETag', '')
modified = resp.getheader('Last-Modified', '')
if not force:
if etag == previous_etag:
return True
if modified == previous_modified:
return True
try:
os.makedirs(self.basedir, exist_ok=True)
except OSError as e:
raise base.SyncError(
f'failed creating repo dir {self.basedir!r}: {e.strerror}'
) from e
length = resp.getheader('content-length')
if length:
length = int(length)
blocksize = max(4096, length // 100)
else:
blocksize = 1000000
try:
self._download = AtomicWriteFile(dest, binary=True, perms=0o644)
except OSError as e:
raise base.PathError(self.basedir, e.strerror) from e
# retrieve the file while providing simple progress output
size = 0
while True:
buf = resp.read(blocksize)
if not buf:
if length:
sys.stdout.write('\n')
break
self._download.write(buf)
size += len(buf)
if length:
sys.stdout.write('\r')
progress = '=' * int(size / length * 50)
percent = int(size / length * 100)
sys.stdout.write("[%-50s] %d%%" % (progress, percent))
self._post_download(dest)
# TODO: store this in pkgcore cache dir instead?
# update cached ETag/Last-Modified values
if etag:
with open(etag_path, 'w') as f:
f.write(etag)
if modified:
with open(modified_path, 'w') as f:
f.write(modified)
return True