def update(self):
'''Internal update function which performs the transfer'''
opts = self.options.get('emerge_config').opts
self.usersync_uid = self.options.get('usersync_uid', None)
enter_invalid = '--ask-enter-invalid' in opts
quiet = '--quiet' in opts
out = portage.output.EOutput(quiet=quiet)
syncuri = self.repo.sync_uri
if self.repo.module_specific_options.get('sync-rsync-vcs-ignore',
'false').lower() == 'true':
vcs_dirs = ()
else:
vcs_dirs = frozenset(VCS_DIRS)
vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
for vcs_dir in vcs_dirs:
writemsg_level(("!!! %s appears to be under revision " + \
"control (contains %s).\n!!! Aborting rsync sync "
"(override with \"sync-rsync-vcs-ignore = true\" in repos.conf).\n") % \
(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
return (1, False)
self.timeout = 180
rsync_opts = []
if self.settings["PORTAGE_RSYNC_OPTS"] == "":
rsync_opts = self._set_rsync_defaults()
else:
rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
self.extra_rsync_opts = list()
if self.repo.module_specific_options.get('sync-rsync-extra-opts'):
self.extra_rsync_opts.extend(
portage.util.shlex_split(
self.repo.module_specific_options['sync-rsync-extra-opts'])
)
exitcode = 0
verify_failure = False
# Process GLEP74 verification options.
# Default verification to 'no'; it's enabled for ::gentoo
# via default repos.conf though.
self.verify_metamanifest = (self.repo.module_specific_options.get(
'sync-rsync-verify-metamanifest', 'no') in ('yes', 'true'))
# Support overriding job count.
self.verify_jobs = self.repo.module_specific_options.get(
'sync-rsync-verify-jobs', None)
if self.verify_jobs is not None:
try:
self.verify_jobs = int(self.verify_jobs)
if self.verify_jobs < 0:
raise ValueError(self.verify_jobs)
except ValueError:
writemsg_level(
"!!! sync-rsync-verify-jobs not a positive integer: %s\n" %
(self.verify_jobs, ),
level=logging.WARNING,
noiselevel=-1)
self.verify_jobs = None
else:
if self.verify_jobs == 0:
# Use the apparent number of processors if gemato
# supports it.
self.verify_jobs = None
# Support overriding max age.
self.max_age = self.repo.module_specific_options.get(
'sync-rsync-verify-max-age', '')
if self.max_age:
try:
self.max_age = int(self.max_age)
if self.max_age < 0:
raise ValueError(self.max_age)
except ValueError:
writemsg_level(
"!!! sync-rsync-max-age must be a non-negative integer: %s\n"
% (self.max_age, ),
level=logging.WARNING,
noiselevel=-1)
self.max_age = 0
else:
self.max_age = 0
openpgp_env = None
if self.verify_metamanifest and gemato is not None:
# Use isolated environment if key is specified,
# system environment otherwise
if self.repo.sync_openpgp_key_path is not None:
openpgp_env = gemato.openpgp.OpenPGPEnvironment()
else:
openpgp_env = gemato.openpgp.OpenPGPSystemEnvironment()
try:
# Load and update the keyring early. If it fails, then verification
# will not be performed and the user will have to fix it and try again,
# so we may as well bail out before actual rsync happens.
if openpgp_env is not None and self.repo.sync_openpgp_key_path is not None:
try:
out.einfo('Using keys from %s' %
(self.repo.sync_openpgp_key_path, ))
with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
openpgp_env.import_key(f)
self._refresh_keys(openpgp_env)
except (GematoException, asyncio.TimeoutError) as e:
writemsg_level(
"!!! Manifest verification impossible due to keyring problem:\n%s\n"
% (e, ),
level=logging.ERROR,
noiselevel=-1)
return (1, False)
# Real local timestamp file.
self.servertimestampfile = os.path.join(self.repo.location,
"metadata",
"timestamp.chk")
content = portage.util.grabfile(self.servertimestampfile)
timestamp = 0
if content:
try:
timestamp = time.mktime(
time.strptime(content[0], TIMESTAMP_FORMAT))
except (OverflowError, ValueError):
pass
del content
try:
self.rsync_initial_timeout = \
int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
except ValueError:
self.rsync_initial_timeout = 15
try:
maxretries = int(self.settings["PORTAGE_RSYNC_RETRIES"])
except SystemExit as e:
raise # Needed else can't exit
except:
maxretries = -1 #default number of retries
if syncuri.startswith("file://"):
self.proto = "file"
dosyncuri = syncuri[7:]
unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
dosyncuri, timestamp, opts)
self._process_exitcode(exitcode, dosyncuri, out, 1)
if exitcode == 0:
if unchanged:
self.repo_storage.abort_update()
else:
self.repo_storage.commit_update()
self.repo_storage.garbage_collection()
return (exitcode, updatecache_flg)
retries = 0
try:
self.proto, user_name, hostname, port = re.split(
r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
syncuri,
maxsplit=4)[1:5]
except ValueError:
writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri,
noiselevel=-1,
level=logging.ERROR)
return (1, False)
self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
if port is None:
port = ""
if user_name is None:
user_name = ""
if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None:
getaddrinfo_host = hostname
else:
# getaddrinfo needs the brackets stripped
getaddrinfo_host = hostname[1:-1]
updatecache_flg = False
all_rsync_opts = set(self.rsync_opts)
all_rsync_opts.update(self.extra_rsync_opts)
family = socket.AF_UNSPEC
if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
family = socket.AF_INET
elif socket.has_ipv6 and \
("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts):
family = socket.AF_INET6
addrinfos = None
uris = []
try:
addrinfos = getaddrinfo_validate(
socket.getaddrinfo(getaddrinfo_host, None, family,
socket.SOCK_STREAM))
except socket.error as e:
writemsg_level("!!! getaddrinfo failed for '%s': %s\n" %
(_unicode_decode(hostname), _unicode(e)),
noiselevel=-1,
level=logging.ERROR)
if addrinfos:
AF_INET = socket.AF_INET
AF_INET6 = None
if socket.has_ipv6:
AF_INET6 = socket.AF_INET6
ips_v4 = []
ips_v6 = []
for addrinfo in addrinfos:
if addrinfo[0] == AF_INET:
ips_v4.append("%s" % addrinfo[4][0])
elif AF_INET6 is not None and addrinfo[0] == AF_INET6:
# IPv6 addresses need to be enclosed in square brackets
ips_v6.append("[%s]" % addrinfo[4][0])
random.shuffle(ips_v4)
random.shuffle(ips_v6)
# Give priority to the address family that
# getaddrinfo() returned first.
if AF_INET6 is not None and addrinfos and \
addrinfos[0][0] == AF_INET6:
ips = ips_v6 + ips_v4
else:
ips = ips_v4 + ips_v6
for ip in ips:
uris.append(
syncuri.replace(
"//" + user_name + hostname + port + "/",
"//" + user_name + ip + port + "/", 1))
if not uris:
# With some configurations we need to use the plain hostname
# rather than try to resolve the ip addresses (bug #340817).
uris.append(syncuri)
# reverse, for use with pop()
uris.reverse()
uris_orig = uris[:]
effective_maxretries = maxretries
if effective_maxretries < 0:
effective_maxretries = len(uris) - 1
local_state_unchanged = True
while (1):
if uris:
dosyncuri = uris.pop()
elif maxretries < 0 or retries > maxretries:
writemsg("!!! Exhausted addresses for %s\n" %
_unicode_decode(hostname),
noiselevel=-1)
return (1, False)
else:
uris.extend(uris_orig)
dosyncuri = uris.pop()
if (retries == 0):
if "--ask" in opts:
uq = UserQuery(opts)
if uq.query("Do you want to sync your ebuild repository " + \
"with the mirror at\n" + blue(dosyncuri) + bold("?"),
enter_invalid) == "No":
print()
print("Quitting.")
print()
sys.exit(128 + signal.SIGINT)
self.logger(self.xterm_titles,
">>> Starting rsync with " + dosyncuri)
if "--quiet" not in opts:
print(">>> Starting rsync with " + dosyncuri + "...")
else:
self.logger(self.xterm_titles,
">>> Starting retry %d of %d with %s" % \
(retries, effective_maxretries, dosyncuri))
writemsg_stdout(
"\n\n>>> Starting retry %d of %d with %s\n" % \
(retries, effective_maxretries, dosyncuri), noiselevel=-1)
if dosyncuri.startswith('ssh://'):
dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
dosyncuri, timestamp, opts)
if not unchanged:
local_state_unchanged = False
if is_synced:
break
retries = retries + 1
if maxretries < 0 or retries <= maxretries:
print(">>> Retrying...")
else:
# over retries
# exit loop
exitcode = EXCEEDED_MAX_RETRIES
break
self._process_exitcode(exitcode, dosyncuri, out, maxretries)
if local_state_unchanged:
# The quarantine download_dir is not intended to exist
# in this case, so refer gemato to the normal repository
# location.
download_dir = self.repo.location
else:
download_dir = self.download_dir
# if synced successfully, verify now
if exitcode == 0 and self.verify_metamanifest:
if gemato is None:
writemsg_level(
"!!! Unable to verify: gemato-11.0+ is required\n",
level=logging.ERROR,
noiselevel=-1)
exitcode = 127
else:
try:
# we always verify the Manifest signature, in case
# we had to deal with key revocation case
m = gemato.recursiveloader.ManifestRecursiveLoader(
os.path.join(download_dir, 'Manifest'),
verify_openpgp=True,
openpgp_env=openpgp_env,
max_jobs=self.verify_jobs)
if not m.openpgp_signed:
raise RuntimeError(
'OpenPGP signature not found on Manifest')
ts = m.find_timestamp()
if ts is None:
raise RuntimeError(
'Timestamp not found in Manifest')
if (self.max_age != 0
and (datetime.datetime.utcnow() - ts.ts).days >
self.max_age):
out.quiet = False
out.ewarn(
'Manifest is over %d days old, this is suspicious!'
% (self.max_age, ))
out.ewarn(
'You may want to try using another mirror and/or reporting this one:'
)
out.ewarn(' %s' % (dosyncuri, ))
out.ewarn('')
out.quiet = quiet
out.einfo('Manifest timestamp: %s UTC' % (ts.ts, ))
out.einfo('Valid OpenPGP signature found:')
out.einfo(
'- primary key: %s' %
(m.openpgp_signature.primary_key_fingerprint))
out.einfo('- subkey: %s' %
(m.openpgp_signature.fingerprint))
out.einfo('- timestamp: %s UTC' %
(m.openpgp_signature.timestamp))
# if nothing has changed, skip the actual Manifest
# verification
if not local_state_unchanged:
out.ebegin('Verifying %s' % (download_dir, ))
m.assert_directory_verifies()
out.eend(0)
except GematoException as e:
writemsg_level(
"!!! Manifest verification failed:\n%s\n" % (e, ),
level=logging.ERROR,
noiselevel=-1)
exitcode = 1
verify_failure = True
if exitcode == 0 and not local_state_unchanged:
self.repo_storage.commit_update()
self.repo_storage.garbage_collection()
return (exitcode, updatecache_flg)
finally:
# Don't delete the update if verification failed, in case
# the cause needs to be investigated.
if not verify_failure:
self.repo_storage.abort_update()
if openpgp_env is not None:
openpgp_env.close()
def update(self):
'''Internal update function which performs the transfer'''
opts = self.options.get('emerge_config').opts
self.usersync_uid = self.options.get('usersync_uid', None)
enter_invalid = '--ask-enter-invalid' in opts
out = portage.output.EOutput()
syncuri = self.repo.sync_uri
if self.repo.module_specific_options.get('sync-rsync-vcs-ignore',
'false').lower() == 'true':
vcs_dirs = ()
else:
vcs_dirs = frozenset(VCS_DIRS)
vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
for vcs_dir in vcs_dirs:
writemsg_level(("!!! %s appears to be under revision " + \
"control (contains %s).\n!!! Aborting rsync sync "
"(override with \"sync-rsync-vcs-ignore = true\" in repos.conf).\n") % \
(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
return (1, False)
self.timeout = 180
rsync_opts = []
if self.settings["PORTAGE_RSYNC_OPTS"] == "":
rsync_opts = self._set_rsync_defaults()
else:
rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
self.extra_rsync_opts = list()
if self.repo.module_specific_options.get('sync-rsync-extra-opts'):
self.extra_rsync_opts.extend(
portage.util.shlex_split(
self.repo.module_specific_options['sync-rsync-extra-opts'])
)
# Real local timestamp file.
self.servertimestampfile = os.path.join(self.repo.location, "metadata",
"timestamp.chk")
content = portage.util.grabfile(self.servertimestampfile)
timestamp = 0
if content:
try:
timestamp = time.mktime(
time.strptime(content[0], TIMESTAMP_FORMAT))
except (OverflowError, ValueError):
pass
del content
try:
self.rsync_initial_timeout = \
int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
except ValueError:
self.rsync_initial_timeout = 15
try:
maxretries = int(self.settings["PORTAGE_RSYNC_RETRIES"])
except SystemExit as e:
raise # Needed else can't exit
except:
maxretries = -1 #default number of retries
if syncuri.startswith("file://"):
self.proto = "file"
dosyncuri = syncuri[7:]
is_synced, exitcode, updatecache_flg = self._do_rsync(
dosyncuri, timestamp, opts)
self._process_exitcode(exitcode, dosyncuri, out, 1)
return (exitcode, updatecache_flg)
retries = 0
try:
self.proto, user_name, hostname, port = re.split(
r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
syncuri,
maxsplit=4)[1:5]
except ValueError:
writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri,
noiselevel=-1,
level=logging.ERROR)
return (1, False)
self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
if port is None:
port = ""
if user_name is None:
user_name = ""
if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None:
getaddrinfo_host = hostname
else:
# getaddrinfo needs the brackets stripped
getaddrinfo_host = hostname[1:-1]
updatecache_flg = False
all_rsync_opts = set(self.rsync_opts)
all_rsync_opts.update(self.extra_rsync_opts)
family = socket.AF_UNSPEC
if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
family = socket.AF_INET
elif socket.has_ipv6 and \
("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts):
family = socket.AF_INET6
addrinfos = None
uris = []
try:
addrinfos = getaddrinfo_validate(
socket.getaddrinfo(getaddrinfo_host, None, family,
socket.SOCK_STREAM))
except socket.error as e:
writemsg_level("!!! getaddrinfo failed for '%s': %s\n" %
(_unicode_decode(hostname), _unicode(e)),
noiselevel=-1,
level=logging.ERROR)
if addrinfos:
AF_INET = socket.AF_INET
AF_INET6 = None
if socket.has_ipv6:
AF_INET6 = socket.AF_INET6
ips_v4 = []
ips_v6 = []
for addrinfo in addrinfos:
if addrinfo[0] == AF_INET:
ips_v4.append("%s" % addrinfo[4][0])
elif AF_INET6 is not None and addrinfo[0] == AF_INET6:
# IPv6 addresses need to be enclosed in square brackets
ips_v6.append("[%s]" % addrinfo[4][0])
random.shuffle(ips_v4)
random.shuffle(ips_v6)
# Give priority to the address family that
# getaddrinfo() returned first.
if AF_INET6 is not None and addrinfos and \
addrinfos[0][0] == AF_INET6:
ips = ips_v6 + ips_v4
else:
ips = ips_v4 + ips_v6
for ip in ips:
uris.append(
syncuri.replace("//" + user_name + hostname + port + "/",
"//" + user_name + ip + port + "/", 1))
if not uris:
# With some configurations we need to use the plain hostname
# rather than try to resolve the ip addresses (bug #340817).
uris.append(syncuri)
# reverse, for use with pop()
uris.reverse()
uris_orig = uris[:]
effective_maxretries = maxretries
if effective_maxretries < 0:
effective_maxretries = len(uris) - 1
while (1):
if uris:
dosyncuri = uris.pop()
elif maxretries < 0 or retries > maxretries:
writemsg("!!! Exhausted addresses for %s\n" %
_unicode_decode(hostname),
noiselevel=-1)
return (1, False)
else:
uris.extend(uris_orig)
dosyncuri = uris.pop()
if (retries == 0):
if "--ask" in opts:
uq = UserQuery(opts)
if uq.query("Do you want to sync your Portage tree " + \
"with the mirror at\n" + blue(dosyncuri) + bold("?"),
enter_invalid) == "No":
print()
print("Quitting.")
print()
sys.exit(128 + signal.SIGINT)
self.logger(self.xterm_titles,
">>> Starting rsync with " + dosyncuri)
if "--quiet" not in opts:
print(">>> Starting rsync with " + dosyncuri + "...")
else:
self.logger(self.xterm_titles,
">>> Starting retry %d of %d with %s" % \
(retries, effective_maxretries, dosyncuri))
writemsg_stdout(
"\n\n>>> Starting retry %d of %d with %s\n" % \
(retries, effective_maxretries, dosyncuri), noiselevel=-1)
if dosyncuri.startswith('ssh://'):
dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
is_synced, exitcode, updatecache_flg = self._do_rsync(
dosyncuri, timestamp, opts)
if is_synced:
break
retries = retries + 1
if maxretries < 0 or retries <= maxretries:
print(">>> Retrying...")
else:
# over retries
# exit loop
exitcode = EXCEEDED_MAX_RETRIES
break
self._process_exitcode(exitcode, dosyncuri, out, maxretries)
return (exitcode, updatecache_flg)
def _sync(self):
'''Internal sync function which performs only the sync'''
opts = self.options.get('emerge_config').opts
self.usersync_uid = self.options.get('usersync_uid', None)
enter_invalid = '--ask-enter-invalid' in opts
out = portage.output.EOutput()
syncuri = self.repo.sync_uri
vcs_dirs = frozenset(VCS_DIRS)
vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
for vcs_dir in vcs_dirs:
writemsg_level(("!!! %s appears to be under revision " + \
"control (contains %s).\n!!! Aborting rsync sync.\n") % \
(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
return (1, False)
self.timeout=180
rsync_opts = []
if self.settings["PORTAGE_RSYNC_OPTS"] == "":
rsync_opts = self._set_rsync_defaults()
else:
rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
self.extra_rsync_opts = portage.util.shlex_split(
self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
# Real local timestamp file.
self.servertimestampfile = os.path.join(
self.repo.location, "metadata", "timestamp.chk")
content = portage.util.grabfile(self.servertimestampfile)
timestamp = 0
if content:
try:
timestamp = time.mktime(time.strptime(content[0],
TIMESTAMP_FORMAT))
except (OverflowError, ValueError):
pass
del content
try:
self.rsync_initial_timeout = \
int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
except ValueError:
self.rsync_initial_timeout = 15
try:
maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
except SystemExit as e:
raise # Needed else can't exit
except:
maxretries = -1 #default number of retries
if syncuri.startswith("file://"):
self.proto = "file"
dosyncuri = syncuri[6:]
is_synced, exitcode = self._do_rsync(
dosyncuri, timestamp, opts)
self._process_exitcode(exitcode, dosyncuri, out, 1)
return (exitcode, exitcode == os.EX_OK)
retries=0
try:
self.proto, user_name, hostname, port = re.split(
r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
syncuri, maxsplit=4)[1:5]
except ValueError:
writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri,
noiselevel=-1, level=logging.ERROR)
return (1, False)
self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
if port is None:
port=""
if user_name is None:
user_name=""
if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None:
getaddrinfo_host = hostname
else:
# getaddrinfo needs the brackets stripped
getaddrinfo_host = hostname[1:-1]
updatecache_flg=True
all_rsync_opts = set(self.rsync_opts)
all_rsync_opts.update(self.extra_rsync_opts)
family = socket.AF_UNSPEC
if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
family = socket.AF_INET
elif socket.has_ipv6 and \
("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts):
family = socket.AF_INET6
addrinfos = None
uris = []
try:
addrinfos = getaddrinfo_validate(
socket.getaddrinfo(getaddrinfo_host, None,
family, socket.SOCK_STREAM))
except socket.error as e:
writemsg_level(
"!!! getaddrinfo failed for '%s': %s\n" % (hostname, e),
noiselevel=-1, level=logging.ERROR)
if addrinfos:
AF_INET = socket.AF_INET
AF_INET6 = None
if socket.has_ipv6:
AF_INET6 = socket.AF_INET6
ips_v4 = []
ips_v6 = []
for addrinfo in addrinfos:
if addrinfo[0] == AF_INET:
ips_v4.append("%s" % addrinfo[4][0])
elif AF_INET6 is not None and addrinfo[0] == AF_INET6:
# IPv6 addresses need to be enclosed in square brackets
ips_v6.append("[%s]" % addrinfo[4][0])
random.shuffle(ips_v4)
random.shuffle(ips_v6)
# Give priority to the address family that
# getaddrinfo() returned first.
if AF_INET6 is not None and addrinfos and \
addrinfos[0][0] == AF_INET6:
ips = ips_v6 + ips_v4
else:
ips = ips_v4 + ips_v6
for ip in ips:
uris.append(syncuri.replace(
"//" + user_name + hostname + port + "/",
"//" + user_name + ip + port + "/", 1))
if not uris:
# With some configurations we need to use the plain hostname
# rather than try to resolve the ip addresses (bug #340817).
uris.append(syncuri)
# reverse, for use with pop()
uris.reverse()
effective_maxretries = maxretries
if effective_maxretries < 0:
effective_maxretries = len(uris) - 1
while (1):
if uris:
dosyncuri = uris.pop()
else:
writemsg("!!! Exhausted addresses for %s\n" % \
hostname, noiselevel=-1)
return (1, False)
if (retries==0):
if "--ask" in opts:
uq = UserQuery(opts)
if uq.query("Do you want to sync your Portage tree " + \
"with the mirror at\n" + blue(dosyncuri) + bold("?"),
enter_invalid) == "No":
print()
print("Quitting.")
print()
sys.exit(128 + signal.SIGINT)
self.logger(self.xterm_titles,
">>> Starting rsync with " + dosyncuri)
if "--quiet" not in opts:
print(">>> Starting rsync with "+dosyncuri+"...")
else:
self.logger(self.xterm_titles,
">>> Starting retry %d of %d with %s" % \
(retries, effective_maxretries, dosyncuri))
writemsg_stdout(
"\n\n>>> Starting retry %d of %d with %s\n" % \
(retries, effective_maxretries, dosyncuri), noiselevel=-1)
if dosyncuri.startswith('ssh://'):
dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
if is_synced:
break
retries=retries+1
if maxretries < 0 or retries <= maxretries:
print(">>> Retrying...")
else:
# over retries
# exit loop
updatecache_flg=False
exitcode = EXCEEDED_MAX_RETRIES
break
self._process_exitcode(exitcode, dosyncuri, out, maxretries)
return (exitcode, updatecache_flg)
def update(self):
'''Internal update function which performs the transfer'''
opts = self.options.get('emerge_config').opts
self.usersync_uid = self.options.get('usersync_uid', None)
enter_invalid = '--ask-enter-invalid' in opts
quiet = '--quiet' in opts
out = portage.output.EOutput(quiet=quiet)
syncuri = self.repo.sync_uri
if self.repo.module_specific_options.get(
'sync-rsync-vcs-ignore', 'false').lower() == 'true':
vcs_dirs = ()
else:
vcs_dirs = frozenset(VCS_DIRS)
vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
for vcs_dir in vcs_dirs:
writemsg_level(("!!! %s appears to be under revision " + \
"control (contains %s).\n!!! Aborting rsync sync "
"(override with \"sync-rsync-vcs-ignore = true\" in repos.conf).\n") % \
(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
return (1, False)
self.timeout=180
rsync_opts = []
if self.settings["PORTAGE_RSYNC_OPTS"] == "":
rsync_opts = self._set_rsync_defaults()
else:
rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
self.extra_rsync_opts = list()
if self.repo.module_specific_options.get('sync-rsync-extra-opts'):
self.extra_rsync_opts.extend(portage.util.shlex_split(
self.repo.module_specific_options['sync-rsync-extra-opts']))
exitcode = 0
verify_failure = False
# Process GLEP74 verification options.
# Default verification to 'no'; it's enabled for ::gentoo
# via default repos.conf though.
self.verify_metamanifest = (
self.repo.module_specific_options.get(
'sync-rsync-verify-metamanifest', 'no') in ('yes', 'true'))
# Support overriding job count.
self.verify_jobs = self.repo.module_specific_options.get(
'sync-rsync-verify-jobs', None)
if self.verify_jobs is not None:
try:
self.verify_jobs = int(self.verify_jobs)
if self.verify_jobs < 0:
raise ValueError(self.verify_jobs)
except ValueError:
writemsg_level("!!! sync-rsync-verify-jobs not a positive integer: %s\n" % (self.verify_jobs,),
level=logging.WARNING, noiselevel=-1)
self.verify_jobs = None
else:
if self.verify_jobs == 0:
# Use the apparent number of processors if gemato
# supports it.
self.verify_jobs = None
# Support overriding max age.
self.max_age = self.repo.module_specific_options.get(
'sync-rsync-verify-max-age', '')
if self.max_age:
try:
self.max_age = int(self.max_age)
if self.max_age < 0:
raise ValueError(self.max_age)
except ValueError:
writemsg_level("!!! sync-rsync-max-age must be a non-negative integer: %s\n" % (self.max_age,),
level=logging.WARNING, noiselevel=-1)
self.max_age = 0
else:
self.max_age = 0
openpgp_env = None
if self.verify_metamanifest and gemato is not None:
# Use isolated environment if key is specified,
# system environment otherwise
if self.repo.sync_openpgp_key_path is not None:
openpgp_env = gemato.openpgp.OpenPGPEnvironment()
else:
openpgp_env = gemato.openpgp.OpenPGPSystemEnvironment()
try:
# Load and update the keyring early. If it fails, then verification
# will not be performed and the user will have to fix it and try again,
# so we may as well bail out before actual rsync happens.
if openpgp_env is not None and self.repo.sync_openpgp_key_path is not None:
try:
out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
openpgp_env.import_key(f)
self._refresh_keys(openpgp_env)
except (GematoException, asyncio.TimeoutError) as e:
writemsg_level("!!! Manifest verification impossible due to keyring problem:\n%s\n"
% (e,),
level=logging.ERROR, noiselevel=-1)
return (1, False)
# Real local timestamp file.
self.servertimestampfile = os.path.join(
self.repo.location, "metadata", "timestamp.chk")
content = portage.util.grabfile(self.servertimestampfile)
timestamp = 0
if content:
try:
timestamp = time.mktime(time.strptime(content[0],
TIMESTAMP_FORMAT))
except (OverflowError, ValueError):
pass
del content
try:
self.rsync_initial_timeout = \
int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
except ValueError:
self.rsync_initial_timeout = 15
try:
maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
except SystemExit as e:
raise # Needed else can't exit
except:
maxretries = -1 #default number of retries
if syncuri.startswith("file://"):
self.proto = "file"
dosyncuri = syncuri[7:]
unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
dosyncuri, timestamp, opts)
self._process_exitcode(exitcode, dosyncuri, out, 1)
if exitcode == 0:
if unchanged:
self.repo_storage.abort_update()
else:
self.repo_storage.commit_update()
self.repo_storage.garbage_collection()
return (exitcode, updatecache_flg)
retries=0
try:
self.proto, user_name, hostname, port = re.split(
r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
syncuri, maxsplit=4)[1:5]
except ValueError:
writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri,
noiselevel=-1, level=logging.ERROR)
return (1, False)
self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
if port is None:
port=""
if user_name is None:
user_name=""
if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None:
getaddrinfo_host = hostname
else:
# getaddrinfo needs the brackets stripped
getaddrinfo_host = hostname[1:-1]
updatecache_flg = False
all_rsync_opts = set(self.rsync_opts)
all_rsync_opts.update(self.extra_rsync_opts)
family = socket.AF_UNSPEC
if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
family = socket.AF_INET
elif socket.has_ipv6 and \
("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts):
family = socket.AF_INET6
addrinfos = None
uris = []
try:
addrinfos = getaddrinfo_validate(
socket.getaddrinfo(getaddrinfo_host, None,
family, socket.SOCK_STREAM))
except socket.error as e:
writemsg_level(
"!!! getaddrinfo failed for '%s': %s\n"
% (_unicode_decode(hostname), _unicode(e)),
noiselevel=-1, level=logging.ERROR)
if addrinfos:
AF_INET = socket.AF_INET
AF_INET6 = None
if socket.has_ipv6:
AF_INET6 = socket.AF_INET6
ips_v4 = []
ips_v6 = []
for addrinfo in addrinfos:
if addrinfo[0] == AF_INET:
ips_v4.append("%s" % addrinfo[4][0])
elif AF_INET6 is not None and addrinfo[0] == AF_INET6:
# IPv6 addresses need to be enclosed in square brackets
ips_v6.append("[%s]" % addrinfo[4][0])
random.shuffle(ips_v4)
random.shuffle(ips_v6)
# Give priority to the address family that
# getaddrinfo() returned first.
if AF_INET6 is not None and addrinfos and \
addrinfos[0][0] == AF_INET6:
ips = ips_v6 + ips_v4
else:
ips = ips_v4 + ips_v6
for ip in ips:
uris.append(syncuri.replace(
"//" + user_name + hostname + port + "/",
"//" + user_name + ip + port + "/", 1))
if not uris:
# With some configurations we need to use the plain hostname
# rather than try to resolve the ip addresses (bug #340817).
uris.append(syncuri)
# reverse, for use with pop()
uris.reverse()
uris_orig = uris[:]
effective_maxretries = maxretries
if effective_maxretries < 0:
effective_maxretries = len(uris) - 1
local_state_unchanged = True
while (1):
if uris:
dosyncuri = uris.pop()
elif maxretries < 0 or retries > maxretries:
writemsg("!!! Exhausted addresses for %s\n"
% _unicode_decode(hostname), noiselevel=-1)
return (1, False)
else:
uris.extend(uris_orig)
dosyncuri = uris.pop()
if (retries==0):
if "--ask" in opts:
uq = UserQuery(opts)
if uq.query("Do you want to sync your ebuild repository " + \
"with the mirror at\n" + blue(dosyncuri) + bold("?"),
enter_invalid) == "No":
print()
print("Quitting.")
print()
sys.exit(128 + signal.SIGINT)
self.logger(self.xterm_titles,
">>> Starting rsync with " + dosyncuri)
if "--quiet" not in opts:
print(">>> Starting rsync with "+dosyncuri+"...")
else:
self.logger(self.xterm_titles,
">>> Starting retry %d of %d with %s" % \
(retries, effective_maxretries, dosyncuri))
writemsg_stdout(
"\n\n>>> Starting retry %d of %d with %s\n" % \
(retries, effective_maxretries, dosyncuri), noiselevel=-1)
if dosyncuri.startswith('ssh://'):
dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync(
dosyncuri, timestamp, opts)
if not unchanged:
local_state_unchanged = False
if is_synced:
break
retries=retries+1
if maxretries < 0 or retries <= maxretries:
print(">>> Retrying...")
else:
# over retries
# exit loop
exitcode = EXCEEDED_MAX_RETRIES
break
self._process_exitcode(exitcode, dosyncuri, out, maxretries)
if local_state_unchanged:
# The quarantine download_dir is not intended to exist
# in this case, so refer gemato to the normal repository
# location.
download_dir = self.repo.location
else:
download_dir = self.download_dir
# if synced successfully, verify now
if exitcode == 0 and self.verify_metamanifest:
if gemato is None:
writemsg_level("!!! Unable to verify: gemato-11.0+ is required\n",
level=logging.ERROR, noiselevel=-1)
exitcode = 127
else:
try:
# we always verify the Manifest signature, in case
# we had to deal with key revocation case
m = gemato.recursiveloader.ManifestRecursiveLoader(
os.path.join(download_dir, 'Manifest'),
verify_openpgp=True,
openpgp_env=openpgp_env,
max_jobs=self.verify_jobs)
if not m.openpgp_signed:
raise RuntimeError('OpenPGP signature not found on Manifest')
ts = m.find_timestamp()
if ts is None:
raise RuntimeError('Timestamp not found in Manifest')
if (self.max_age != 0 and
(datetime.datetime.utcnow() - ts.ts).days > self.max_age):
out.quiet = False
out.ewarn('Manifest is over %d days old, this is suspicious!' % (self.max_age,))
out.ewarn('You may want to try using another mirror and/or reporting this one:')
out.ewarn(' %s' % (dosyncuri,))
out.ewarn('')
out.quiet = quiet
out.einfo('Manifest timestamp: %s UTC' % (ts.ts,))
out.einfo('Valid OpenPGP signature found:')
out.einfo('- primary key: %s' % (
m.openpgp_signature.primary_key_fingerprint))
out.einfo('- subkey: %s' % (
m.openpgp_signature.fingerprint))
out.einfo('- timestamp: %s UTC' % (
m.openpgp_signature.timestamp))
# if nothing has changed, skip the actual Manifest
# verification
if not local_state_unchanged:
out.ebegin('Verifying %s' % (download_dir,))
m.assert_directory_verifies()
out.eend(0)
except GematoException as e:
writemsg_level("!!! Manifest verification failed:\n%s\n"
% (e,),
level=logging.ERROR, noiselevel=-1)
exitcode = 1
verify_failure = True
if exitcode == 0 and not local_state_unchanged:
self.repo_storage.commit_update()
self.repo_storage.garbage_collection()
return (exitcode, updatecache_flg)
finally:
# Don't delete the update if verification failed, in case
# the cause needs to be investigated.
if not verify_failure:
self.repo_storage.abort_update()
if openpgp_env is not None:
openpgp_env.close()
