def _status(self, entropy_server):
"""
Actual Eit lock|unlock --status code. Just show repo status.
"""
repositories = entropy_server.repositories()
if self._repository_id not in repositories:
entropy_server.output(
"%s: %s" % (
_("Invalid Repository"),
self._repository_id),
level="error",
importance=1)
return 1
if not self._quiet:
entropy_server.output(
"%s:" % (darkgreen(_("Mirrors status")),),
header=brown(" * "))
dbstatus = entropy_server.Mirrors.mirrors_status(
self._repository_id)
for uri, server_lock, client_lock in dbstatus:
host = EntropyTransceiver.get_uri_name(uri)
if not self._quiet:
entropy_server.output(
"[%s]" % (purple(host),),
header=darkgreen(" @@ "))
if server_lock:
lock_str = darkred(_("Locked"))
quiet_lock_str = "locked"
else:
lock_str = darkgreen(_("Unlocked"))
quiet_lock_str = "unlocked"
if self._quiet:
entropy_server.output(
"%s server %s" % (host, quiet_lock_str),
level="generic")
else:
entropy_server.output(
"%s: %s" % (blue(_("server")), lock_str),
header=brown(" # "))
if client_lock:
lock_str = darkred(_("Locked"))
quiet_lock_str = "locked"
else:
lock_str = darkgreen(_("Unlocked"))
quiet_lock_str = "unlocked"
if self._quiet:
entropy_server.output(
"%s client %s" % (host, quiet_lock_str),
level="generic")
else:
entropy_server.output(
"%s: %s" % (blue(_("client")), lock_str),
header=brown(" # "))
return 0
def _status(self, entropy_server):
"""
Actual Eit lock|unlock --status code. Just show repo status.
"""
repositories = entropy_server.repositories()
if self._repository_id not in repositories:
entropy_server.output(
"%s: %s" % (_("Invalid Repository"), self._repository_id),
level="error",
importance=1)
return 1
if not self._quiet:
entropy_server.output("%s:" % (darkgreen(_("Mirrors status")), ),
header=brown(" * "))
dbstatus = entropy_server.Mirrors.mirrors_status(self._repository_id)
for uri, server_lock, client_lock in dbstatus:
host = EntropyTransceiver.get_uri_name(uri)
if not self._quiet:
entropy_server.output("[%s]" % (purple(host), ),
header=darkgreen(" @@ "))
if server_lock:
lock_str = darkred(_("Locked"))
quiet_lock_str = "locked"
else:
lock_str = darkgreen(_("Unlocked"))
quiet_lock_str = "unlocked"
if self._quiet:
entropy_server.output("%s server %s" % (host, quiet_lock_str),
level="generic")
else:
entropy_server.output("%s: %s" % (blue(_("server")), lock_str),
header=brown(" # "))
if client_lock:
lock_str = darkred(_("Locked"))
quiet_lock_str = "locked"
else:
lock_str = darkgreen(_("Unlocked"))
quiet_lock_str = "unlocked"
if self._quiet:
entropy_server.output("%s client %s" % (host, quiet_lock_str),
level="generic")
else:
entropy_server.output("%s: %s" % (blue(_("client")), lock_str),
header=brown(" # "))
return 0
def go(self):
broken_uris = set()
fine_uris = set()
errors = False
action = 'push'
if self.download:
action = 'pull'
elif self.remove:
action = 'remove'
for uri in self.uris:
crippled_uri = EntropyTransceiver.get_uri_name(uri)
self._entropy.output(
"[%s|%s] %s..." % (
blue(crippled_uri),
brown(action),
blue(_("connecting to mirror")),
),
importance = 0,
level = "info",
header = blue(" @@ ")
)
self._entropy.output(
"[%s|%s] %s %s..." % (
blue(crippled_uri),
brown(action),
blue(_("setting directory to")),
darkgreen(self.txc_basedir),
),
importance = 0,
level = "info",
header = blue(" @@ ")
)
fail, fine, broken = self._transceive(uri)
fine_uris |= fine
broken_uris |= broken
if fail:
errors = True
return errors, fine_uris, broken_uris
def go(self):
broken_uris = set()
fine_uris = set()
errors = False
action = 'push'
if self.download:
action = 'pull'
elif self.remove:
action = 'remove'
for uri in self.uris:
crippled_uri = EntropyTransceiver.get_uri_name(uri)
self._entropy.output("[%s|%s] %s..." % (
blue(crippled_uri),
brown(action),
blue(_("connecting to mirror")),
),
importance=0,
level="info",
header=blue(" @@ "))
self._entropy.output("[%s|%s] %s %s..." % (
blue(crippled_uri),
brown(action),
blue(_("setting directory to")),
darkgreen(self.txc_basedir),
),
importance=0,
level="info",
header=blue(" @@ "))
fail, fine, broken = self._transceive(uri)
fine_uris |= fine
broken_uris |= broken
if fail:
errors = True
return errors, fine_uris, broken_uris
def print_repository_status(entropy_server, repository_id):
remote_db_status = entropy_server.Mirrors.remote_repository_status(
repository_id)
entropy_server.output("%s:" %
(brown(_("Entropy Repository Status")), ),
importance=1,
header=darkgreen(" * "))
for url, revision in remote_db_status.items():
host = EntropyTransceiver.get_uri_name(url)
entropy_server.output("%s: %s" %
(darkgreen(_("Host")), bold(host)),
header=" ")
entropy_server.output("%s: %s" %
(purple(_("Remote")), blue(str(revision))),
header=" ")
local_revision = entropy_server.local_repository_revision(
repository_id)
entropy_server.output("%s: %s" %
(brown(_("Local")), teal(str(local_revision))),
header=" ")
def print_repository_status(entropy_server, repository_id):
remote_db_status = entropy_server.Mirrors.remote_repository_status(
repository_id)
entropy_server.output(
"%s:" % (brown(_("Entropy Repository Status")),),
importance=1,
header=darkgreen(" * ")
)
for url, revision in remote_db_status.items():
host = EntropyTransceiver.get_uri_name(url)
entropy_server.output(
"%s: %s" % (darkgreen(_("Host")), bold(host)),
header=" ")
entropy_server.output(
"%s: %s" % (purple(_("Remote")), blue(str(revision))),
header=" ")
local_revision = entropy_server.local_repository_revision(
repository_id)
entropy_server.output(
"%s: %s" % (brown(_("Local")), teal(str(local_revision))),
header=" ")
def handler_verify_upload(self, local_filepath, uri, counter, maxcount,
tries, remote_md5 = None):
crippled_uri = EntropyTransceiver.get_uri_name(uri)
self._entropy.output(
"[%s|#%s|(%s/%s)] %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
darkgreen(_("verifying upload (if supported)")),
blue(os.path.basename(local_filepath)),
),
importance = 0,
level = "info",
header = red(" @@ "),
back = True
)
valid_remote_md5 = True
# if remote server supports MD5 commands, remote_md5 is filled
if const_isstring(remote_md5):
valid_md5 = is_valid_md5(remote_md5)
ckres = False
if valid_md5: # seems valid
ckres = compare_md5(local_filepath, remote_md5)
if ckres:
self._entropy.output(
"[%s|#%s|(%s/%s)] %s: %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(_("digest verification")),
os.path.basename(local_filepath),
darkgreen(_("so far, so good!")),
),
importance = 0,
level = "info",
header = red(" @@ ")
)
return True
# ouch!
elif not valid_md5:
# mmmh... malformed md5, try with handlers
self._entropy.output(
"[%s|#%s|(%s/%s)] %s: %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(_("digest verification")),
os.path.basename(local_filepath),
bold(_("malformed md5 provided to function")),
),
importance = 0,
level = "warning",
header = brown(" @@ ")
)
else: # it's really bad!
self._entropy.output(
"[%s|#%s|(%s/%s)] %s: %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(_("digest verification")),
os.path.basename(local_filepath),
bold(_("remote md5 is invalid")),
),
importance = 0,
level = "warning",
header = brown(" @@ ")
)
valid_remote_md5 = False
return valid_remote_md5 # always valid
def _transceive(self, uri):
fine = set()
broken = set()
fail = False
crippled_uri = EntropyTransceiver.get_uri_name(uri)
action = 'push'
if self.download:
action = 'pull'
elif self.remove:
action = 'remove'
try:
txc = EntropyTransceiver(uri)
if const_isnumber(self.speed_limit):
txc.set_speed_limit(self.speed_limit)
txc.set_output_interface(self._entropy)
except TransceiverConnectionError:
print_traceback()
return True, fine, broken # issues
maxcount = len(self.myfiles)
counter = 0
with txc as handler:
for mypath in self.myfiles:
base_dir = self.txc_basedir
if isinstance(mypath, tuple):
if len(mypath) < 2:
continue
base_dir, mypath = mypath
if not handler.is_dir(base_dir):
handler.makedirs(base_dir)
mypath_fn = os.path.basename(mypath)
remote_path = os.path.join(base_dir, mypath_fn)
syncer = handler.upload
myargs = (mypath, remote_path)
if self.download:
syncer = handler.download
local_path = os.path.join(self.local_basedir, mypath_fn)
myargs = (remote_path, local_path)
elif self.remove:
syncer = handler.delete
myargs = (remote_path,)
fallback_syncer, fallback_args = None, None
# upload -> remote copy herustic support
# if a package file might have been already uploaded
# to remote mirror, try to look in other repositories'
# package directories if a file, with the same md5 and name
# is already available. In this case, use remote copy instead
# of upload to save bandwidth.
if self._copy_herustic and (syncer == handler.upload):
# copy herustic support enabled
# we are uploading
new_syncer, new_args = self._copy_herustic_support(
handler, mypath, base_dir, remote_path)
if new_syncer is not None:
fallback_syncer, fallback_args = syncer, myargs
syncer, myargs = new_syncer, new_args
action = "copy"
counter += 1
tries = 0
done = False
lastrc = None
while tries < 5:
tries += 1
self._entropy.output(
"[%s|#%s|(%s/%s)] %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(action),
red(os.path.basename(mypath)),
),
importance = 0,
level = "info",
header = red(" @@ ")
)
rc = syncer(*myargs)
if (not rc) and (fallback_syncer is not None):
# if we have a fallback syncer, try it first
# before giving up.
rc = fallback_syncer(*myargs)
if rc and not (self.download or self.remove):
remote_md5 = handler.get_md5(remote_path)
rc = self.handler_verify_upload(mypath, uri,
counter, maxcount, tries, remote_md5 = remote_md5)
if rc:
self._entropy.output(
"[%s|#%s|(%s/%s)] %s %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(action),
_("successful"),
red(os.path.basename(mypath)),
),
importance = 0,
level = "info",
header = darkgreen(" @@ ")
)
done = True
fine.add(uri)
break
else:
self._entropy.output(
"[%s|#%s|(%s/%s)] %s %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(action),
brown(_("failed, retrying")),
red(os.path.basename(mypath)),
),
importance = 0,
level = "warning",
header = brown(" @@ ")
)
lastrc = rc
continue
if not done:
self._entropy.output(
"[%s|(%s/%s)] %s %s: %s - %s: %s" % (
blue(crippled_uri),
blue(str(counter)),
bold(str(maxcount)),
blue(action),
darkred("failed, giving up"),
red(os.path.basename(mypath)),
_("error"),
lastrc,
),
importance = 1,
level = "error",
header = darkred(" !!! ")
)
if mypath not in self.critical_files:
self._entropy.output(
"[%s|(%s/%s)] %s: %s, %s..." % (
blue(crippled_uri),
blue(str(counter)),
bold(str(maxcount)),
blue(_("not critical")),
os.path.basename(mypath),
blue(_("continuing")),
),
importance = 1,
level = "warning",
header = brown(" @@ ")
)
continue
fail = True
broken.add((uri, lastrc))
# next mirror
break
return fail, fine, broken
def handler_verify_upload(self,
local_filepath,
uri,
counter,
maxcount,
tries,
remote_md5=None):
crippled_uri = EntropyTransceiver.get_uri_name(uri)
self._entropy.output("[%s|#%s|(%s/%s)] %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
darkgreen(_("verifying upload (if supported)")),
blue(os.path.basename(local_filepath)),
),
importance=0,
level="info",
header=red(" @@ "),
back=True)
valid_remote_md5 = True
# if remote server supports MD5 commands, remote_md5 is filled
if const_isstring(remote_md5):
valid_md5 = is_valid_md5(remote_md5)
ckres = False
if valid_md5: # seems valid
ckres = compare_md5(local_filepath, remote_md5)
if ckres:
self._entropy.output("[%s|#%s|(%s/%s)] %s: %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(_("digest verification")),
os.path.basename(local_filepath),
darkgreen(_("so far, so good!")),
),
importance=0,
level="info",
header=red(" @@ "))
return True
# ouch!
elif not valid_md5:
# mmmh... malformed md5, try with handlers
self._entropy.output("[%s|#%s|(%s/%s)] %s: %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(_("digest verification")),
os.path.basename(local_filepath),
bold(_("malformed md5 provided to function")),
),
importance=0,
level="warning",
header=brown(" @@ "))
else: # it's really bad!
self._entropy.output("[%s|#%s|(%s/%s)] %s: %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(_("digest verification")),
os.path.basename(local_filepath),
bold(_("remote md5 is invalid")),
),
importance=0,
level="warning",
header=brown(" @@ "))
valid_remote_md5 = False
return valid_remote_md5 # always valid
def _transceive(self, uri):
fine = set()
broken = set()
fail = False
crippled_uri = EntropyTransceiver.get_uri_name(uri)
action = 'push'
if self.download:
action = 'pull'
elif self.remove:
action = 'remove'
try:
txc = EntropyTransceiver(uri)
if const_isnumber(self.speed_limit):
txc.set_speed_limit(self.speed_limit)
txc.set_output_interface(self._entropy)
except TransceiverConnectionError:
print_traceback()
return True, fine, broken # issues
maxcount = len(self.myfiles)
counter = 0
with txc as handler:
for mypath in self.myfiles:
base_dir = self.txc_basedir
if isinstance(mypath, tuple):
if len(mypath) < 2:
continue
base_dir, mypath = mypath
if not handler.is_dir(base_dir):
handler.makedirs(base_dir)
mypath_fn = os.path.basename(mypath)
remote_path = os.path.join(base_dir, mypath_fn)
syncer = handler.upload
myargs = (mypath, remote_path)
if self.download:
syncer = handler.download
local_path = os.path.join(self.local_basedir, mypath_fn)
myargs = (remote_path, local_path)
elif self.remove:
syncer = handler.delete
myargs = (remote_path, )
fallback_syncer, fallback_args = None, None
# upload -> remote copy herustic support
# if a package file might have been already uploaded
# to remote mirror, try to look in other repositories'
# package directories if a file, with the same md5 and name
# is already available. In this case, use remote copy instead
# of upload to save bandwidth.
if self._copy_herustic and (syncer == handler.upload):
# copy herustic support enabled
# we are uploading
new_syncer, new_args = self._copy_herustic_support(
handler, mypath, base_dir, remote_path)
if new_syncer is not None:
fallback_syncer, fallback_args = syncer, myargs
syncer, myargs = new_syncer, new_args
action = "copy"
counter += 1
tries = 0
done = False
lastrc = None
while tries < 5:
tries += 1
self._entropy.output("[%s|#%s|(%s/%s)] %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(action),
red(os.path.basename(mypath)),
),
importance=0,
level="info",
header=red(" @@ "))
rc = syncer(*myargs)
if (not rc) and (fallback_syncer is not None):
# if we have a fallback syncer, try it first
# before giving up.
rc = fallback_syncer(*myargs)
if rc and not (self.download or self.remove):
remote_md5 = handler.get_md5(remote_path)
rc = self.handler_verify_upload(mypath,
uri,
counter,
maxcount,
tries,
remote_md5=remote_md5)
if rc:
self._entropy.output("[%s|#%s|(%s/%s)] %s %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(action),
_("successful"),
red(os.path.basename(mypath)),
),
importance=0,
level="info",
header=darkgreen(" @@ "))
done = True
fine.add(uri)
break
else:
self._entropy.output("[%s|#%s|(%s/%s)] %s %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(action),
brown(_("failed, retrying")),
red(os.path.basename(mypath)),
),
importance=0,
level="warning",
header=brown(" @@ "))
lastrc = rc
continue
if not done:
self._entropy.output("[%s|(%s/%s)] %s %s: %s - %s: %s" % (
blue(crippled_uri),
blue(str(counter)),
bold(str(maxcount)),
blue(action),
darkred("failed, giving up"),
red(os.path.basename(mypath)),
_("error"),
lastrc,
),
importance=1,
level="error",
header=darkred(" !!! "))
if mypath not in self.critical_files:
self._entropy.output("[%s|(%s/%s)] %s: %s, %s..." % (
blue(crippled_uri),
blue(str(counter)),
bold(str(maxcount)),
blue(_("not critical")),
os.path.basename(mypath),
blue(_("continuing")),
),
importance=1,
level="warning",
header=brown(" @@ "))
continue
fail = True
broken.add((uri, lastrc))
# next mirror
break
return fail, fine, broken
