def _status(self, entropy_server): """ Actual Eit lock|unlock --status code. Just show repo status. """ repositories = entropy_server.repositories() if self._repository_id not in repositories: entropy_server.output( "%s: %s" % ( _("Invalid Repository"), self._repository_id), level="error", importance=1) return 1 if not self._quiet: entropy_server.output( "%s:" % (darkgreen(_("Mirrors status")),), header=brown(" * ")) dbstatus = entropy_server.Mirrors.mirrors_status( self._repository_id) for uri, server_lock, client_lock in dbstatus: host = EntropyTransceiver.get_uri_name(uri) if not self._quiet: entropy_server.output( "[%s]" % (purple(host),), header=darkgreen(" @@ ")) if server_lock: lock_str = darkred(_("Locked")) quiet_lock_str = "locked" else: lock_str = darkgreen(_("Unlocked")) quiet_lock_str = "unlocked" if self._quiet: entropy_server.output( "%s server %s" % (host, quiet_lock_str), level="generic") else: entropy_server.output( "%s: %s" % (blue(_("server")), lock_str), header=brown(" # ")) if client_lock: lock_str = darkred(_("Locked")) quiet_lock_str = "locked" else: lock_str = darkgreen(_("Unlocked")) quiet_lock_str = "unlocked" if self._quiet: entropy_server.output( "%s client %s" % (host, quiet_lock_str), level="generic") else: entropy_server.output( "%s: %s" % (blue(_("client")), lock_str), header=brown(" # ")) return 0
def _status(self, entropy_server): """ Actual Eit lock|unlock --status code. Just show repo status. """ repositories = entropy_server.repositories() if self._repository_id not in repositories: entropy_server.output( "%s: %s" % (_("Invalid Repository"), self._repository_id), level="error", importance=1) return 1 if not self._quiet: entropy_server.output("%s:" % (darkgreen(_("Mirrors status")), ), header=brown(" * ")) dbstatus = entropy_server.Mirrors.mirrors_status(self._repository_id) for uri, server_lock, client_lock in dbstatus: host = EntropyTransceiver.get_uri_name(uri) if not self._quiet: entropy_server.output("[%s]" % (purple(host), ), header=darkgreen(" @@ ")) if server_lock: lock_str = darkred(_("Locked")) quiet_lock_str = "locked" else: lock_str = darkgreen(_("Unlocked")) quiet_lock_str = "unlocked" if self._quiet: entropy_server.output("%s server %s" % (host, quiet_lock_str), level="generic") else: entropy_server.output("%s: %s" % (blue(_("server")), lock_str), header=brown(" # ")) if client_lock: lock_str = darkred(_("Locked")) quiet_lock_str = "locked" else: lock_str = darkgreen(_("Unlocked")) quiet_lock_str = "unlocked" if self._quiet: entropy_server.output("%s client %s" % (host, quiet_lock_str), level="generic") else: entropy_server.output("%s: %s" % (blue(_("client")), lock_str), header=brown(" # ")) return 0
def go(self): broken_uris = set() fine_uris = set() errors = False action = 'push' if self.download: action = 'pull' elif self.remove: action = 'remove' for uri in self.uris: crippled_uri = EntropyTransceiver.get_uri_name(uri) self._entropy.output( "[%s|%s] %s..." % ( blue(crippled_uri), brown(action), blue(_("connecting to mirror")), ), importance = 0, level = "info", header = blue(" @@ ") ) self._entropy.output( "[%s|%s] %s %s..." % ( blue(crippled_uri), brown(action), blue(_("setting directory to")), darkgreen(self.txc_basedir), ), importance = 0, level = "info", header = blue(" @@ ") ) fail, fine, broken = self._transceive(uri) fine_uris |= fine broken_uris |= broken if fail: errors = True return errors, fine_uris, broken_uris
def go(self): broken_uris = set() fine_uris = set() errors = False action = 'push' if self.download: action = 'pull' elif self.remove: action = 'remove' for uri in self.uris: crippled_uri = EntropyTransceiver.get_uri_name(uri) self._entropy.output("[%s|%s] %s..." % ( blue(crippled_uri), brown(action), blue(_("connecting to mirror")), ), importance=0, level="info", header=blue(" @@ ")) self._entropy.output("[%s|%s] %s %s..." % ( blue(crippled_uri), brown(action), blue(_("setting directory to")), darkgreen(self.txc_basedir), ), importance=0, level="info", header=blue(" @@ ")) fail, fine, broken = self._transceive(uri) fine_uris |= fine broken_uris |= broken if fail: errors = True return errors, fine_uris, broken_uris
def print_repository_status(entropy_server, repository_id): remote_db_status = entropy_server.Mirrors.remote_repository_status( repository_id) entropy_server.output("%s:" % (brown(_("Entropy Repository Status")), ), importance=1, header=darkgreen(" * ")) for url, revision in remote_db_status.items(): host = EntropyTransceiver.get_uri_name(url) entropy_server.output("%s: %s" % (darkgreen(_("Host")), bold(host)), header=" ") entropy_server.output("%s: %s" % (purple(_("Remote")), blue(str(revision))), header=" ") local_revision = entropy_server.local_repository_revision( repository_id) entropy_server.output("%s: %s" % (brown(_("Local")), teal(str(local_revision))), header=" ")
def print_repository_status(entropy_server, repository_id): remote_db_status = entropy_server.Mirrors.remote_repository_status( repository_id) entropy_server.output( "%s:" % (brown(_("Entropy Repository Status")),), importance=1, header=darkgreen(" * ") ) for url, revision in remote_db_status.items(): host = EntropyTransceiver.get_uri_name(url) entropy_server.output( "%s: %s" % (darkgreen(_("Host")), bold(host)), header=" ") entropy_server.output( "%s: %s" % (purple(_("Remote")), blue(str(revision))), header=" ") local_revision = entropy_server.local_repository_revision( repository_id) entropy_server.output( "%s: %s" % (brown(_("Local")), teal(str(local_revision))), header=" ")
def handler_verify_upload(self, local_filepath, uri, counter, maxcount, tries, remote_md5 = None): crippled_uri = EntropyTransceiver.get_uri_name(uri) self._entropy.output( "[%s|#%s|(%s/%s)] %s: %s" % ( blue(crippled_uri), darkgreen(str(tries)), blue(str(counter)), bold(str(maxcount)), darkgreen(_("verifying upload (if supported)")), blue(os.path.basename(local_filepath)), ), importance = 0, level = "info", header = red(" @@ "), back = True ) valid_remote_md5 = True # if remote server supports MD5 commands, remote_md5 is filled if const_isstring(remote_md5): valid_md5 = is_valid_md5(remote_md5) ckres = False if valid_md5: # seems valid ckres = compare_md5(local_filepath, remote_md5) if ckres: self._entropy.output( "[%s|#%s|(%s/%s)] %s: %s: %s" % ( blue(crippled_uri), darkgreen(str(tries)), blue(str(counter)), bold(str(maxcount)), blue(_("digest verification")), os.path.basename(local_filepath), darkgreen(_("so far, so good!")), ), importance = 0, level = "info", header = red(" @@ ") ) return True # ouch! elif not valid_md5: # mmmh... malformed md5, try with handlers self._entropy.output( "[%s|#%s|(%s/%s)] %s: %s: %s" % ( blue(crippled_uri), darkgreen(str(tries)), blue(str(counter)), bold(str(maxcount)), blue(_("digest verification")), os.path.basename(local_filepath), bold(_("malformed md5 provided to function")), ), importance = 0, level = "warning", header = brown(" @@ ") ) else: # it's really bad! self._entropy.output( "[%s|#%s|(%s/%s)] %s: %s: %s" % ( blue(crippled_uri), darkgreen(str(tries)), blue(str(counter)), bold(str(maxcount)), blue(_("digest verification")), os.path.basename(local_filepath), bold(_("remote md5 is invalid")), ), importance = 0, level = "warning", header = brown(" @@ ") ) valid_remote_md5 = False return valid_remote_md5 # always valid
def _transceive(self, uri): fine = set() broken = set() fail = False crippled_uri = EntropyTransceiver.get_uri_name(uri) action = 'push' if self.download: action = 'pull' elif self.remove: action = 'remove' try: txc = EntropyTransceiver(uri) if const_isnumber(self.speed_limit): txc.set_speed_limit(self.speed_limit) txc.set_output_interface(self._entropy) except TransceiverConnectionError: print_traceback() return True, fine, broken # issues maxcount = len(self.myfiles) counter = 0 with txc as handler: for mypath in self.myfiles: base_dir = self.txc_basedir if isinstance(mypath, tuple): if len(mypath) < 2: continue base_dir, mypath = mypath if not handler.is_dir(base_dir): handler.makedirs(base_dir) mypath_fn = os.path.basename(mypath) remote_path = os.path.join(base_dir, mypath_fn) syncer = handler.upload myargs = (mypath, remote_path) if self.download: syncer = handler.download local_path = os.path.join(self.local_basedir, mypath_fn) myargs = (remote_path, local_path) elif self.remove: syncer = handler.delete myargs = (remote_path,) fallback_syncer, fallback_args = None, None # upload -> remote copy herustic support # if a package file might have been already uploaded # to remote mirror, try to look in other repositories' # package directories if a file, with the same md5 and name # is already available. In this case, use remote copy instead # of upload to save bandwidth. if self._copy_herustic and (syncer == handler.upload): # copy herustic support enabled # we are uploading new_syncer, new_args = self._copy_herustic_support( handler, mypath, base_dir, remote_path) if new_syncer is not None: fallback_syncer, fallback_args = syncer, myargs syncer, myargs = new_syncer, new_args action = "copy" counter += 1 tries = 0 done = False lastrc = None while tries < 5: tries += 1 self._entropy.output( "[%s|#%s|(%s/%s)] %s: %s" % ( blue(crippled_uri), darkgreen(str(tries)), blue(str(counter)), bold(str(maxcount)), blue(action), red(os.path.basename(mypath)), ), importance = 0, level = "info", header = red(" @@ ") ) rc = syncer(*myargs) if (not rc) and (fallback_syncer is not None): # if we have a fallback syncer, try it first # before giving up. rc = fallback_syncer(*myargs) if rc and not (self.download or self.remove): remote_md5 = handler.get_md5(remote_path) rc = self.handler_verify_upload(mypath, uri, counter, maxcount, tries, remote_md5 = remote_md5) if rc: self._entropy.output( "[%s|#%s|(%s/%s)] %s %s: %s" % ( blue(crippled_uri), darkgreen(str(tries)), blue(str(counter)), bold(str(maxcount)), blue(action), _("successful"), red(os.path.basename(mypath)), ), importance = 0, level = "info", header = darkgreen(" @@ ") ) done = True fine.add(uri) break else: self._entropy.output( "[%s|#%s|(%s/%s)] %s %s: %s" % ( blue(crippled_uri), darkgreen(str(tries)), blue(str(counter)), bold(str(maxcount)), blue(action), brown(_("failed, retrying")), red(os.path.basename(mypath)), ), importance = 0, level = "warning", header = brown(" @@ ") ) lastrc = rc continue if not done: self._entropy.output( "[%s|(%s/%s)] %s %s: %s - %s: %s" % ( blue(crippled_uri), blue(str(counter)), bold(str(maxcount)), blue(action), darkred("failed, giving up"), red(os.path.basename(mypath)), _("error"), lastrc, ), importance = 1, level = "error", header = darkred(" !!! ") ) if mypath not in self.critical_files: self._entropy.output( "[%s|(%s/%s)] %s: %s, %s..." % ( blue(crippled_uri), blue(str(counter)), bold(str(maxcount)), blue(_("not critical")), os.path.basename(mypath), blue(_("continuing")), ), importance = 1, level = "warning", header = brown(" @@ ") ) continue fail = True broken.add((uri, lastrc)) # next mirror break return fail, fine, broken
def handler_verify_upload(self, local_filepath, uri, counter, maxcount, tries, remote_md5=None): crippled_uri = EntropyTransceiver.get_uri_name(uri) self._entropy.output("[%s|#%s|(%s/%s)] %s: %s" % ( blue(crippled_uri), darkgreen(str(tries)), blue(str(counter)), bold(str(maxcount)), darkgreen(_("verifying upload (if supported)")), blue(os.path.basename(local_filepath)), ), importance=0, level="info", header=red(" @@ "), back=True) valid_remote_md5 = True # if remote server supports MD5 commands, remote_md5 is filled if const_isstring(remote_md5): valid_md5 = is_valid_md5(remote_md5) ckres = False if valid_md5: # seems valid ckres = compare_md5(local_filepath, remote_md5) if ckres: self._entropy.output("[%s|#%s|(%s/%s)] %s: %s: %s" % ( blue(crippled_uri), darkgreen(str(tries)), blue(str(counter)), bold(str(maxcount)), blue(_("digest verification")), os.path.basename(local_filepath), darkgreen(_("so far, so good!")), ), importance=0, level="info", header=red(" @@ ")) return True # ouch! elif not valid_md5: # mmmh... malformed md5, try with handlers self._entropy.output("[%s|#%s|(%s/%s)] %s: %s: %s" % ( blue(crippled_uri), darkgreen(str(tries)), blue(str(counter)), bold(str(maxcount)), blue(_("digest verification")), os.path.basename(local_filepath), bold(_("malformed md5 provided to function")), ), importance=0, level="warning", header=brown(" @@ ")) else: # it's really bad! self._entropy.output("[%s|#%s|(%s/%s)] %s: %s: %s" % ( blue(crippled_uri), darkgreen(str(tries)), blue(str(counter)), bold(str(maxcount)), blue(_("digest verification")), os.path.basename(local_filepath), bold(_("remote md5 is invalid")), ), importance=0, level="warning", header=brown(" @@ ")) valid_remote_md5 = False return valid_remote_md5 # always valid
def _transceive(self, uri): fine = set() broken = set() fail = False crippled_uri = EntropyTransceiver.get_uri_name(uri) action = 'push' if self.download: action = 'pull' elif self.remove: action = 'remove' try: txc = EntropyTransceiver(uri) if const_isnumber(self.speed_limit): txc.set_speed_limit(self.speed_limit) txc.set_output_interface(self._entropy) except TransceiverConnectionError: print_traceback() return True, fine, broken # issues maxcount = len(self.myfiles) counter = 0 with txc as handler: for mypath in self.myfiles: base_dir = self.txc_basedir if isinstance(mypath, tuple): if len(mypath) < 2: continue base_dir, mypath = mypath if not handler.is_dir(base_dir): handler.makedirs(base_dir) mypath_fn = os.path.basename(mypath) remote_path = os.path.join(base_dir, mypath_fn) syncer = handler.upload myargs = (mypath, remote_path) if self.download: syncer = handler.download local_path = os.path.join(self.local_basedir, mypath_fn) myargs = (remote_path, local_path) elif self.remove: syncer = handler.delete myargs = (remote_path, ) fallback_syncer, fallback_args = None, None # upload -> remote copy herustic support # if a package file might have been already uploaded # to remote mirror, try to look in other repositories' # package directories if a file, with the same md5 and name # is already available. In this case, use remote copy instead # of upload to save bandwidth. if self._copy_herustic and (syncer == handler.upload): # copy herustic support enabled # we are uploading new_syncer, new_args = self._copy_herustic_support( handler, mypath, base_dir, remote_path) if new_syncer is not None: fallback_syncer, fallback_args = syncer, myargs syncer, myargs = new_syncer, new_args action = "copy" counter += 1 tries = 0 done = False lastrc = None while tries < 5: tries += 1 self._entropy.output("[%s|#%s|(%s/%s)] %s: %s" % ( blue(crippled_uri), darkgreen(str(tries)), blue(str(counter)), bold(str(maxcount)), blue(action), red(os.path.basename(mypath)), ), importance=0, level="info", header=red(" @@ ")) rc = syncer(*myargs) if (not rc) and (fallback_syncer is not None): # if we have a fallback syncer, try it first # before giving up. rc = fallback_syncer(*myargs) if rc and not (self.download or self.remove): remote_md5 = handler.get_md5(remote_path) rc = self.handler_verify_upload(mypath, uri, counter, maxcount, tries, remote_md5=remote_md5) if rc: self._entropy.output("[%s|#%s|(%s/%s)] %s %s: %s" % ( blue(crippled_uri), darkgreen(str(tries)), blue(str(counter)), bold(str(maxcount)), blue(action), _("successful"), red(os.path.basename(mypath)), ), importance=0, level="info", header=darkgreen(" @@ ")) done = True fine.add(uri) break else: self._entropy.output("[%s|#%s|(%s/%s)] %s %s: %s" % ( blue(crippled_uri), darkgreen(str(tries)), blue(str(counter)), bold(str(maxcount)), blue(action), brown(_("failed, retrying")), red(os.path.basename(mypath)), ), importance=0, level="warning", header=brown(" @@ ")) lastrc = rc continue if not done: self._entropy.output("[%s|(%s/%s)] %s %s: %s - %s: %s" % ( blue(crippled_uri), blue(str(counter)), bold(str(maxcount)), blue(action), darkred("failed, giving up"), red(os.path.basename(mypath)), _("error"), lastrc, ), importance=1, level="error", header=darkred(" !!! ")) if mypath not in self.critical_files: self._entropy.output("[%s|(%s/%s)] %s: %s, %s..." % ( blue(crippled_uri), blue(str(counter)), bold(str(maxcount)), blue(_("not critical")), os.path.basename(mypath), blue(_("continuing")), ), importance=1, level="warning", header=brown(" @@ ")) continue fail = True broken.add((uri, lastrc)) # next mirror break return fail, fine, broken