def swallow(self):
"""
Given the URI at the constructor, this method returns the first valid
URI handler instance found that can be used to do required action.
@raise entropy.exceptions.UriHandlerNotFound: when URI handler for given
URI is not available.
"""
handlers = EntropyTransceiver.get_uri_handlers()
for handler in handlers:
if handler.approve_uri(self._uri):
handler_instance = handler(self._uri)
if self._output_interface is not None:
handler_instance.set_output_interface(
self._output_interface)
if const_isnumber(self._speed_limit):
handler_instance.set_speed_limit(self._speed_limit)
handler_instance.set_verbosity(self._verbose)
handler_instance.set_silent(self._silent)
if const_isnumber(self._timeout):
handler_instance.set_timeout(self._timeout)
return handler_instance
raise UriHandlerNotFound(
"no URI handler available for %s" % (self._uri,))
def set_timeout(self, timeout):
"""
Set transceiver tx/rx timeout value in seconds.
@param timeout: timeout in seconds
@type timeout: int
"""
if not const_isnumber(timeout):
raise AttributeError("not a number")
self._timeout = timeout
def set_speed_limit(self, speed_limit):
"""
Set download/upload speed limit in kb/sec form.
@param speed_limit: speed limit in kb/sec form.
@type speed_limit: int
"""
if not const_isnumber(speed_limit):
raise AttributeError("not a number")
self._speed_limit = speed_limit
def set_speed_limit(self, speed_limit):
"""
Set download/upload speed limit in kb/sec form.
@param speed_limit: speed limit in kb/sec form.
@type speed_limit: int
"""
if not const_isnumber(speed_limit):
raise AttributeError("not a number")
self._speed_limit = speed_limit
def set_speed_limit(self, speed_limit):
"""
Set download/upload speed limit in kb/sec form.
Zero value will be considered as "disable speed limiter".
@param speed_limit: speed limit in kb/sec form.
@type speed_limit: int
@raise AttributeError: if speed_limit is not an integer
"""
if not const_isnumber(speed_limit):
raise AttributeError("expected a valid number")
self._speed_limit = speed_limit
def _setup_common_args(self, remote_path):
args = []
if const_isnumber(self._timeout):
args += ["-o", "ConnectTimeout=%s" % (self._timeout,),
"-o", "ServerAliveCountMax=4", # hardcoded
"-o", "ServerAliveInterval=15"] # hardcoded
if self._speed_limit:
args += ["-l", str(self._speed_limit*8)] # scp wants kbits/sec
remote_ptr = os.path.join(self.__dir, remote_path)
remote_str = ""
if self.__user:
remote_str += self.__user + "@"
remote_str += self.__host + ":" + remote_ptr
return args, remote_str
def _transceive(self, uri):
fine = set()
broken = set()
fail = False
crippled_uri = EntropyTransceiver.get_uri_name(uri)
action = 'push'
if self.download:
action = 'pull'
elif self.remove:
action = 'remove'
try:
txc = EntropyTransceiver(uri)
if const_isnumber(self.speed_limit):
txc.set_speed_limit(self.speed_limit)
txc.set_output_interface(self._entropy)
except TransceiverConnectionError:
print_traceback()
return True, fine, broken # issues
maxcount = len(self.myfiles)
counter = 0
with txc as handler:
for mypath in self.myfiles:
base_dir = self.txc_basedir
if isinstance(mypath, tuple):
if len(mypath) < 2:
continue
base_dir, mypath = mypath
if not handler.is_dir(base_dir):
handler.makedirs(base_dir)
mypath_fn = os.path.basename(mypath)
remote_path = os.path.join(base_dir, mypath_fn)
syncer = handler.upload
myargs = (mypath, remote_path)
if self.download:
syncer = handler.download
local_path = os.path.join(self.local_basedir, mypath_fn)
myargs = (remote_path, local_path)
elif self.remove:
syncer = handler.delete
myargs = (remote_path,)
fallback_syncer, fallback_args = None, None
# upload -> remote copy herustic support
# if a package file might have been already uploaded
# to remote mirror, try to look in other repositories'
# package directories if a file, with the same md5 and name
# is already available. In this case, use remote copy instead
# of upload to save bandwidth.
if self._copy_herustic and (syncer == handler.upload):
# copy herustic support enabled
# we are uploading
new_syncer, new_args = self._copy_herustic_support(
handler, mypath, base_dir, remote_path)
if new_syncer is not None:
fallback_syncer, fallback_args = syncer, myargs
syncer, myargs = new_syncer, new_args
action = "copy"
counter += 1
tries = 0
done = False
lastrc = None
while tries < 5:
tries += 1
self._entropy.output(
"[%s|#%s|(%s/%s)] %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(action),
red(os.path.basename(mypath)),
),
importance = 0,
level = "info",
header = red(" @@ ")
)
rc = syncer(*myargs)
if (not rc) and (fallback_syncer is not None):
# if we have a fallback syncer, try it first
# before giving up.
rc = fallback_syncer(*myargs)
if rc and not (self.download or self.remove):
remote_md5 = handler.get_md5(remote_path)
rc = self.handler_verify_upload(mypath, uri,
counter, maxcount, tries, remote_md5 = remote_md5)
if rc:
self._entropy.output(
"[%s|#%s|(%s/%s)] %s %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(action),
_("successful"),
red(os.path.basename(mypath)),
),
importance = 0,
level = "info",
header = darkgreen(" @@ ")
)
done = True
fine.add(uri)
break
else:
self._entropy.output(
"[%s|#%s|(%s/%s)] %s %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(action),
brown(_("failed, retrying")),
red(os.path.basename(mypath)),
),
importance = 0,
level = "warning",
header = brown(" @@ ")
)
lastrc = rc
continue
if not done:
self._entropy.output(
"[%s|(%s/%s)] %s %s: %s - %s: %s" % (
blue(crippled_uri),
blue(str(counter)),
bold(str(maxcount)),
blue(action),
darkred("failed, giving up"),
red(os.path.basename(mypath)),
_("error"),
lastrc,
),
importance = 1,
level = "error",
header = darkred(" !!! ")
)
if mypath not in self.critical_files:
self._entropy.output(
"[%s|(%s/%s)] %s: %s, %s..." % (
blue(crippled_uri),
blue(str(counter)),
bold(str(maxcount)),
blue(_("not critical")),
os.path.basename(mypath),
blue(_("continuing")),
),
importance = 1,
level = "warning",
header = brown(" @@ ")
)
continue
fail = True
broken.add((uri, lastrc))
# next mirror
break
return fail, fine, broken
def _transceive(self, uri):
fine = set()
broken = set()
fail = False
crippled_uri = EntropyTransceiver.get_uri_name(uri)
action = 'push'
if self.download:
action = 'pull'
elif self.remove:
action = 'remove'
try:
txc = EntropyTransceiver(uri)
if const_isnumber(self.speed_limit):
txc.set_speed_limit(self.speed_limit)
txc.set_output_interface(self._entropy)
except TransceiverConnectionError:
print_traceback()
return True, fine, broken # issues
maxcount = len(self.myfiles)
counter = 0
with txc as handler:
for mypath in self.myfiles:
base_dir = self.txc_basedir
if isinstance(mypath, tuple):
if len(mypath) < 2:
continue
base_dir, mypath = mypath
if not handler.is_dir(base_dir):
handler.makedirs(base_dir)
mypath_fn = os.path.basename(mypath)
remote_path = os.path.join(base_dir, mypath_fn)
syncer = handler.upload
myargs = (mypath, remote_path)
if self.download:
syncer = handler.download
local_path = os.path.join(self.local_basedir, mypath_fn)
myargs = (remote_path, local_path)
elif self.remove:
syncer = handler.delete
myargs = (remote_path, )
fallback_syncer, fallback_args = None, None
# upload -> remote copy herustic support
# if a package file might have been already uploaded
# to remote mirror, try to look in other repositories'
# package directories if a file, with the same md5 and name
# is already available. In this case, use remote copy instead
# of upload to save bandwidth.
if self._copy_herustic and (syncer == handler.upload):
# copy herustic support enabled
# we are uploading
new_syncer, new_args = self._copy_herustic_support(
handler, mypath, base_dir, remote_path)
if new_syncer is not None:
fallback_syncer, fallback_args = syncer, myargs
syncer, myargs = new_syncer, new_args
action = "copy"
counter += 1
tries = 0
done = False
lastrc = None
while tries < 5:
tries += 1
self._entropy.output("[%s|#%s|(%s/%s)] %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(action),
red(os.path.basename(mypath)),
),
importance=0,
level="info",
header=red(" @@ "))
rc = syncer(*myargs)
if (not rc) and (fallback_syncer is not None):
# if we have a fallback syncer, try it first
# before giving up.
rc = fallback_syncer(*myargs)
if rc and not (self.download or self.remove):
remote_md5 = handler.get_md5(remote_path)
rc = self.handler_verify_upload(mypath,
uri,
counter,
maxcount,
tries,
remote_md5=remote_md5)
if rc:
self._entropy.output("[%s|#%s|(%s/%s)] %s %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(action),
_("successful"),
red(os.path.basename(mypath)),
),
importance=0,
level="info",
header=darkgreen(" @@ "))
done = True
fine.add(uri)
break
else:
self._entropy.output("[%s|#%s|(%s/%s)] %s %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(action),
brown(_("failed, retrying")),
red(os.path.basename(mypath)),
),
importance=0,
level="warning",
header=brown(" @@ "))
lastrc = rc
continue
if not done:
self._entropy.output("[%s|(%s/%s)] %s %s: %s - %s: %s" % (
blue(crippled_uri),
blue(str(counter)),
bold(str(maxcount)),
blue(action),
darkred("failed, giving up"),
red(os.path.basename(mypath)),
_("error"),
lastrc,
),
importance=1,
level="error",
header=darkred(" !!! "))
if mypath not in self.critical_files:
self._entropy.output("[%s|(%s/%s)] %s: %s, %s..." % (
blue(crippled_uri),
blue(str(counter)),
bold(str(maxcount)),
blue(_("not critical")),
os.path.basename(mypath),
blue(_("continuing")),
),
importance=1,
level="warning",
header=brown(" @@ "))
continue
fail = True
broken.add((uri, lastrc))
# next mirror
break
return fail, fine, broken