Exemple #1
0
    def swallow(self):
        """
        Given the URI at the constructor, this method returns the first valid
        URI handler instance found that can be used to do required action.

        @raise entropy.exceptions.UriHandlerNotFound: when URI handler for given
            URI is not available.
        """
        handlers = EntropyTransceiver.get_uri_handlers()
        for handler in handlers:
            if handler.approve_uri(self._uri):
                handler_instance = handler(self._uri)
                if self._output_interface is not None:
                    handler_instance.set_output_interface(
                        self._output_interface)
                if const_isnumber(self._speed_limit):
                    handler_instance.set_speed_limit(self._speed_limit)
                handler_instance.set_verbosity(self._verbose)
                handler_instance.set_silent(self._silent)
                if const_isnumber(self._timeout):
                    handler_instance.set_timeout(self._timeout)
                return handler_instance

        raise UriHandlerNotFound(
            "no URI handler available for %s" % (self._uri,))
Exemple #2
0
    def set_timeout(self, timeout):
        """
        Set transceiver tx/rx timeout value in seconds.

        @param timeout: timeout in seconds
        @type timeout: int
        """
        if not const_isnumber(timeout):
            raise AttributeError("not a number")
        self._timeout = timeout
Exemple #3
0
    def set_speed_limit(self, speed_limit):
        """
        Set download/upload speed limit in kb/sec form.

        @param speed_limit: speed limit in kb/sec form.
        @type speed_limit: int
        """
        if not const_isnumber(speed_limit):
            raise AttributeError("not a number")
        self._speed_limit = speed_limit
Exemple #4
0
    def set_speed_limit(self, speed_limit):
        """
        Set download/upload speed limit in kb/sec form.

        @param speed_limit: speed limit in kb/sec form.
        @type speed_limit: int
        """
        if not const_isnumber(speed_limit):
            raise AttributeError("not a number")
        self._speed_limit = speed_limit
Exemple #5
0
    def set_speed_limit(self, speed_limit):
        """
        Set download/upload speed limit in kb/sec form.
        Zero value will be considered as "disable speed limiter".

        @param speed_limit: speed limit in kb/sec form.
        @type speed_limit: int
        @raise AttributeError: if speed_limit is not an integer
        """
        if not const_isnumber(speed_limit):
            raise AttributeError("expected a valid number")
        self._speed_limit = speed_limit
Exemple #6
0
    def _setup_common_args(self, remote_path):
        args = []
        if const_isnumber(self._timeout):
            args += ["-o", "ConnectTimeout=%s" % (self._timeout,),
                "-o", "ServerAliveCountMax=4", # hardcoded
                "-o", "ServerAliveInterval=15"] # hardcoded
        if self._speed_limit:
            args += ["-l", str(self._speed_limit*8)] # scp wants kbits/sec
        remote_ptr = os.path.join(self.__dir, remote_path)
        remote_str = ""
        if self.__user:
            remote_str += self.__user + "@"
        remote_str += self.__host + ":" + remote_ptr

        return args, remote_str
Exemple #7
0
    def _transceive(self, uri):

        fine = set()
        broken = set()
        fail = False
        crippled_uri = EntropyTransceiver.get_uri_name(uri)
        action = 'push'
        if self.download:
            action = 'pull'
        elif self.remove:
            action = 'remove'

        try:
            txc = EntropyTransceiver(uri)
            if const_isnumber(self.speed_limit):
                txc.set_speed_limit(self.speed_limit)
            txc.set_output_interface(self._entropy)
        except TransceiverConnectionError:
            print_traceback()
            return True, fine, broken # issues

        maxcount = len(self.myfiles)
        counter = 0

        with txc as handler:

            for mypath in self.myfiles:

                base_dir = self.txc_basedir

                if isinstance(mypath, tuple):
                    if len(mypath) < 2:
                        continue
                    base_dir, mypath = mypath

                if not handler.is_dir(base_dir):
                    handler.makedirs(base_dir)

                mypath_fn = os.path.basename(mypath)
                remote_path = os.path.join(base_dir, mypath_fn)

                syncer = handler.upload
                myargs = (mypath, remote_path)
                if self.download:
                    syncer = handler.download
                    local_path = os.path.join(self.local_basedir, mypath_fn)
                    myargs = (remote_path, local_path)
                elif self.remove:
                    syncer = handler.delete
                    myargs = (remote_path,)

                fallback_syncer, fallback_args = None, None
                # upload -> remote copy herustic support
                # if a package file might have been already uploaded
                # to remote mirror, try to look in other repositories'
                # package directories if a file, with the same md5 and name
                # is already available. In this case, use remote copy instead
                # of upload to save bandwidth.
                if self._copy_herustic and (syncer == handler.upload):
                    # copy herustic support enabled
                    # we are uploading
                    new_syncer, new_args = self._copy_herustic_support(
                        handler, mypath, base_dir, remote_path)
                    if new_syncer is not None:
                        fallback_syncer, fallback_args = syncer, myargs
                        syncer, myargs = new_syncer, new_args
                        action = "copy"

                counter += 1
                tries = 0
                done = False
                lastrc = None

                while tries < 5:
                    tries += 1
                    self._entropy.output(
                        "[%s|#%s|(%s/%s)] %s: %s" % (
                            blue(crippled_uri),
                            darkgreen(str(tries)),
                            blue(str(counter)),
                            bold(str(maxcount)),
                            blue(action),
                            red(os.path.basename(mypath)),
                        ),
                        importance = 0,
                        level = "info",
                        header = red(" @@ ")
                    )
                    rc = syncer(*myargs)
                    if (not rc) and (fallback_syncer is not None):
                        # if we have a fallback syncer, try it first
                        # before giving up.
                        rc = fallback_syncer(*myargs)

                    if rc and not (self.download or self.remove):
                        remote_md5 = handler.get_md5(remote_path)
                        rc = self.handler_verify_upload(mypath, uri,
                            counter, maxcount, tries, remote_md5 = remote_md5)
                    if rc:
                        self._entropy.output(
                            "[%s|#%s|(%s/%s)] %s %s: %s" % (
                                        blue(crippled_uri),
                                        darkgreen(str(tries)),
                                        blue(str(counter)),
                                        bold(str(maxcount)),
                                        blue(action),
                                        _("successful"),
                                        red(os.path.basename(mypath)),
                            ),
                            importance = 0,
                            level = "info",
                            header = darkgreen(" @@ ")
                        )
                        done = True
                        fine.add(uri)
                        break
                    else:
                        self._entropy.output(
                            "[%s|#%s|(%s/%s)] %s %s: %s" % (
                                        blue(crippled_uri),
                                        darkgreen(str(tries)),
                                        blue(str(counter)),
                                        bold(str(maxcount)),
                                        blue(action),
                                        brown(_("failed, retrying")),
                                        red(os.path.basename(mypath)),
                                ),
                            importance = 0,
                            level = "warning",
                            header = brown(" @@ ")
                        )
                        lastrc = rc
                        continue

                if not done:

                    self._entropy.output(
                        "[%s|(%s/%s)] %s %s: %s - %s: %s" % (
                                blue(crippled_uri),
                                blue(str(counter)),
                                bold(str(maxcount)),
                                blue(action),
                                darkred("failed, giving up"),
                                red(os.path.basename(mypath)),
                                _("error"),
                                lastrc,
                        ),
                        importance = 1,
                        level = "error",
                        header = darkred(" !!! ")
                    )

                    if mypath not in self.critical_files:
                        self._entropy.output(
                            "[%s|(%s/%s)] %s: %s, %s..." % (
                                blue(crippled_uri),
                                blue(str(counter)),
                                bold(str(maxcount)),
                                blue(_("not critical")),
                                os.path.basename(mypath),
                                blue(_("continuing")),
                            ),
                            importance = 1,
                            level = "warning",
                            header = brown(" @@ ")
                        )
                        continue

                    fail = True
                    broken.add((uri, lastrc))
                    # next mirror
                    break

        return fail, fine, broken
Exemple #8
0
    def _transceive(self, uri):

        fine = set()
        broken = set()
        fail = False
        crippled_uri = EntropyTransceiver.get_uri_name(uri)
        action = 'push'
        if self.download:
            action = 'pull'
        elif self.remove:
            action = 'remove'

        try:
            txc = EntropyTransceiver(uri)
            if const_isnumber(self.speed_limit):
                txc.set_speed_limit(self.speed_limit)
            txc.set_output_interface(self._entropy)
        except TransceiverConnectionError:
            print_traceback()
            return True, fine, broken  # issues

        maxcount = len(self.myfiles)
        counter = 0

        with txc as handler:

            for mypath in self.myfiles:

                base_dir = self.txc_basedir

                if isinstance(mypath, tuple):
                    if len(mypath) < 2:
                        continue
                    base_dir, mypath = mypath

                if not handler.is_dir(base_dir):
                    handler.makedirs(base_dir)

                mypath_fn = os.path.basename(mypath)
                remote_path = os.path.join(base_dir, mypath_fn)

                syncer = handler.upload
                myargs = (mypath, remote_path)
                if self.download:
                    syncer = handler.download
                    local_path = os.path.join(self.local_basedir, mypath_fn)
                    myargs = (remote_path, local_path)
                elif self.remove:
                    syncer = handler.delete
                    myargs = (remote_path, )

                fallback_syncer, fallback_args = None, None
                # upload -> remote copy herustic support
                # if a package file might have been already uploaded
                # to remote mirror, try to look in other repositories'
                # package directories if a file, with the same md5 and name
                # is already available. In this case, use remote copy instead
                # of upload to save bandwidth.
                if self._copy_herustic and (syncer == handler.upload):
                    # copy herustic support enabled
                    # we are uploading
                    new_syncer, new_args = self._copy_herustic_support(
                        handler, mypath, base_dir, remote_path)
                    if new_syncer is not None:
                        fallback_syncer, fallback_args = syncer, myargs
                        syncer, myargs = new_syncer, new_args
                        action = "copy"

                counter += 1
                tries = 0
                done = False
                lastrc = None

                while tries < 5:
                    tries += 1
                    self._entropy.output("[%s|#%s|(%s/%s)] %s: %s" % (
                        blue(crippled_uri),
                        darkgreen(str(tries)),
                        blue(str(counter)),
                        bold(str(maxcount)),
                        blue(action),
                        red(os.path.basename(mypath)),
                    ),
                                         importance=0,
                                         level="info",
                                         header=red(" @@ "))
                    rc = syncer(*myargs)
                    if (not rc) and (fallback_syncer is not None):
                        # if we have a fallback syncer, try it first
                        # before giving up.
                        rc = fallback_syncer(*myargs)

                    if rc and not (self.download or self.remove):
                        remote_md5 = handler.get_md5(remote_path)
                        rc = self.handler_verify_upload(mypath,
                                                        uri,
                                                        counter,
                                                        maxcount,
                                                        tries,
                                                        remote_md5=remote_md5)
                    if rc:
                        self._entropy.output("[%s|#%s|(%s/%s)] %s %s: %s" % (
                            blue(crippled_uri),
                            darkgreen(str(tries)),
                            blue(str(counter)),
                            bold(str(maxcount)),
                            blue(action),
                            _("successful"),
                            red(os.path.basename(mypath)),
                        ),
                                             importance=0,
                                             level="info",
                                             header=darkgreen(" @@ "))
                        done = True
                        fine.add(uri)
                        break
                    else:
                        self._entropy.output("[%s|#%s|(%s/%s)] %s %s: %s" % (
                            blue(crippled_uri),
                            darkgreen(str(tries)),
                            blue(str(counter)),
                            bold(str(maxcount)),
                            blue(action),
                            brown(_("failed, retrying")),
                            red(os.path.basename(mypath)),
                        ),
                                             importance=0,
                                             level="warning",
                                             header=brown(" @@ "))
                        lastrc = rc
                        continue

                if not done:

                    self._entropy.output("[%s|(%s/%s)] %s %s: %s - %s: %s" % (
                        blue(crippled_uri),
                        blue(str(counter)),
                        bold(str(maxcount)),
                        blue(action),
                        darkred("failed, giving up"),
                        red(os.path.basename(mypath)),
                        _("error"),
                        lastrc,
                    ),
                                         importance=1,
                                         level="error",
                                         header=darkred(" !!! "))

                    if mypath not in self.critical_files:
                        self._entropy.output("[%s|(%s/%s)] %s: %s, %s..." % (
                            blue(crippled_uri),
                            blue(str(counter)),
                            bold(str(maxcount)),
                            blue(_("not critical")),
                            os.path.basename(mypath),
                            blue(_("continuing")),
                        ),
                                             importance=1,
                                             level="warning",
                                             header=brown(" @@ "))
                        continue

                    fail = True
                    broken.add((uri, lastrc))
                    # next mirror
                    break

        return fail, fine, broken