Exemple #1
0
    def copy_bbcp(self, host, position):
        """
        Check that the bbcp is installed on the host otherwise copy bbcp to
        the host.

        :param host: Host to which the bbcp will be copied
        :param position: src or dst
        """
        runner = self.runner(host, position)

        LOG.debug("Checking if bbcp is installed on '%s' host", host)

        bbcp_installed = (_bbcp_installed(runner, 'bbcp')
                          or _bbcp_installed(runner, BBCP_PATH))

        if bbcp_installed:
            return

        if position == 'src':
            bbcp = CONF.bbcp.src_path
            user = CONF.src.ssh_user
        else:
            bbcp = CONF.bbcp.dst_path
            user = CONF.dst.ssh_user

        LOG.debug("Copying %s to %s:%s", bbcp, host, BBCP_PATH)
        scp_bbcp = "scp {ssh_opts} {bbcp} {user}@{host}:{tmp}"
        local.run(scp_bbcp.format(ssh_opts=ssh_util.default_ssh_options(),
                                  bbcp=bbcp,
                                  user=user,
                                  host=host,
                                  tmp=BBCP_PATH),
                  capture_output=False)
        chmod_bbcp = "chmod 777 {path}"
        runner.run(chmod_bbcp, path=BBCP_PATH)
Exemple #2
0
    def transfer(self, data):
        host_src = data['host_src']
        path_src = data['path_src']
        host_dst = data['host_dst']
        path_dst = data['path_dst']

        cmd = "{bbcp} {options} {src} {dst} 2>&1"

        options = CONF.bbcp.options
        additional_options = []
        # -f: forces the copy by first unlinking the target file before
        # copying.
        # -S: command to start bbcp on the source node.
        # -T: command to start bbcp on the target node.
        forced_options = ['-f']
        if CONF.migrate.copy_with_md5_verification:
            # -e: error check data for transmission errors using md5 checksum.
            forced_options.append('-e')
        for o in forced_options:
            if o not in options:
                additional_options.append(o)
        run_options = ['-T']
        if CONF.migrate.direct_transfer:
            src = path_src
            bbcp = BBCP_PATH
            runner = self.runner(host_src, 'src', data.get('gateway'))
            run = runner.run
        else:
            run_options += ['-S']
            src = '{user_src}@{host_src}:{path_src}'.format(
                user_src=CONF.src.ssh_user,
                host_src=host_src,
                path_src=path_src,
            )
            bbcp = CONF.bbcp.path
            run = local.run
        dst = '{user_dst}@{host_dst}:{path_dst}'.format(
            user_dst=CONF.dst.ssh_user,
            host_dst=host_dst,
            path_dst=path_dst,
        )
        for o in run_options:
            if o not in options:
                additional_options.append(o + " '{remote_bbcp}'")
        remote_bbcp = "ssh {ssh_opts} %I -l %U %H {path}".format(
            ssh_opts=ssh_util.default_ssh_options(), path=BBCP_PATH)
        options += ' ' + ' '.join(additional_options).format(
            remote_bbcp=remote_bbcp)

        retrier = retrying.Retry(max_attempts=CONF.migrate.retry, timeout=0)
        try:
            retrier.run(run,
                        cmd.format(bbcp=bbcp,
                                   options=options,
                                   src=src,
                                   dst=dst),
                        capture_output=False)
        except retrying.MaxAttemptsReached:
            self.clean_dst(host_dst, path_dst)
            raise base.FileCopyError(**data)
Exemple #3
0
    def copy_bbcp(self, host, position):
        """
        Check that the bbcp is installed on the host otherwise copy bbcp to
        the host.

        :param host: Host to which the bbcp will be copied
        :param position: src or dst
        """
        runner = self.runner(host, position)

        LOG.debug("Checking if bbcp is installed on '%s' host", host)

        bbcp_installed = (_bbcp_installed(runner, 'bbcp') or
                          _bbcp_installed(runner, BBCP_PATH))

        if bbcp_installed:
            return

        if position == 'src':
            bbcp = CONF.bbcp.src_path
            user = CONF.src.ssh_user
        else:
            bbcp = CONF.bbcp.dst_path
            user = CONF.dst.ssh_user

        LOG.debug("Copying %s to %s:%s", bbcp, host, BBCP_PATH)
        scp_bbcp = "scp {ssh_opts} {bbcp} {user}@{host}:{tmp}"
        local.run(scp_bbcp.format(ssh_opts=ssh_util.default_ssh_options(),
                                  bbcp=bbcp,
                                  user=user,
                                  host=host,
                                  tmp=BBCP_PATH),
                  capture_output=False)
        chmod_bbcp = "chmod 777 {path}"
        runner.run(chmod_bbcp, path=BBCP_PATH)
Exemple #4
0
    def run_scp(self, host_src, path_src, host_dst, path_dst, gateway=None):
        """
        Copy a file from source host to destination by scp.

        :param host_src: Source host.
        :param path_src: Path to a file on source host.
        :param host_dst: Destination host.
        :param path_dst: Path to a file on destination host.
        :raise: FileCopyError in case the file was not copied by any reasons.
        """
        cmd = "scp {ssh_opts} {src} {dst}"

        if CONF.migrate.direct_transfer:
            src = path_src
            runner = self.runner(host_src, 'src', gateway)
            run = runner.run
        else:
            # -3: Copies between two remote hosts are transferred through
            # the local host.
            src = '-3 {src_user}@{src_host}:{src_path}'.format(
                src_user=CONF.src.ssh_user,
                src_host=host_src,
                src_path=path_src,
            )
            run = local.run
        dst = '{dst_user}@{dst_host}:{dst_path}'.format(
            dst_user=CONF.dst.ssh_user,
            dst_host=host_dst,
            dst_path=path_dst,
        )

        kwargs = {
            'host_src': host_src,
            'path_src': path_src,
            'host_dst': host_dst,
            'path_dst': path_dst,
            'gateway': gateway
        }

        retrier = retrying.Retry(
            max_attempts=CONF.migrate.retry,
            predicate=self.verify,
            predicate_kwargs=kwargs,
        )
        LOG.info("Copying file '%s' to '%s'", path_src, host_dst)
        try:
            retrier.run(run,
                        cmd.format(ssh_opts=ssh_util.default_ssh_options(),
                                   src=src,
                                   dst=dst),
                        capture_output=False)
        except retrying.MaxAttemptsReached:
            self.clean_dst(host_dst, path_dst)
            raise base.FileCopyError(host_src=host_src,
                                     path_src=path_src,
                                     host_dst=host_dst,
                                     path_dst=path_dst)
Exemple #5
0
    def run_scp(self, host_src, path_src, host_dst, path_dst, gateway=None):
        """
        Copy a file from source host to destination by scp.

        :param host_src: Source host.
        :param path_src: Path to a file on source host.
        :param host_dst: Destination host.
        :param path_dst: Path to a file on destination host.
        :raise: FileCopyError in case the file was not copied by any reasons.
        """
        cmd = "scp {ssh_opts} {src} {dst}"

        if CONF.migrate.direct_transfer:
            src = path_src
            runner = self.runner(host_src, 'src', gateway)
            run = runner.run
        else:
            # -3: Copies between two remote hosts are transferred through
            # the local host.
            src = '-3 {src_user}@{src_host}:{src_path}'.format(
                src_user=CONF.src.ssh_user,
                src_host=host_src,
                src_path=path_src,
            )
            run = local.run
        dst = '{dst_user}@{dst_host}:{dst_path}'.format(
            dst_user=CONF.dst.ssh_user,
            dst_host=host_dst,
            dst_path=path_dst,
        )

        kwargs = {'host_src': host_src,
                  'path_src': path_src,
                  'host_dst': host_dst,
                  'path_dst': path_dst,
                  'gateway': gateway}

        retrier = retrying.Retry(
            max_attempts=CONF.migrate.retry,
            predicate=self.verify,
            predicate_kwargs=kwargs,
        )
        LOG.info("Copying file '%s' to '%s'", path_src, host_dst)
        try:
            retrier.run(run, cmd.format(
                ssh_opts=ssh_util.default_ssh_options(),
                src=src,
                dst=dst),
                        capture_output=False)
        except retrying.MaxAttemptsReached:
            self.clean_dst(host_dst, path_dst)
            raise base.FileCopyError(host_src=host_src,
                                     path_src=path_src,
                                     host_dst=host_dst,
                                     path_dst=path_dst)
    def transfer(self, data):
        host_src = data['host_src']
        path_src = data['path_src']
        host_dst = data['host_dst']
        path_dst = data['path_dst']

        cmd = ("rsync "
               "--partial "
               "--inplace "
               "--perms "
               "--times "
               "--compress "
               "--verbose "
               "--progress "
               "--rsh='ssh {ssh_opts}' "
               "{path_src} "
               "{user_dst}@{host_dst}:{path_dst}")
        ssh_opts = ssh_util.default_ssh_options()

        if CONF.migrate.direct_transfer:
            remote_tunnel = None
        else:
            ssh_opts += " -p {port}".format(port=CONF.rsync.port)
            remote_tunnel = RsyncRemoteTunnelOptions(host_dst)
            host_dst = "localhost"

        runner = self.runner(host_src,
                             'src',
                             data.get('gateway'),
                             remote_tunnel=remote_tunnel)
        try:
            runner.run_repeat_on_errors(cmd,
                                        ssh_opts=ssh_opts,
                                        path_src=path_src,
                                        user_dst=CONF.dst.ssh_user,
                                        host_dst=host_dst,
                                        path_dst=path_dst)
        except remote_runner.RemoteExecutionError:
            self.clean_dst(host_dst, path_dst)
            raise base.FileCopyError(**data)
Exemple #7
0
    def copy_bbcp(self, host, position):
        """
        Check that the bbcp is installed on the host otherwise copy bbcp to
        the host.

        :param host: Host to which the bbcp will be copied
        :param position: src or dst
        """
        runner = self.runner(host, position)

        LOG.debug("Checking if bbcp is installed on '%s' host", host)
        cmd = "bbcp --help &>/dev/null"
        try:
            runner.run(cmd)
        except remote_runner.RemoteExecutionError:
            pass
        else:
            return

        if position == 'src':
            bbcp = CONF.bbcp.src_path
            user = CONF.src.ssh_user
        else:
            bbcp = CONF.bbcp.dst_path
            user = CONF.dst.ssh_user

        LOG.debug("Copying %s to %s:%s", bbcp, host, BBCP_PATH)
        cmd = "scp {ssh_opts} {bbcp} {user}@{host}:{tmp}"
        local.run(cmd.format(ssh_opts=ssh_util.default_ssh_options(),
                             bbcp=bbcp,
                             user=user,
                             host=host,
                             tmp=BBCP_PATH),
                  capture_output=False)
        cmd = "chmod 755 {path}"
        runner.run(cmd, path=BBCP_PATH)
Exemple #8
0
    def transfer(self, data):
        host_src = data['host_src']
        path_src = data['path_src']
        host_dst = data['host_dst']
        path_dst = data['path_dst']

        cmd = ("rsync "
               "--partial "
               "--inplace "
               "--compress "
               "--verbose "
               "--progress "
               "--rsh='ssh {ssh_opts}' "
               "{path_src} "
               "{user_dst}@{host_dst}:{path_dst}")
        ssh_opts = ssh_util.default_ssh_options()

        if CONF.migrate.direct_transfer:
            remote_tunnel = None
        else:
            ssh_opts += " -p {port}".format(port=CONF.rsync.port)
            remote_tunnel = RsyncRemoteTunnelOptions(host_dst)
            host_dst = "localhost"

        runner = self.runner(host_src, 'src', data.get('gateway'),
                             remote_tunnel=remote_tunnel)
        try:
            runner.run_repeat_on_errors(cmd,
                                        ssh_opts=ssh_opts,
                                        path_src=path_src,
                                        user_dst=CONF.dst.ssh_user,
                                        host_dst=host_dst,
                                        path_dst=path_dst)
        except remote_runner.RemoteExecutionError:
            self.clean_dst(host_dst, path_dst)
            raise base.FileCopyError(**data)
    def run(self, **kwargs):
        cfg = self.cloud.cloud_config.cloud
        runner = remote_runner.RemoteRunner(cfg.ssh_host, cfg.ssh_user)

        with files.LocalTempDir('check_band_XXXX') as local_dir:
            with files.RemoteTempDir(runner, 'check_band_XXXX') as remote_dir:
                filename = str(uuid.uuid4())
                local_filepath = os.path.join(local_dir.dirname, filename)
                remote_filepath = os.path.join(remote_dir.dirname, filename)
                claimed_bandw = self.cloud.cloud_config.initial_check.\
                    claimed_bandwidth
                filesize = self.cloud.cloud_config.initial_check.test_file_size
                factor = self.cloud.cloud_config.initial_check.factor
                req_bandwidth = claimed_bandw * factor

                scp_download = "scp {ssh_opts} {user}@{host}:{filepath} " \
                               "{dirname}"
                scp_upload = "scp {ssh_opts} {filepath} {user}@{host}:" \
                             "{dirname}"
                dd_command = "dd if=/dev/zero of={filepath} bs=1 count=0 " \
                             "seek={filesize}"
                runner.run(dd_command,
                           filepath=remote_filepath,
                           filesize=filesize)

                LOG.info("Checking download speed... Wait please.")
                period_download = utils.timer(
                    local.run,
                    scp_download.format(
                        ssh_opts=ssh_util.default_ssh_options(),
                        user=cfg.ssh_user,
                        host=cfg.ssh_host,
                        filepath=remote_filepath,
                        dirname=local_dir.dirname
                    )
                )

                LOG.info("Checking upload speed... Wait please.")
                period_upload = utils.timer(
                    local.run,
                    scp_upload.format(
                        ssh_opts=ssh_util.default_ssh_options(),
                        filepath=local_filepath,
                        user=cfg.ssh_user,
                        host=cfg.ssh_host,
                        dirname=remote_dir.dirname
                    )
                )

        # To have Megabits per second
        upload_speed = filesize / period_upload * 8
        download_speed = filesize / period_download * 8

        if upload_speed < req_bandwidth or download_speed < req_bandwidth:
            raise RuntimeError('Bandwidth is not OK. '
                               'Claimed bandwidth: %s Mb/s. '
                               'Required speed: %s Mb/s. '
                               'Actual upload speed: %.2f Mb/s. '
                               'Actual download speed: %.2f Mb/s. '
                               'Aborting migration...' %
                               (claimed_bandw,
                                req_bandwidth,
                                upload_speed,
                                download_speed))

        LOG.info("Bandwith is OK. "
                 "Required speed: %.2f Mb/s. "
                 "Upload speed: %.2f Mb/s. "
                 "Download speed: %.2f Mb/s",
                 req_bandwidth,
                 upload_speed,
                 download_speed)
Exemple #10
0
    def run(self, **kwargs):
        cfg = self.cloud.cloud_config.cloud
        runner = remote_runner.RemoteRunner(cfg.ssh_host, cfg.ssh_user)

        with files.LocalTempDir('check_band_XXXX') as local_dir:
            with files.RemoteTempDir(runner, 'check_band_XXXX') as remote_dir:
                filename = str(uuid.uuid4())
                local_filepath = os.path.join(local_dir.dirname, filename)
                remote_filepath = os.path.join(remote_dir.dirname, filename)
                claimed_bandw = self.cloud.cloud_config.initial_check.\
                    claimed_bandwidth
                filesize = self.cloud.cloud_config.initial_check.test_file_size
                factor = self.cloud.cloud_config.initial_check.factor
                req_bandwidth = claimed_bandw * factor

                scp_download = "scp {ssh_opts} {user}@{host}:{filepath} " \
                               "{dirname}"
                scp_upload = "scp {ssh_opts} {filepath} {user}@{host}:" \
                             "{dirname}"
                dd_command = "dd if=/dev/zero of={filepath} bs=1 count=0 " \
                             "seek={filesize}"
                runner.run(dd_command,
                           filepath=remote_filepath,
                           filesize=filesize)

                LOG.info("Checking download speed... Wait please.")
                period_download = utils.timer(
                    local.run,
                    scp_download.format(
                        ssh_opts=ssh_util.default_ssh_options(),
                        user=cfg.ssh_user,
                        host=cfg.ssh_host,
                        filepath=remote_filepath,
                        dirname=local_dir.dirname))

                LOG.info("Checking upload speed... Wait please.")
                period_upload = utils.timer(
                    local.run,
                    scp_upload.format(ssh_opts=ssh_util.default_ssh_options(),
                                      filepath=local_filepath,
                                      user=cfg.ssh_user,
                                      host=cfg.ssh_host,
                                      dirname=remote_dir.dirname))

        # To have Megabits per second
        upload_speed = filesize / period_upload * 8
        download_speed = filesize / period_download * 8

        if upload_speed < req_bandwidth or download_speed < req_bandwidth:
            raise RuntimeError(
                'Bandwidth is not OK. '
                'Claimed bandwidth: %s Mb/s. '
                'Required speed: %s Mb/s. '
                'Actual upload speed: %.2f Mb/s. '
                'Actual download speed: %.2f Mb/s. '
                'Aborting migration...' %
                (claimed_bandw, req_bandwidth, upload_speed, download_speed))

        LOG.info(
            "Bandwith is OK. "
            "Required speed: %.2f Mb/s. "
            "Upload speed: %.2f Mb/s. "
            "Download speed: %.2f Mb/s", req_bandwidth, upload_speed,
            download_speed)
Exemple #11
0
    def transfer(self, data):
        host_src = data['host_src']
        path_src = data['path_src']
        host_dst = data['host_dst']
        path_dst = data['path_dst']

        cmd = "{bbcp} {options} {src} {dst} 2>&1"

        options = CONF.bbcp.options
        additional_options = []
        # -f: forces the copy by first unlinking the target file before
        # copying.
        # -S: command to start bbcp on the source node.
        # -T: command to start bbcp on the target node.
        forced_options = ['-f']
        if CONF.migrate.copy_with_md5_verification:
            # -e: error check data for transmission errors using md5 checksum.
            forced_options.append('-e')
        for o in forced_options:
            if o not in options:
                additional_options.append(o)
        run_options = ['-T']
        if CONF.migrate.direct_transfer:
            src = path_src
            bbcp = BBCP_PATH
            runner = self.runner(host_src, 'src', data.get('gateway'))
            run = runner.run
        else:
            run_options += ['-S']
            src = '{user_src}@{host_src}:{path_src}'.format(
                user_src=CONF.src.ssh_user,
                host_src=host_src,
                path_src=path_src,
            )
            bbcp = CONF.bbcp.path
            run = local.run
        dst = '{user_dst}@{host_dst}:{path_dst}'.format(
            user_dst=CONF.dst.ssh_user,
            host_dst=host_dst,
            path_dst=path_dst,
        )
        for o in run_options:
            if o not in options:
                additional_options.append(o + " '{remote_bbcp}'")
        remote_bbcp = "ssh {ssh_opts} %I -l %U %H {path}".format(
            ssh_opts=ssh_util.default_ssh_options(),
            path=BBCP_PATH
        )
        options += ' ' + ' '.join(additional_options).format(
            remote_bbcp=remote_bbcp)

        retrier = retrying.Retry(
            max_attempts=CONF.migrate.retry,
            timeout=0
        )
        try:
            retrier.run(run, cmd.format(bbcp=bbcp, options=options, src=src,
                                        dst=dst),
                        capture_output=False)
        except retrying.MaxAttemptsReached:
            self.clean_dst(host_dst, path_dst)
            raise base.FileCopyError(**data)