def copy_bbcp(self, host, position):
"""
Check that the bbcp is installed on the host otherwise copy bbcp to
the host.
:param host: Host to which the bbcp will be copied
:param position: src or dst
"""
runner = self.runner(host, position)
LOG.debug("Checking if bbcp is installed on '%s' host", host)
bbcp_installed = (_bbcp_installed(runner, 'bbcp')
or _bbcp_installed(runner, BBCP_PATH))
if bbcp_installed:
return
if position == 'src':
bbcp = CONF.bbcp.src_path
user = CONF.src.ssh_user
else:
bbcp = CONF.bbcp.dst_path
user = CONF.dst.ssh_user
LOG.debug("Copying %s to %s:%s", bbcp, host, BBCP_PATH)
scp_bbcp = "scp {ssh_opts} {bbcp} {user}@{host}:{tmp}"
local.run(scp_bbcp.format(ssh_opts=ssh_util.default_ssh_options(),
bbcp=bbcp,
user=user,
host=host,
tmp=BBCP_PATH),
capture_output=False)
chmod_bbcp = "chmod 777 {path}"
runner.run(chmod_bbcp, path=BBCP_PATH)
def transfer(self, data):
host_src = data['host_src']
path_src = data['path_src']
host_dst = data['host_dst']
path_dst = data['path_dst']
cmd = "{bbcp} {options} {src} {dst} 2>&1"
options = CONF.bbcp.options
additional_options = []
# -f: forces the copy by first unlinking the target file before
# copying.
# -S: command to start bbcp on the source node.
# -T: command to start bbcp on the target node.
forced_options = ['-f']
if CONF.migrate.copy_with_md5_verification:
# -e: error check data for transmission errors using md5 checksum.
forced_options.append('-e')
for o in forced_options:
if o not in options:
additional_options.append(o)
run_options = ['-T']
if CONF.migrate.direct_transfer:
src = path_src
bbcp = BBCP_PATH
runner = self.runner(host_src, 'src', data.get('gateway'))
run = runner.run
else:
run_options += ['-S']
src = '{user_src}@{host_src}:{path_src}'.format(
user_src=CONF.src.ssh_user,
host_src=host_src,
path_src=path_src,
)
bbcp = CONF.bbcp.path
run = local.run
dst = '{user_dst}@{host_dst}:{path_dst}'.format(
user_dst=CONF.dst.ssh_user,
host_dst=host_dst,
path_dst=path_dst,
)
for o in run_options:
if o not in options:
additional_options.append(o + " '{remote_bbcp}'")
remote_bbcp = "ssh {ssh_opts} %I -l %U %H {path}".format(
ssh_opts=ssh_util.default_ssh_options(), path=BBCP_PATH)
options += ' ' + ' '.join(additional_options).format(
remote_bbcp=remote_bbcp)
retrier = retrying.Retry(max_attempts=CONF.migrate.retry, timeout=0)
try:
retrier.run(run,
cmd.format(bbcp=bbcp,
options=options,
src=src,
dst=dst),
capture_output=False)
except retrying.MaxAttemptsReached:
self.clean_dst(host_dst, path_dst)
raise base.FileCopyError(**data)
def copy_bbcp(self, host, position):
"""
Check that the bbcp is installed on the host otherwise copy bbcp to
the host.
:param host: Host to which the bbcp will be copied
:param position: src or dst
"""
runner = self.runner(host, position)
LOG.debug("Checking if bbcp is installed on '%s' host", host)
bbcp_installed = (_bbcp_installed(runner, 'bbcp') or
_bbcp_installed(runner, BBCP_PATH))
if bbcp_installed:
return
if position == 'src':
bbcp = CONF.bbcp.src_path
user = CONF.src.ssh_user
else:
bbcp = CONF.bbcp.dst_path
user = CONF.dst.ssh_user
LOG.debug("Copying %s to %s:%s", bbcp, host, BBCP_PATH)
scp_bbcp = "scp {ssh_opts} {bbcp} {user}@{host}:{tmp}"
local.run(scp_bbcp.format(ssh_opts=ssh_util.default_ssh_options(),
bbcp=bbcp,
user=user,
host=host,
tmp=BBCP_PATH),
capture_output=False)
chmod_bbcp = "chmod 777 {path}"
runner.run(chmod_bbcp, path=BBCP_PATH)
def run_scp(self, host_src, path_src, host_dst, path_dst, gateway=None):
"""
Copy a file from source host to destination by scp.
:param host_src: Source host.
:param path_src: Path to a file on source host.
:param host_dst: Destination host.
:param path_dst: Path to a file on destination host.
:raise: FileCopyError in case the file was not copied by any reasons.
"""
cmd = "scp {ssh_opts} {src} {dst}"
if CONF.migrate.direct_transfer:
src = path_src
runner = self.runner(host_src, 'src', gateway)
run = runner.run
else:
# -3: Copies between two remote hosts are transferred through
# the local host.
src = '-3 {src_user}@{src_host}:{src_path}'.format(
src_user=CONF.src.ssh_user,
src_host=host_src,
src_path=path_src,
)
run = local.run
dst = '{dst_user}@{dst_host}:{dst_path}'.format(
dst_user=CONF.dst.ssh_user,
dst_host=host_dst,
dst_path=path_dst,
)
kwargs = {
'host_src': host_src,
'path_src': path_src,
'host_dst': host_dst,
'path_dst': path_dst,
'gateway': gateway
}
retrier = retrying.Retry(
max_attempts=CONF.migrate.retry,
predicate=self.verify,
predicate_kwargs=kwargs,
)
LOG.info("Copying file '%s' to '%s'", path_src, host_dst)
try:
retrier.run(run,
cmd.format(ssh_opts=ssh_util.default_ssh_options(),
src=src,
dst=dst),
capture_output=False)
except retrying.MaxAttemptsReached:
self.clean_dst(host_dst, path_dst)
raise base.FileCopyError(host_src=host_src,
path_src=path_src,
host_dst=host_dst,
path_dst=path_dst)
def run_scp(self, host_src, path_src, host_dst, path_dst, gateway=None):
"""
Copy a file from source host to destination by scp.
:param host_src: Source host.
:param path_src: Path to a file on source host.
:param host_dst: Destination host.
:param path_dst: Path to a file on destination host.
:raise: FileCopyError in case the file was not copied by any reasons.
"""
cmd = "scp {ssh_opts} {src} {dst}"
if CONF.migrate.direct_transfer:
src = path_src
runner = self.runner(host_src, 'src', gateway)
run = runner.run
else:
# -3: Copies between two remote hosts are transferred through
# the local host.
src = '-3 {src_user}@{src_host}:{src_path}'.format(
src_user=CONF.src.ssh_user,
src_host=host_src,
src_path=path_src,
)
run = local.run
dst = '{dst_user}@{dst_host}:{dst_path}'.format(
dst_user=CONF.dst.ssh_user,
dst_host=host_dst,
dst_path=path_dst,
)
kwargs = {'host_src': host_src,
'path_src': path_src,
'host_dst': host_dst,
'path_dst': path_dst,
'gateway': gateway}
retrier = retrying.Retry(
max_attempts=CONF.migrate.retry,
predicate=self.verify,
predicate_kwargs=kwargs,
)
LOG.info("Copying file '%s' to '%s'", path_src, host_dst)
try:
retrier.run(run, cmd.format(
ssh_opts=ssh_util.default_ssh_options(),
src=src,
dst=dst),
capture_output=False)
except retrying.MaxAttemptsReached:
self.clean_dst(host_dst, path_dst)
raise base.FileCopyError(host_src=host_src,
path_src=path_src,
host_dst=host_dst,
path_dst=path_dst)
def transfer(self, data):
host_src = data['host_src']
path_src = data['path_src']
host_dst = data['host_dst']
path_dst = data['path_dst']
cmd = ("rsync "
"--partial "
"--inplace "
"--perms "
"--times "
"--compress "
"--verbose "
"--progress "
"--rsh='ssh {ssh_opts}' "
"{path_src} "
"{user_dst}@{host_dst}:{path_dst}")
ssh_opts = ssh_util.default_ssh_options()
if CONF.migrate.direct_transfer:
remote_tunnel = None
else:
ssh_opts += " -p {port}".format(port=CONF.rsync.port)
remote_tunnel = RsyncRemoteTunnelOptions(host_dst)
host_dst = "localhost"
runner = self.runner(host_src,
'src',
data.get('gateway'),
remote_tunnel=remote_tunnel)
try:
runner.run_repeat_on_errors(cmd,
ssh_opts=ssh_opts,
path_src=path_src,
user_dst=CONF.dst.ssh_user,
host_dst=host_dst,
path_dst=path_dst)
except remote_runner.RemoteExecutionError:
self.clean_dst(host_dst, path_dst)
raise base.FileCopyError(**data)
def copy_bbcp(self, host, position):
"""
Check that the bbcp is installed on the host otherwise copy bbcp to
the host.
:param host: Host to which the bbcp will be copied
:param position: src or dst
"""
runner = self.runner(host, position)
LOG.debug("Checking if bbcp is installed on '%s' host", host)
cmd = "bbcp --help &>/dev/null"
try:
runner.run(cmd)
except remote_runner.RemoteExecutionError:
pass
else:
return
if position == 'src':
bbcp = CONF.bbcp.src_path
user = CONF.src.ssh_user
else:
bbcp = CONF.bbcp.dst_path
user = CONF.dst.ssh_user
LOG.debug("Copying %s to %s:%s", bbcp, host, BBCP_PATH)
cmd = "scp {ssh_opts} {bbcp} {user}@{host}:{tmp}"
local.run(cmd.format(ssh_opts=ssh_util.default_ssh_options(),
bbcp=bbcp,
user=user,
host=host,
tmp=BBCP_PATH),
capture_output=False)
cmd = "chmod 755 {path}"
runner.run(cmd, path=BBCP_PATH)
def transfer(self, data):
host_src = data['host_src']
path_src = data['path_src']
host_dst = data['host_dst']
path_dst = data['path_dst']
cmd = ("rsync "
"--partial "
"--inplace "
"--compress "
"--verbose "
"--progress "
"--rsh='ssh {ssh_opts}' "
"{path_src} "
"{user_dst}@{host_dst}:{path_dst}")
ssh_opts = ssh_util.default_ssh_options()
if CONF.migrate.direct_transfer:
remote_tunnel = None
else:
ssh_opts += " -p {port}".format(port=CONF.rsync.port)
remote_tunnel = RsyncRemoteTunnelOptions(host_dst)
host_dst = "localhost"
runner = self.runner(host_src, 'src', data.get('gateway'),
remote_tunnel=remote_tunnel)
try:
runner.run_repeat_on_errors(cmd,
ssh_opts=ssh_opts,
path_src=path_src,
user_dst=CONF.dst.ssh_user,
host_dst=host_dst,
path_dst=path_dst)
except remote_runner.RemoteExecutionError:
self.clean_dst(host_dst, path_dst)
raise base.FileCopyError(**data)
def run(self, **kwargs):
cfg = self.cloud.cloud_config.cloud
runner = remote_runner.RemoteRunner(cfg.ssh_host, cfg.ssh_user)
with files.LocalTempDir('check_band_XXXX') as local_dir:
with files.RemoteTempDir(runner, 'check_band_XXXX') as remote_dir:
filename = str(uuid.uuid4())
local_filepath = os.path.join(local_dir.dirname, filename)
remote_filepath = os.path.join(remote_dir.dirname, filename)
claimed_bandw = self.cloud.cloud_config.initial_check.\
claimed_bandwidth
filesize = self.cloud.cloud_config.initial_check.test_file_size
factor = self.cloud.cloud_config.initial_check.factor
req_bandwidth = claimed_bandw * factor
scp_download = "scp {ssh_opts} {user}@{host}:{filepath} " \
"{dirname}"
scp_upload = "scp {ssh_opts} {filepath} {user}@{host}:" \
"{dirname}"
dd_command = "dd if=/dev/zero of={filepath} bs=1 count=0 " \
"seek={filesize}"
runner.run(dd_command,
filepath=remote_filepath,
filesize=filesize)
LOG.info("Checking download speed... Wait please.")
period_download = utils.timer(
local.run,
scp_download.format(
ssh_opts=ssh_util.default_ssh_options(),
user=cfg.ssh_user,
host=cfg.ssh_host,
filepath=remote_filepath,
dirname=local_dir.dirname
)
)
LOG.info("Checking upload speed... Wait please.")
period_upload = utils.timer(
local.run,
scp_upload.format(
ssh_opts=ssh_util.default_ssh_options(),
filepath=local_filepath,
user=cfg.ssh_user,
host=cfg.ssh_host,
dirname=remote_dir.dirname
)
)
# To have Megabits per second
upload_speed = filesize / period_upload * 8
download_speed = filesize / period_download * 8
if upload_speed < req_bandwidth or download_speed < req_bandwidth:
raise RuntimeError('Bandwidth is not OK. '
'Claimed bandwidth: %s Mb/s. '
'Required speed: %s Mb/s. '
'Actual upload speed: %.2f Mb/s. '
'Actual download speed: %.2f Mb/s. '
'Aborting migration...' %
(claimed_bandw,
req_bandwidth,
upload_speed,
download_speed))
LOG.info("Bandwith is OK. "
"Required speed: %.2f Mb/s. "
"Upload speed: %.2f Mb/s. "
"Download speed: %.2f Mb/s",
req_bandwidth,
upload_speed,
download_speed)
def run(self, **kwargs):
cfg = self.cloud.cloud_config.cloud
runner = remote_runner.RemoteRunner(cfg.ssh_host, cfg.ssh_user)
with files.LocalTempDir('check_band_XXXX') as local_dir:
with files.RemoteTempDir(runner, 'check_band_XXXX') as remote_dir:
filename = str(uuid.uuid4())
local_filepath = os.path.join(local_dir.dirname, filename)
remote_filepath = os.path.join(remote_dir.dirname, filename)
claimed_bandw = self.cloud.cloud_config.initial_check.\
claimed_bandwidth
filesize = self.cloud.cloud_config.initial_check.test_file_size
factor = self.cloud.cloud_config.initial_check.factor
req_bandwidth = claimed_bandw * factor
scp_download = "scp {ssh_opts} {user}@{host}:{filepath} " \
"{dirname}"
scp_upload = "scp {ssh_opts} {filepath} {user}@{host}:" \
"{dirname}"
dd_command = "dd if=/dev/zero of={filepath} bs=1 count=0 " \
"seek={filesize}"
runner.run(dd_command,
filepath=remote_filepath,
filesize=filesize)
LOG.info("Checking download speed... Wait please.")
period_download = utils.timer(
local.run,
scp_download.format(
ssh_opts=ssh_util.default_ssh_options(),
user=cfg.ssh_user,
host=cfg.ssh_host,
filepath=remote_filepath,
dirname=local_dir.dirname))
LOG.info("Checking upload speed... Wait please.")
period_upload = utils.timer(
local.run,
scp_upload.format(ssh_opts=ssh_util.default_ssh_options(),
filepath=local_filepath,
user=cfg.ssh_user,
host=cfg.ssh_host,
dirname=remote_dir.dirname))
# To have Megabits per second
upload_speed = filesize / period_upload * 8
download_speed = filesize / period_download * 8
if upload_speed < req_bandwidth or download_speed < req_bandwidth:
raise RuntimeError(
'Bandwidth is not OK. '
'Claimed bandwidth: %s Mb/s. '
'Required speed: %s Mb/s. '
'Actual upload speed: %.2f Mb/s. '
'Actual download speed: %.2f Mb/s. '
'Aborting migration...' %
(claimed_bandw, req_bandwidth, upload_speed, download_speed))
LOG.info(
"Bandwith is OK. "
"Required speed: %.2f Mb/s. "
"Upload speed: %.2f Mb/s. "
"Download speed: %.2f Mb/s", req_bandwidth, upload_speed,
download_speed)
def transfer(self, data):
host_src = data['host_src']
path_src = data['path_src']
host_dst = data['host_dst']
path_dst = data['path_dst']
cmd = "{bbcp} {options} {src} {dst} 2>&1"
options = CONF.bbcp.options
additional_options = []
# -f: forces the copy by first unlinking the target file before
# copying.
# -S: command to start bbcp on the source node.
# -T: command to start bbcp on the target node.
forced_options = ['-f']
if CONF.migrate.copy_with_md5_verification:
# -e: error check data for transmission errors using md5 checksum.
forced_options.append('-e')
for o in forced_options:
if o not in options:
additional_options.append(o)
run_options = ['-T']
if CONF.migrate.direct_transfer:
src = path_src
bbcp = BBCP_PATH
runner = self.runner(host_src, 'src', data.get('gateway'))
run = runner.run
else:
run_options += ['-S']
src = '{user_src}@{host_src}:{path_src}'.format(
user_src=CONF.src.ssh_user,
host_src=host_src,
path_src=path_src,
)
bbcp = CONF.bbcp.path
run = local.run
dst = '{user_dst}@{host_dst}:{path_dst}'.format(
user_dst=CONF.dst.ssh_user,
host_dst=host_dst,
path_dst=path_dst,
)
for o in run_options:
if o not in options:
additional_options.append(o + " '{remote_bbcp}'")
remote_bbcp = "ssh {ssh_opts} %I -l %U %H {path}".format(
ssh_opts=ssh_util.default_ssh_options(),
path=BBCP_PATH
)
options += ' ' + ' '.join(additional_options).format(
remote_bbcp=remote_bbcp)
retrier = retrying.Retry(
max_attempts=CONF.migrate.retry,
timeout=0
)
try:
retrier.run(run, cmd.format(bbcp=bbcp, options=options, src=src,
dst=dst),
capture_output=False)
except retrying.MaxAttemptsReached:
self.clean_dst(host_dst, path_dst)
raise base.FileCopyError(**data)