def transfer(self, data):
src_host = data['host_src']
src_path = data['path_src']
dst_host = data['host_dst']
dst_path = data['path_dst']
gateway = data.get('gateway')
cmd = ("rsync "
"--partial "
"--inplace "
"--perms "
"--times "
"--compress "
"--verbose "
"--progress "
"--rsh='ssh {ssh_opts} {ssh_cipher}' "
"{source_file} "
"{dst_user}@{dst_host}:{dst_path}")
ssh_opts = " ".join(["-o {}".format(opt)
for opt in ["UserKnownHostsFile=/dev/null",
"StrictHostKeyChecking=no"]])
src_runner = self.runner(src_host, 'src', gateway=gateway)
try:
src_runner.run_repeat_on_errors(
cmd,
ssh_cipher=ssh_util.get_cipher_option(),
ssh_opts=ssh_opts,
source_file=src_path,
dst_user=CONF.dst.ssh_user,
dst_host=dst_host,
dst_path=dst_path)
except remote_runner.RemoteExecutionError:
self.clean_dst(data)
raise base.FileCopyError(**data)
def run(self, *args, **kwargs):
db_host = mysql_connector.get_db_host(self.cloud.cloud_config)
# dump mysql to file
# probably, we have to choose what databases we have to dump
# by default we dump all databases
options = ["--user={user}", "--opt", "--all-databases"]
if self.cloud.cloud_config.mysql.db_password:
options.append("--password={password}")
options = " ".join(options).format(
user=self.cloud.cloud_config.mysql.db_user,
password=self.cloud.cloud_config.mysql.db_password)
command = "mysqldump {options} > {path}".format(
options=options,
path=self.cloud.cloud_config.snapshot.snapshot_path)
LOG.info("dumping database with command '%s'", command)
self.cloud.ssh_util.execute(command, host_exec=db_host)
# copy dump file to host with cloudferry (for now just in case)
# in future we will store snapshot for every step of migration
key_string = ' -i '.join(self.cloud.config.migrate.key_filename)
context = {
'host_src': db_host,
'path_src': self.cloud.cloud_config.snapshot.snapshot_path,
'user_src': self.cloud.cloud_config.cloud.ssh_user,
'key': key_string,
'path_dst': self.cloud.cloud_config.snapshot.snapshot_path,
'cipher': ssh_util.get_cipher_option(),
}
command = ("scp {cipher} -o StrictHostKeyChecking=no -i {key} "
"{user_src}@{host_src}:{path_src} {path_dst}".format(
**context))
LOG.info("EXECUTING {command} local".format(command=command))
local(command)
return {}
def run_scp(self, runner, src_path, dst_host, dst_path):
"""
Copy a file from source host to destination by scp.
:param runner: Runner to run a command on source host.
:param src_path: Path to a file on source host.
:param dst_host: Destination host.
:param dst_path: Path to a file on destination host.
:raise: FileCopyError in case the file was not copied by any reasons.
"""
data = {'host_src': runner.host,
'path_src': src_path,
'host_dst': dst_host,
'path_dst': dst_path}
retrier = retrying.Retry(
max_attempts=CONF.migrate.retry,
predicate=self.verify,
predicate_kwargs={'data': data},
timeout=0,
)
LOG.info("Copying file '%s' to '%s'", src_path, dst_host)
cmd = "scp {cipher} -o {opts} {file} {user}@{host}:{path}"
try:
retrier.run(runner.run, cmd, opts='StrictHostKeyChecking=no',
file=src_path, user=CONF.dst.ssh_user, host=dst_host,
path=dst_path, cipher=ssh_util.get_cipher_option())
except retrying.MaxAttemptsReached:
raise base.FileCopyError(**data)
def remote_scp(runner, dst_user, src_path, dst_host, dst_path):
scp_file_to_dest = "scp {cipher} -o {opts} {file} {user}@{host}:{path}"
runner.run(scp_file_to_dest.format(
opts='StrictHostKeyChecking=no',
file=src_path,
user=dst_user,
path=dst_path,
host=dst_host,
cipher=ssh_util.get_cipher_option()))
def copy_bbcp(self, host, position):
"""
Check that the bbcp is installed on the host otherwise copy bbcp to
the host.
:param host: Host to which the bbcp will be copied
:param position: src or dst
"""
runner = self.runner(host, position)
LOG.debug("Checking if bbcp is installed on '%s' host", host)
cmd = "bbcp --help &>/dev/null"
try:
runner.run(cmd)
except remote_runner.RemoteExecutionError:
pass
else:
return
tmp_path = '/tmp/bbcp'
if position == 'src':
bbcp = CONF.bbcp.src_path
user = CONF.src.ssh_user
else:
bbcp = CONF.bbcp.dst_path
user = CONF.dst.ssh_user
LOG.debug("Copying %s to %s:/usr/local/bin/bbcp", bbcp, host)
cmd = "scp {cipher} -o {opts} {bbcp} {user}@{host}:{tmp}"
local.run(cmd.format(opts='StrictHostKeyChecking=no',
bbcp=bbcp,
user=user,
host=host,
tmp=tmp_path,
cipher=ssh_util.get_cipher_option()),
capture_output=False)
cmd = "mv {tmp} {path} && chmod 755 {path}"
runner.run(cmd, tmp=tmp_path, path=BBCP_PATH)
def transfer(self, data):
src_host = data['host_src']
src_path = data['path_src']
dst_host = data['host_dst']
dst_path = data['path_dst']
src_user = self.cfg.src.ssh_user
dst_user = self.cfg.dst.ssh_user
src_password = self.cfg.src.ssh_sudo_password
src_runner = remote_runner.RemoteRunner(src_host,
src_user,
password=src_password,
sudo=True)
ssh_cipher = ssh_util.get_cipher_option()
ssh_opts = ["UserKnownHostsFile=/dev/null", "StrictHostKeyChecking=no"]
rsync = ("rsync "
"--partial "
"--inplace "
"--perms "
"--times "
"--compress "
"--verbose "
"--progress "
"--rsh='ssh {ssh_opts} {ssh_cipher}' "
"{source_file} "
"{dst_user}@{dst_host}:{dst_path}").format(
ssh_cipher=ssh_cipher,
ssh_opts=" ".join(["-o {}".format(opt) for opt in ssh_opts]),
source_file=src_path,
dst_user=dst_user,
dst_host=dst_host,
dst_path=dst_path)
src_runner.run_repeat_on_errors(rsync)
def run(self, *args, **kwargs):
db_host = mysql_connector.get_db_host(self.cloud.cloud_config)
# dump mysql to file
# probably, we have to choose what databases we have to dump
# by default we dump all databases
options = ["--user={user}", "--opt", "--all-databases"]
if self.cloud.cloud_config.mysql.db_password:
options.append("--password={password}")
options = " ".join(options).format(
user=self.cloud.cloud_config.mysql.db_user,
password=self.cloud.cloud_config.mysql.db_password
)
command = "mysqldump {options} > {path}".format(
options=options,
path=self.cloud.cloud_config.snapshot.snapshot_path
)
LOG.info("dumping database with command '%s'", command)
self.cloud.ssh_util.execute(command, host_exec=db_host)
# copy dump file to host with cloudferry (for now just in case)
# in future we will store snapshot for every step of migration
key_string = ' -i '.join(self.cloud.config.migrate.key_filename)
context = {
'host_src': db_host,
'path_src': self.cloud.cloud_config.snapshot.snapshot_path,
'user_src': self.cloud.cloud_config.cloud.ssh_user,
'key': key_string,
'path_dst': self.cloud.cloud_config.snapshot.snapshot_path,
'cipher': ssh_util.get_cipher_option(),
}
command = (
"scp {cipher} -o StrictHostKeyChecking=no -i {key} "
"{user_src}@{host_src}:{path_src} {path_dst}".format(**context))
LOG.info("EXECUTING {command} local".format(command=command))
local(command)
return {}
def transfer(self, data):
src_host = data['host_src']
src_path = data['path_src']
dst_host = data['host_dst']
dst_path = data['path_dst']
gateway = data.get('gateway')
cmd = ("rsync "
"--partial "
"--inplace "
"--perms "
"--times "
"--compress "
"--verbose "
"--progress "
"--rsh='ssh {ssh_opts} {ssh_cipher}' "
"{source_file} "
"{dst_user}@{dst_host}:{dst_path}")
ssh_opts = " ".join([
"-o {}".format(opt) for opt in
["UserKnownHostsFile=/dev/null", "StrictHostKeyChecking=no"]
])
src_runner = self.runner(src_host, 'src', gateway=gateway)
try:
src_runner.run_repeat_on_errors(
cmd,
ssh_cipher=ssh_util.get_cipher_option(),
ssh_opts=ssh_opts,
source_file=src_path,
dst_user=CONF.dst.ssh_user,
dst_host=dst_host,
dst_path=dst_path)
except remote_runner.RemoteExecutionError:
self.clean_dst(data)
raise base.FileCopyError(**data)
def run(self, **kwargs):
cfg = self.cloud.cloud_config.cloud
runner = remote_runner.RemoteRunner(cfg.ssh_host, cfg.ssh_user)
temp_dir_name = os.popen('mktemp -dt check_band_XXXX').read().rstrip()
temp_file_name = str(uuid.uuid4())
claimed_bandw = self.cloud.cloud_config.initial_check.claimed_bandwidth
test_file_size = self.cloud.cloud_config.initial_check.test_file_size
ssh_user = self.cloud.cloud_config.cloud.ssh_user
factor = self.cloud.cloud_config.initial_check.factor
req_bandwidth = claimed_bandw * factor
local_file_path = os.path.join(temp_dir_name, temp_file_name)
remote_file_path = os.path.join(temp_dir_name, temp_file_name)
scp_upload = cmd_cfg.scp_cmd(ssh_util.get_cipher_option(), '',
ssh_user, cfg.ssh_host, remote_file_path,
temp_dir_name)
scp_download = cmd_cfg.scp_cmd(ssh_util.get_cipher_option(),
local_file_path, ssh_user, cfg.ssh_host,
temp_dir_name, '')
with files.RemoteDir(runner, temp_dir_name):
try:
with utils.forward_agent(env.key_filename):
dd_command = cmd_cfg.dd_full('/dev/zero', remote_file_path,
1, 0, test_file_size)
self.cloud.ssh_util.execute(dd_command)
LOG.info("Checking upload speed... Wait please.")
period_upload = utils.timer(subprocess.call,
str(scp_upload),
shell=True)
LOG.info("Checking download speed... Wait please.")
period_download = utils.timer(subprocess.call,
str(scp_download),
shell=True)
finally:
if len(temp_dir_name) > 1:
os.system("rm -rf {}".format(temp_dir_name))
else:
raise RuntimeError('Wrong dirname %s, stopping' %
temp_dir_name)
# To have Megabits per second
upload_speed = test_file_size / period_upload * 8
download_speed = test_file_size / period_download * 8
if upload_speed < req_bandwidth or download_speed < req_bandwidth:
raise RuntimeError(
'Bandwidth is not OK. '
'Claimed bandwidth: %s Mb/s. '
'Required speed: %s Mb/s. '
'Actual upload speed: %.2f Mb/s. '
'Actual download speed: %.2f Mb/s. '
'Aborting migration...' %
(claimed_bandw, req_bandwidth, upload_speed, download_speed))
LOG.info(
"Bandwith is OK. "
"Required speed: %.2f Mb/s. "
"Upload speed: %.2f Mb/s. "
"Download speed: %.2f Mb/s", req_bandwidth, upload_speed,
download_speed)
def run(self, **kwargs):
cfg = self.cloud.cloud_config.cloud
runner = remote_runner.RemoteRunner(cfg.ssh_host, cfg.ssh_user)
temp_dir_name = os.popen('mktemp -dt check_band_XXXX').read().rstrip()
temp_file_name = str(uuid.uuid4())
claimed_bandw = self.cloud.cloud_config.initial_check.claimed_bandwidth
test_file_size = self.cloud.cloud_config.initial_check.test_file_size
ssh_user = self.cloud.cloud_config.cloud.ssh_user
factor = self.cloud.cloud_config.initial_check.factor
req_bandwidth = claimed_bandw * factor
local_file_path = os.path.join(temp_dir_name, temp_file_name)
remote_file_path = os.path.join(temp_dir_name,
temp_file_name)
scp_upload = cmd_cfg.scp_cmd(ssh_util.get_cipher_option(),
'',
ssh_user,
cfg.ssh_host,
remote_file_path,
temp_dir_name)
scp_download = cmd_cfg.scp_cmd(ssh_util.get_cipher_option(),
local_file_path,
ssh_user,
cfg.ssh_host,
temp_dir_name,
'')
with files.RemoteDir(runner, temp_dir_name):
try:
with utils.forward_agent(env.key_filename):
dd_command = cmd_cfg.dd_full('/dev/zero',
remote_file_path,
1,
0,
test_file_size)
self.cloud.ssh_util.execute(dd_command)
LOG.info("Checking upload speed... Wait please.")
period_upload = utils.timer(subprocess.call,
str(scp_upload),
shell=True)
LOG.info("Checking download speed... Wait please.")
period_download = utils.timer(subprocess.call,
str(scp_download),
shell=True)
finally:
if len(temp_dir_name) > 1:
os.system("rm -rf {}".format(temp_dir_name))
else:
raise RuntimeError('Wrong dirname %s, stopping' %
temp_dir_name)
# To have Megabits per second
upload_speed = test_file_size / period_upload * 8
download_speed = test_file_size / period_download * 8
if upload_speed < req_bandwidth or download_speed < req_bandwidth:
raise RuntimeError('Bandwidth is not OK. '
'Claimed bandwidth: %s Mb/s. '
'Required speed: %s Mb/s. '
'Actual upload speed: %.2f Mb/s. '
'Actual download speed: %.2f Mb/s. '
'Aborting migration...' %
(claimed_bandw,
req_bandwidth,
upload_speed,
download_speed))
LOG.info("Bandwith is OK. "
"Required speed: %.2f Mb/s. "
"Upload speed: %.2f Mb/s. "
"Download speed: %.2f Mb/s",
req_bandwidth,
upload_speed,
download_speed)