def transfer(self, data):
host_src = data['host_src']
path_src = data['path_src']
host_dst = data['host_dst']
path_dst = data['path_dst']
cmd = "{bbcp} {options} {src} {dst} 2>&1"
options = CONF.bbcp.options
additional_options = []
# -f: forces the copy by first unlinking the target file before
# copying.
# -S: command to start bbcp on the source node.
# -T: command to start bbcp on the target node.
forced_options = ['-f']
if CONF.migrate.copy_with_md5_verification:
# -e: error check data for transmission errors using md5 checksum.
forced_options.append('-e')
for o in forced_options:
if o not in options:
additional_options.append(o)
run_options = ['-T']
if CONF.migrate.direct_transfer:
src = path_src
bbcp = BBCP_PATH
runner = self.runner(host_src, 'src', data.get('gateway'))
run = runner.run
else:
run_options += ['-S']
src = '{user_src}@{host_src}:{path_src}'.format(
user_src=CONF.src.ssh_user,
host_src=host_src,
path_src=path_src,
)
bbcp = CONF.bbcp.path
run = local.run
dst = '{user_dst}@{host_dst}:{path_dst}'.format(
user_dst=CONF.dst.ssh_user,
host_dst=host_dst,
path_dst=path_dst,
)
for o in run_options:
if o not in options:
additional_options.append(o + " '{remote_bbcp}'")
remote_bbcp = "ssh {ssh_opts} %I -l %U %H {path}".format(
ssh_opts=ssh_util.default_ssh_options(), path=BBCP_PATH)
options += ' ' + ' '.join(additional_options).format(
remote_bbcp=remote_bbcp)
retrier = retrying.Retry(max_attempts=CONF.migrate.retry, timeout=0)
try:
retrier.run(run,
cmd.format(bbcp=bbcp,
options=options,
src=src,
dst=dst),
capture_output=False)
except retrying.MaxAttemptsReached:
self.clean_dst(host_dst, path_dst)
raise base.FileCopyError(**data)
def run_scp(self, host_src, path_src, host_dst, path_dst, gateway=None):
"""
Copy a file from source host to destination by scp.
:param host_src: Source host.
:param path_src: Path to a file on source host.
:param host_dst: Destination host.
:param path_dst: Path to a file on destination host.
:raise: FileCopyError in case the file was not copied by any reasons.
"""
cmd = "scp {ssh_opts} {src} {dst}"
if CONF.migrate.direct_transfer:
src = path_src
runner = self.runner(host_src, 'src', gateway)
run = runner.run
else:
# -3: Copies between two remote hosts are transferred through
# the local host.
src = '-3 {src_user}@{src_host}:{src_path}'.format(
src_user=CONF.src.ssh_user,
src_host=host_src,
src_path=path_src,
)
run = local.run
dst = '{dst_user}@{dst_host}:{dst_path}'.format(
dst_user=CONF.dst.ssh_user,
dst_host=host_dst,
dst_path=path_dst,
)
kwargs = {
'host_src': host_src,
'path_src': path_src,
'host_dst': host_dst,
'path_dst': path_dst,
'gateway': gateway
}
retrier = retrying.Retry(
max_attempts=CONF.migrate.retry,
predicate=self.verify,
predicate_kwargs=kwargs,
)
LOG.info("Copying file '%s' to '%s'", path_src, host_dst)
try:
retrier.run(run,
cmd.format(ssh_opts=ssh_util.default_ssh_options(),
src=src,
dst=dst),
capture_output=False)
except retrying.MaxAttemptsReached:
self.clean_dst(host_dst, path_dst)
raise base.FileCopyError(host_src=host_src,
path_src=path_src,
host_dst=host_dst,
path_dst=path_dst)
def transfer(self, data):
host_src = data['host_src']
path_src = data['path_src']
host_dst = data['host_dst']
path_dst = data['path_dst']
gateway = data.get('gateway')
src_runner = self.runner(host_src, 'src', gateway)
dst_runner = self.runner(host_dst, 'dst', gateway)
block_size = CONF.migrate.ssh_chunk_size
file_size = files.remote_file_size_mb(src_runner, path_src)
num_blocks = int(math.ceil(float(file_size) / block_size))
src_temp_dir = os.path.join(os.path.basename(path_src), '.cf.copy')
dst_temp_dir = os.path.join(os.path.basename(path_dst), '.cf.copy')
partial_files = []
with files.RemoteDir(src_runner, src_temp_dir) as src_temp, \
files.RemoteDir(dst_runner, dst_temp_dir) as dst_temp:
for i in xrange(num_blocks):
part = os.path.basename(path_src) + '.part{i}'.format(i=i)
part_path = os.path.join(src_temp.dirname, part)
files.remote_split_file(src_runner, path_src, part_path, i,
block_size)
gzipped_path = files.remote_gzip(src_runner, part_path)
gzipped_filename = os.path.basename(gzipped_path)
dst_gzipped_path = os.path.join(dst_temp.dirname,
gzipped_filename)
self.run_scp(host_src, gzipped_path, host_dst,
dst_gzipped_path, gateway)
files.remote_unzip(dst_runner, dst_gzipped_path)
partial_files.append(os.path.join(dst_temp.dirname, part))
for i in xrange(num_blocks):
files.remote_join_file(dst_runner, path_dst, partial_files[i],
i, block_size)
if not self.verify(host_src, path_src, host_dst, path_dst, gateway):
self.clean_dst(host_dst, path_dst)
raise base.FileCopyError(**data)
def transfer(self, data):
host_src = data['host_src']
path_src = data['path_src']
host_dst = data['host_dst']
path_dst = data['path_dst']
cmd = ("rsync "
"--partial "
"--inplace "
"--perms "
"--times "
"--compress "
"--verbose "
"--progress "
"--rsh='ssh {ssh_opts}' "
"{path_src} "
"{user_dst}@{host_dst}:{path_dst}")
ssh_opts = ssh_util.default_ssh_options()
if CONF.migrate.direct_transfer:
remote_tunnel = None
else:
ssh_opts += " -p {port}".format(port=CONF.rsync.port)
remote_tunnel = RsyncRemoteTunnelOptions(host_dst)
host_dst = "localhost"
runner = self.runner(host_src,
'src',
data.get('gateway'),
remote_tunnel=remote_tunnel)
try:
runner.run_repeat_on_errors(cmd,
ssh_opts=ssh_opts,
path_src=path_src,
user_dst=CONF.dst.ssh_user,
host_dst=host_dst,
path_dst=path_dst)
except remote_runner.RemoteExecutionError:
self.clean_dst(host_dst, path_dst)
raise base.FileCopyError(**data)