def _disable_lvm2_lvmetad(ssh):
"""Disables lvm2-lvmetad service. This service is responsible
for automatically activating LVM2 volume groups when a disk is
attached or when a volume group is created. During disk replication
this service needs to be disabled.
"""
cfg = "/etc/lvm/lvm.conf"
if utils.test_ssh_path(ssh, cfg):
utils.exec_ssh_cmd(
ssh,
'sudo sed -i "s/use_lvmetad.*=.*1/use_lvmetad = 0/g" '
'%s' % cfg, get_pty=True)
# NOTE: lvm2-lvmetad is the name of the lvmetad service
# on both debian and RHEL based systems. It needs to be stopped
# before we begin disk replication. We disable it in the config
# just in case some other process starts the daemon later on, as
# a dependency. As the service may not actually exist, even though
# the config is present, we ignore errors when stopping it.
utils.ignore_exceptions(utils.exec_ssh_cmd)(
ssh, "sudo service lvm2-lvmetad stop", get_pty=True)
# disable volume groups. Any volume groups that have volumes in use
# will remain online. However, volume groups belonging to disks
# that have been synced at least once, will be deactivated.
utils.ignore_exceptions(utils.exec_ssh_cmd)(
ssh, "sudo vgchange -an", get_pty=True)
def _reload_and_start():
utils.exec_ssh_cmd(
ssh, "sudo systemctl daemon-reload",
get_pty=True)
utils.exec_ssh_cmd(
ssh, "sudo systemctl start replicator",
get_pty=True)
def _copy_writer(self, ssh):
local_path = os.path.join(
utils.get_resources_bin_dir(), _CORIOLIS_HTTP_WRITER_CMD)
remote_tmp_path = os.path.join("/tmp", _CORIOLIS_HTTP_WRITER_CMD)
with self._lock:
sftp = ssh.open_sftp()
try:
# Check if the remote file already exists
sftp.stat(self._writer_cmd)
except IOError as ex:
if ex.errno != errno.ENOENT:
raise
sftp.put(local_path, remote_tmp_path)
utils.exec_ssh_cmd(
ssh,
"sudo mv %s %s" % (
remote_tmp_path, self._writer_cmd),
get_pty=True
)
utils.exec_ssh_cmd(
ssh,
"sudo chmod +x %s" % self._writer_cmd,
get_pty=True
)
finally:
sftp.close()
def _exec_replicator(self, ssh, args, state_file):
ip = args["ip"]
# add 127.0.0.1 to the mix. If we need to tunnel, we will need it
cert_hosts = ",".join([ip, "127.0.0.1"])
port = args["port"]
self._config_dir = utils.exec_ssh_cmd(
ssh, "mktemp -d").decode().rstrip("\n")
utils.exec_ssh_cmd(
ssh,
"sudo chown %(user)s:%(user)s %(config_dir)s" % {
"config_dir": self._config_dir,
"user": REPLICATOR_USERNAME,
}, get_pty=True)
cmdline = ("/usr/bin/replicator -certificate-hosts=%(cert_hosts)s "
"-config-dir=%(cfgdir)s -hash-method=%(hash_method)s "
"-ignore-mounted-disks=%(ignore_mounted)s "
"-listen-port=%(listen_port)s "
"-chunk-size=%(chunk_size)s "
"-watch-devices=%(watch_devs)s "
"-state-file=%(state_file)s" % {
"cfgdir": self._config_dir,
"cert_hosts": cert_hosts,
"hash_method": self._hash_method,
"ignore_mounted": json.dumps(self._ignore_mounted),
"watch_devs": json.dumps(self._watch_devices),
"listen_port": str(port),
"state_file": state_file,
"chunk_size": self._chunk_size,
})
self._write_system_startup(ssh, cmdline)
def _copy_file(self, ssh, localPath, remotePath):
tmp = tempfile.mktemp(dir="/tmp")
sftp = paramiko.SFTPClient.from_transport(ssh.get_transport())
sftp.put(localPath, tmp)
utils.exec_ssh_cmd(
ssh, "sudo mv %s %s" % (tmp, remotePath), get_pty=True)
sftp.close()
def _fetch_remote_file(self, ssh, remote_file, local_file):
with open(local_file, 'wb') as fd:
utils.exec_ssh_cmd(
ssh,
"sudo chmod +r %s" % remote_file, get_pty=True)
data = utils.retry_on_error()(
utils.read_ssh_file)(ssh, remote_file)
fd.write(data)
def _setup_replicator_user(self, ssh):
user_exists = utils.exec_ssh_cmd(
ssh, "getent passwd replicator > /dev/null && echo 1 || echo 0")
if int(user_exists) == 0:
utils.exec_ssh_cmd(
ssh, "sudo useradd -m -s /bin/bash %s" % REPLICATOR_USERNAME,
get_pty=True)
utils.exec_ssh_cmd(
ssh, "sudo usermod -aG disk %s" % REPLICATOR_USERNAME,
get_pty=True)
def _write_file_sudo(self, chroot_path, content):
# NOTE: writes the file to a temp location due to permission issues
tmp_file = 'tmp/%s' % str(uuid.uuid4())
self._write_file(tmp_file, content)
self._exec_cmd_chroot("cp /%s /%s" % (tmp_file, chroot_path))
self._exec_cmd_chroot("rm /%s" % tmp_file)
utils.exec_ssh_cmd(self._ssh,
"sudo sync",
self._environment,
get_pty=True)
def _setup_replicator(self, ssh):
# copy the binary and execute it.
state_file = self._get_replicator_state_file()
self._copy_file(ssh, state_file, REPLICATOR_STATE)
utils.exec_ssh_cmd(
ssh, "sudo chmod 755 %s" % REPLICATOR_STATE, get_pty=True)
os.remove(state_file)
args = self._parse_replicator_conn_info(self._conn_info)
self._copy_replicator_cmd(ssh)
self._setup_replicator_user(ssh)
self._exec_replicator(ssh, args, REPLICATOR_STATE)
def _setup_replicator_user(self, ssh):
# check for and create replicator user:
user_exists = utils.exec_ssh_cmd(
ssh, "getent passwd %(user)s > /dev/null && echo 1 || echo 0" %
{"user": REPLICATOR_USERNAME})
if int(user_exists) == 0:
utils.exec_ssh_cmd(ssh,
"sudo useradd -m -s /bin/bash -g %s %s" %
(REPLICATOR_GROUP_NAME, REPLICATOR_USERNAME),
get_pty=True)
utils.exec_ssh_cmd(ssh,
"sudo usermod -aG disk %s" %
REPLICATOR_USERNAME,
get_pty=True)
def _write_upstart(self, ssh, cmdline):
serviceFilePath = "/etc/init/replicator.conf"
upstartService = UPSTART_TEMPLATE % {
"cmdline": cmdline,
}
utils.write_ssh_file(
ssh, '/tmp/replicator.conf', upstartService)
utils.exec_ssh_cmd(
ssh,
"sudo mv /tmp/replicator.conf %s" % serviceFilePath,
get_pty=True).decode().rstrip(
"\n")
utils.exec_ssh_cmd(ssh, "start replicator")
def _fetch_certificates(self):
"""
Fetch the client certificates
Returns a dict with paths to the certificates
{
"client_cert": "/tmp/tmp.RAA6wsQG4s/client-cert.pem",
"client_key": "/tmp/tmp.RAA6wsQG4s/client-key.pem",
"ca_cert": "/tmp/tmp.RAA6wsQG4s/ca-cert.pem",
}
"""
if self._cert_dir is None:
self._cert_dir = tempfile.mkdtemp()
clientCrt = os.path.join(self._cert_dir, "client-cert.pem")
clientKey = os.path.join(self._cert_dir, "client-key.pem")
caCert = os.path.join(self._cert_dir, "ca-cert.pem")
def progressf(curr, total):
LOG.debug("Copied %s/%s", curr, total)
if self._config_dir is None:
raise exception.CoriolisException(
"Not initialized. Run _setup_replicator().")
localCertZip = os.path.join(self._cert_dir, "client-creds.zip")
zipFile = os.path.join(
self._config_dir, "ssl/client/client-creds.zip")
utils.exec_ssh_cmd(
self._ssh, "sudo cp -f %s /tmp/creds.zip" % zipFile, get_pty=True)
utils.exec_ssh_cmd(
self._ssh, "sudo chmod +r /tmp/creds.zip", get_pty=True)
sftp = paramiko.SFTPClient.from_transport(
self._ssh.get_transport())
try:
sftp.get("/tmp/creds.zip", localCertZip, callback=progressf)
finally:
sftp.close()
zFile = zipfile.ZipFile(localCertZip)
zFile.extractall(path=self._cert_dir)
zFile.close()
os.remove(localCertZip)
return {
"client_cert": clientCrt,
"client_key": clientKey,
"ca_cert": caCert,
}
def _copy_file(self, ssh, localPath, remotePath):
tmp = os.path.join("/tmp", str(uuid.uuid4()))
sftp = paramiko.SFTPClient.from_transport(ssh.get_transport())
try:
# Check if the remote file already exists
sftp.stat(remotePath)
except IOError as ex:
if ex.errno != errno.ENOENT:
raise
sftp.put(localPath, tmp)
utils.exec_ssh_cmd(
ssh, "sudo mv %s %s" % (tmp, remotePath), get_pty=True)
finally:
sftp.close()
def _folder_exists(self, ssh, folder):
LOG.debug("Checking if %s exists" % folder)
exists = utils.exec_ssh_cmd(
ssh, '[ -d "%s" ] && echo 1 || echo 0' % folder)
if exists.decode().rstrip("\n") == "1":
return True
return False
def _setup_replicator(self, ssh):
# copy the binary, set up the service, generate certificates,
# start service
state_file = self._get_replicator_state_file()
self._copy_file(ssh, state_file, REPLICATOR_STATE)
utils.exec_ssh_cmd(ssh,
"sudo chmod 755 %s" % REPLICATOR_STATE,
get_pty=True)
os.remove(state_file)
args = self._parse_replicator_conn_info(self._conn_info)
self._copy_replicator_cmd(ssh)
self._setup_replicator_user(ssh)
certs = self._setup_certificates(ssh, args)
self._exec_replicator(ssh, args["port"], certs["remote"],
REPLICATOR_STATE)
return certs["local"]
def _copy_file(self, ssh, localPath, remotePath):
tmp = tempfile.mkstemp()[1]
try:
os.remove(tmp)
except BaseException:
pass
sftp = paramiko.SFTPClient.from_transport(ssh.get_transport())
try:
# Check if the remote file already exists
sftp.stat(remotePath)
except IOError as ex:
if ex.errno != errno.ENOENT:
raise
sftp.put(localPath, tmp)
utils.exec_ssh_cmd(
ssh, "sudo mv %s %s" % (tmp, remotePath), get_pty=True)
finally:
sftp.close()
def _setup_certificates(self, ssh):
remote_base_dir = "/etc/coriolis-writer"
ca_crt_name = "ca-cert.pem"
client_crt_name = "client-cert.pem"
client_key_name = "client-key.pem"
srv_crt_name = "srv-cert.pem"
srv_key_name = "srv-key.pem"
remote_ca_crt = os.path.join(remote_base_dir, ca_crt_name)
remote_client_crt = os.path.join(remote_base_dir, client_crt_name)
remote_client_key = os.path.join(remote_base_dir, client_key_name)
remote_srv_crt = os.path.join(remote_base_dir, srv_crt_name)
remote_srv_key = os.path.join(remote_base_dir, srv_key_name)
exist = []
for i in (remote_ca_crt, remote_client_crt, remote_client_key,
remote_srv_crt, remote_srv_key):
exist.append(utils.test_ssh_path(ssh, i))
if not all(exist):
utils.exec_ssh_cmd(ssh,
"sudo mkdir -p %s" % remote_base_dir,
get_pty=True)
utils.exec_ssh_cmd(
ssh,
"sudo %(writer_cmd)s generate-certificates -output-dir "
"%(cert_dir)s -certificate-hosts %(extra_hosts)s" % {
"writer_cmd": self._writer_cmd,
"cert_dir": remote_base_dir,
"extra_hosts": self._ip,
},
get_pty=True)
return {
"srv_crt": remote_srv_crt,
"srv_key": remote_srv_key,
"ca_crt": remote_ca_crt,
"client_crt": remote_client_crt,
"client_key": remote_client_key
}
def _write_systemd(self, ssh, cmdline):
serviceFilePath = "/lib/systemd/system/replicator.service"
def _reload_and_start():
utils.exec_ssh_cmd(
ssh, "sudo systemctl daemon-reload",
get_pty=True)
utils.exec_ssh_cmd(
ssh, "sudo systemctl start replicator",
get_pty=True)
systemdService = SYSTEMD_TEMPLATE % {
"cmdline": cmdline,
"username": REPLICATOR_USERNAME,
}
utils.write_ssh_file(
ssh, '/tmp/replicator.service', systemdService)
utils.exec_ssh_cmd(
ssh,
"sudo mv /tmp/replicator.service %s" % serviceFilePath,
get_pty=True).decode().rstrip("\n")
_reload_and_start()
def _exec_cmd(self, cmd, timeout=None):
if not timeout:
timeout = self._osdetect_operation_timeout
try:
return utils.exec_ssh_cmd(self._conn,
cmd,
environment=self._environment,
get_pty=True,
timeout=timeout)
except exception.MinionMachineCommandTimeout as ex:
raise exception.OSMorphingSSHOperationTimeout(
cmd=cmd, timeout=timeout) from ex
def _setup_replicator(self, ssh):
# copy the binary, set up the service, generate certificates,
# start service
state_file = self._get_replicator_state_file()
self._copy_file(ssh, state_file, REPLICATOR_STATE)
utils.exec_ssh_cmd(
ssh, "sudo chmod 755 %s" % REPLICATOR_STATE, get_pty=True)
os.remove(state_file)
args = self._parse_replicator_conn_info(self._conn_info)
self._copy_replicator_cmd(ssh)
group_existed = self._setup_replicator_group(
ssh, group_name=REPLICATOR_GROUP_NAME)
if not group_existed:
# NOTE: we must reconnect so that our user being added to the new
# Replicator group can take effect:
ssh = self._reconnect_ssh()
self._setup_replicator_user(ssh)
certs = self._setup_certificates(ssh, args)
self._exec_replicator(
ssh, args["port"], certs["remote"], REPLICATOR_STATE)
return certs["local"]
def _setup_replicator_group(self, ssh, group_name=REPLICATOR_GROUP_NAME):
""" Sets up a group with the given name and adds the
user we're connected as to it.
Returns True if the group already existed, else False.
"""
group_exists = utils.exec_ssh_cmd(
ssh,
"getent group %(group)s > /dev/null && echo 1 || echo 0" % {
"group": REPLICATOR_GROUP_NAME})
if int(group_exists) == 0:
utils.exec_ssh_cmd(
ssh, "sudo groupadd %s" % group_name, get_pty=True)
# NOTE: this is required in order for the user we connected
# as to be able to read the certs:
# NOTE2: the group change will only take effect after we reconnect:
utils.exec_ssh_cmd(
ssh, "sudo usermod -aG %s %s" % (
REPLICATOR_GROUP_NAME, self._conn_info['username']),
get_pty=True)
return int(group_exists) == 1
def run_user_script(self, user_script):
if len(user_script) == 0:
return
script_path = "/tmp/coriolis_user_script"
try:
utils.write_ssh_file(self._conn, script_path, user_script)
except Exception as err:
raise exception.CoriolisException(
"Failed to copy user script to target system.") from err
try:
utils.exec_ssh_cmd(self._conn,
"sudo chmod +x %s" % script_path,
get_pty=True)
utils.exec_ssh_cmd(self._conn,
'sudo "%s" "%s"' %
(script_path, self._os_root_dir),
get_pty=True)
except Exception as err:
raise exception.CoriolisException(
"Failed to run user script.") from err
def _inject_iptables_allow(self, ssh):
utils.exec_ssh_cmd(
ssh,
"sudo /sbin/iptables -I INPUT -p tcp --dport %s "
"-j ACCEPT" % self._writer_port, get_pty=True)
def _exec_cmd(self, cmd):
return utils.exec_ssh_cmd(self._ssh, cmd, self._environment)
def _setup_certificates(self, ssh, args):
# TODO(gsamfira): coriolis-replicator and coriolis-writer share
# the functionality of being able to generate certificates
# This will either be replaced with proper certificate management
# in Coriolis, and the needed files will be pushed to the services
# that need them (userdata or ssh), or the two applications will be
# merged into one, and we will deduplicate this functionallity.
remote_base_dir = REPLICATOR_DIR
ip = args["ip"]
ca_crt_name = "ca-cert.pem"
client_crt_name = "client-cert.pem"
client_key_name = "client-key.pem"
srv_crt_name = "srv-cert.pem"
srv_key_name = "srv-key.pem"
remote_ca_crt = os.path.join(remote_base_dir, ca_crt_name)
remote_client_crt = os.path.join(remote_base_dir, client_crt_name)
remote_client_key = os.path.join(remote_base_dir, client_key_name)
remote_srv_crt = os.path.join(remote_base_dir, srv_crt_name)
remote_srv_key = os.path.join(remote_base_dir, srv_key_name)
ca_crt = os.path.join(self._cert_dir, ca_crt_name)
client_crt = os.path.join(self._cert_dir, client_crt_name)
client_key = os.path.join(self._cert_dir, client_key_name)
exist = []
for i in (remote_ca_crt, remote_client_crt, remote_client_key,
remote_srv_crt, remote_srv_key):
exist.append(utils.test_ssh_path(ssh, i))
force_fetch = False
if not all(exist):
utils.exec_ssh_cmd(ssh,
"sudo mkdir -p %s" % remote_base_dir,
get_pty=True)
utils.exec_ssh_cmd(
ssh,
"sudo %(replicator_cmd)s gen-certs -output-dir "
"%(cert_dir)s -certificate-hosts %(extra_hosts)s" % {
"replicator_cmd": REPLICATOR_PATH,
"cert_dir": remote_base_dir,
"extra_hosts": ip,
},
get_pty=True)
utils.exec_ssh_cmd(
ssh,
"sudo chown -R %(user)s:%(group)s %(cert_dir)s" % {
"cert_dir": remote_base_dir,
"user": REPLICATOR_USERNAME,
"group": REPLICATOR_GROUP_NAME
},
get_pty=True)
utils.exec_ssh_cmd(ssh,
"sudo chmod -R g+r %(cert_dir)s" % {
"cert_dir": remote_base_dir,
},
get_pty=True)
force_fetch = True
exists = []
for i in (ca_crt, client_crt, client_key):
exists.append(os.path.isfile(i))
if not all(exists) or force_fetch:
# certificates either are missing, or have been regenerated
# on the writer worker. We need to fetch them.
self._fetch_remote_file(ssh, remote_ca_crt, ca_crt)
self._fetch_remote_file(ssh, remote_client_crt, client_crt)
self._fetch_remote_file(ssh, remote_client_key, client_key)
return {
"local": {
"client_cert": client_crt,
"client_key": client_key,
"ca_cert": ca_crt,
},
"remote": {
"srv_crt": remote_srv_crt,
"srv_key": remote_srv_key,
"ca_crt": remote_ca_crt,
},
}
def _copy_replicator_cmd(self, ssh):
local_path = os.path.join(utils.get_resources_bin_dir(), 'replicator')
self._copy_file(ssh, local_path, REPLICATOR_PATH)
utils.exec_ssh_cmd(ssh,
"sudo chmod +x %s" % REPLICATOR_PATH,
get_pty=True)
def _exec_cmd(self, cmd):
return utils.exec_ssh_cmd(self._ssh, cmd)
def _exec_cmd(self, cmd):
return utils.exec_ssh_cmd(self._ssh,
cmd,
self._environment,
get_pty=True)
def _exec_cmd(self, cmd):
return utils.exec_ssh_cmd(self._ssh, cmd)
def _read_remote_file_sudo(self, remote_path):
contents = utils.exec_ssh_cmd(
self._ssh, "sudo cat %s" % remote_path, get_pty=True)
return contents.decode()
def _setup_certificates(self, ssh):
remote_base_dir = "/etc/coriolis-writer"
ca_crt_name = "ca-cert.pem"
client_crt_name = "client-cert.pem"
client_key_name = "client-key.pem"
srv_crt_name = "srv-cert.pem"
srv_key_name = "srv-key.pem"
remote_ca_crt = os.path.join(remote_base_dir, ca_crt_name)
remote_client_crt = os.path.join(remote_base_dir, client_crt_name)
remote_client_key = os.path.join(remote_base_dir, client_key_name)
remote_srv_crt = os.path.join(remote_base_dir, srv_crt_name)
remote_srv_key = os.path.join(remote_base_dir, srv_key_name)
ca_crt = os.path.join(self._crt_dir, ca_crt_name)
client_crt = os.path.join(self._crt_dir, client_crt_name)
client_key = os.path.join(self._crt_dir, client_key_name)
exist = []
for i in (remote_ca_crt, remote_client_crt, remote_client_key,
remote_srv_crt, remote_srv_key):
exist.append(utils.test_ssh_path(ssh, i))
force_fetch = False
if not all(exist):
utils.exec_ssh_cmd(ssh,
"sudo mkdir -p %s" % remote_base_dir,
get_pty=True)
utils.exec_ssh_cmd(
ssh,
"sudo %(writer_cmd)s generate-certificates -output-dir "
"%(cert_dir)s -certificate-hosts %(extra_hosts)s" % {
"writer_cmd": self._writer_cmd,
"cert_dir": remote_base_dir,
"extra_hosts": self._ip,
},
get_pty=True)
force_fetch = True
exists = []
for i in (ca_crt, client_crt, client_key):
exists.append(os.path.isfile(i))
if not all(exists) or force_fetch:
# certificates either are missing, or have been regenerated
# on the writer worker. We need to fetch them.
self._fetch_remote_file(ssh, remote_ca_crt, ca_crt)
self._fetch_remote_file(ssh, remote_client_crt, client_crt)
self._fetch_remote_file(ssh, remote_client_key, client_key)
return {
"local": {
"client_crt": client_crt,
"client_key": client_key,
"ca_crt": ca_crt,
},
"remote": {
"srv_crt": remote_srv_crt,
"srv_key": remote_srv_key,
"ca_crt": remote_ca_crt,
},
}
