def create_id_volume(self, nodes=None):
"""Create identification volume on the storage backend"""
# Obtain volume object for ID volume
volume = self.get_volume(self._id_volume_name)
# If the storage backend is shared, then only needs to be created on
# a single node (prefer local host if in list of nodes). If not shared,
# ID volume will be created on all nodes
if nodes is None:
nodes = ([get_hostname()]
if get_hostname() in self.nodes else
[self.nodes[0]]) if self.shared else self.nodes
try:
# Create ID volume
volume.create(size=2 ** 20, nodes=nodes)
except ExternalStorageCommandErrorException:
raise ExternalStorageCommandErrorException(
('An error occured whilst adding storage backend: Either the storage '
'backend has no free space (at least 1MB required) or '
'shared storage is being used, but has not been specified'))
try:
# Ensure that ID volume exists on all nodes
volume.ensure_exists(nodes=self.nodes)
except VolumeDoesNotExistError:
raise ExternalStorageCommandErrorException(
('An error ocurred whilst verifying the storage backend: '
'A shared storage has been specified, but it is not'))
def delete(self, local_only=False):
"""Delete the logical volume for the disk"""
vm_object = self.get_virtual_machine()
# Ensure that the user has permissions to add delete storage
self._get_registered_object('auth').assert_permission(
PERMISSIONS.MODIFY_HARD_DRIVE,
vm_object
)
# If the storage backend is available on one to more nodes,
# ensure that the hard drive exists. Prefer local host
check_node = get_hostname() if get_hostname() in self.nodes else self.nodes[0]
if self._is_cluster_master or check_node == get_hostname():
self.ensure_exists(nodes=[check_node])
if vm_object:
vm_object.ensureUnlocked()
vm_object.ensure_stopped()
# Remove the hard drive from the MCVirt VM configuration
self.get_attachment_object().delete(local_only=local_only)
# Remove backing storage
self._removeStorage(local_only=local_only)
# Remove config
nodes = [get_hostname()] if local_only else self._get_registered_object(
'cluster').get_nodes(include_local=True)
self.remove_config(nodes=nodes)
def pre_check_network(self, name, interface):
"""Perform pre-limiary checks on node before determining
that a network can be added"""
self._get_registered_object('auth').assert_user_type('ConnectionUser', 'ClusterUser')
# Ensure that the physical interface exists
self.assert_interface_exists(interface)
# Ensure that there are no interfaces present on the MCVirt instance
# that match the network
if self.check_exists(name):
raise NetworkAlreadyExistsException('Network already exists on node: %s %s' %
(name, get_hostname()))
# Ensure that there is not already a network with the same name defined in
# libvirt
try:
self._get_registered_object(
'libvirt_connector'
).get_connection().networkLookupByName(name)
# If the libvirt connect did not throw an error that
# the network does not exist, raise an exception
# as the network must be pressent
# @TODO: do this more nicely. Get list of networks and
# assert that it's not in the list
raise NetworkAlreadyExistsException(
'Network already defined in libvirt on node: %s %s' %
(name, get_hostname()))
except libvirtError:
pass
def undo(self):
"""Execute the undo method for the function"""
# If the local node is in the list of complete
# commands, then undo it first
if (get_hostname() in self.nodes and
self.nodes[get_hostname()]['complete'] and
hasattr(self.obj, self._undo_function_name)):
# Set current node
local_hostname = get_hostname()
self.current_node = local_hostname
Syslogger.logger().debug('Undo %s %s %s %s' %
(get_hostname(),
self._undo_function_name,
str(self.nodes[get_hostname()]['args']),
str(self.nodes[get_hostname()]['kwargs'])))
getattr(self.obj, self._undo_function_name)(
*self.nodes[get_hostname()]['args'],
**self.nodes[get_hostname()]['kwargs'])
# Iterate through nodes and undo
for node in self.nodes:
# Skip local node or if the function did not complete on the node
if node == get_hostname() or not self.nodes[node]['complete']:
continue
# Run the remote undo method
Syslogger.logger().debug('Undo %s %s %s %s' %
(node,
self.function.__name__,
str(self.nodes[node]['args']),
str(self.nodes[node]['kwargs'])))
self._call_function_remote(node=node, undo=True)
def delete_id_volume(self, nodes=None):
"""Delete the ID volume for the storage backend"""
# Obtain volume object for ID volume
volume = self.get_volume(self._id_volume_name)
# If the storage backend is shared, then only needs to be created on
# a single node (prefer local host if in list of nodes). If not shared,
# ID volume will be created on all nodes
if nodes is None:
nodes = ([get_hostname()]
if get_hostname() in self.nodes else
[self.nodes[0]]) if self.shared else self.nodes
# Create ID volume
volume.delete(nodes=nodes)
def get_remote_factory(self, node=None):
if node is None or node == get_hostname():
return self
node = self._get_registered_object('cluster').get_remote_node(node)
remote_factory = node.get_connection('iso_factory')
node.annotate_object(remote_factory)
return remote_factory
def gitRemove(self, message='', custom_file=None):
"""Remove and commits a configuration file"""
if self._checkGitRepo():
session_obj = self._get_registered_object('mcvirt_session')
username = ''
user = None
if session_obj:
try:
user = session_obj.get_proxy_user_object()
except UserDoesNotExistException:
pass
if user:
username = session_obj.get_proxy_user_object().get_username()
message += "\nUser: %s\nNode: %s" % (username, get_hostname())
try:
System.runCommand([self.GIT,
'rm',
'--cached',
self.config_file],
cwd=DirectoryLocation.BASE_STORAGE_DIR)
System.runCommand([self.GIT, 'commit', '-m', message],
cwd=DirectoryLocation.BASE_STORAGE_DIR)
System.runCommand([self.GIT,
'push'],
raise_exception_on_failure=False,
cwd=DirectoryLocation.BASE_STORAGE_DIR)
except Exception:
pass
def add_node_configuration(self, node_name, ip_address,
connection_user, connection_password,
ca_key):
"""Add MCVirt node to configuration, generates a cluster user on the remote node
and stores credentials against node in the MCVirt configuration.
"""
self._get_registered_object('auth').assert_permission(PERMISSIONS.MANAGE_CLUSTER)
# Create CA file
ssl_object = self._get_registered_object(
'certificate_generator_factory').get_cert_generator(node_name)
ssl_object.ca_pub_file = ca_key
# Connect to node and obtain cluster user
remote = Connection(username=connection_user, password=connection_password,
host=node_name)
remote_user_factory = remote.get_connection('user_factory')
connection_user = remote_user_factory.get_user_by_username(connection_user)
remote.annotate_object(connection_user)
username, password = connection_user.create_cluster_user(host=get_hostname())
# Add node to configuration file
def add_node_config(mcvirt_config):
mcvirt_config['cluster']['nodes'][node_name] = {
'ip_address': ip_address,
'username': username,
'password': password
}
MCVirtConfig().update_config(add_node_config)
def __init__(self):
"""Store required object member variables and create MCVirt object"""
# Initialise Pyro4 with flag to showing that the daemon is being started
Pyro4.current_context.STARTUP_PERIOD = True
# Store nameserver, MCVirt instance and create daemon
self.daemon_lock = DaemonLock()
Pyro4.config.USE_MSG_WAITALL = False
Pyro4.config.CREATE_SOCKET_METHOD = SSLSocket.create_ssl_socket
Pyro4.config.CREATE_BROADCAST_SOCKET_METHOD = SSLSocket.create_broadcast_ssl_socket
Pyro4.config.THREADPOOL_ALLOW_QUEUE = True
Pyro4.config.THREADPOOL_SIZE = 128
self.hostname = get_hostname()
# Ensure that the required SSL certificates exist
ssl_socket = CertificateGeneratorFactory().get_cert_generator('localhost')
ssl_socket.check_certificates(check_client=False)
ssl_socket = None
# Wait for nameserver
self.obtain_connection()
RpcNSMixinDaemon.DAEMON = BaseRpcDaemon(host=self.hostname)
self.register_factories()
# Ensure libvirt is configured
cert_gen_factory = RpcNSMixinDaemon.DAEMON.registered_factories[
'certificate_generator_factory']
cert_gen = cert_gen_factory.get_cert_generator('localhost')
cert_gen.check_certificates()
cert_gen = None
cert_gen_factory = None
def getAllVmNames(self, node=None):
"""Returns a list of all VMs within the cluster or those registered on a specific node"""
if node is not None:
ArgumentValidator.validate_hostname(node)
# If no node was defined, check the local configuration for all VMs
if node is None:
return [vm['name'] for vm in VirtualMachineConfig.get_global_config().values()]
elif node == get_hostname():
# @TODO - Why is this using libvirt?! Should use
# VM objects (i.e. config file to determine)
# and use a seperate function to get libvirt
# registered VMs
# Obtain array of all domains from libvirt
all_domains = self._get_registered_object(
'libvirt_connector').get_connection().listAllDomains()
return [vm.name() for vm in all_domains]
else:
# Return list of VMs registered on remote node
cluster = self._get_registered_object('cluster')
def remote_command(node_connection):
"""Get virtual machine names from remote node"""
virtual_machine_factory = node_connection.get_connection('virtual_machine_factory')
return virtual_machine_factory.getAllVmNames(node=node)
return cluster.run_remote_command(callback_method=remote_command, nodes=[node])[node]
def ensure_active(self):
"""Ensure that voljuem is active, otherwise, raise exception"""
if not self.is_active():
raise VolumeIsNotActiveException(
'Volume %s is not active on %s' %
(self.name, get_hostname())
)
def get_all(self, available_on_local_node=None, nodes=[], drbd=None,
storage_type=None, shared=None, nodes_predefined=False,
global_=None, default_location=None):
"""Return all storage backends, with optional filtering"""
storage_objects = []
for storage_id in self.get_config().keys():
# Obtain storage object
storage_object = self.get_object(storage_id)
# Check if storage backend is global or not
if global_ is not None and storage_object.is_global != global_:
continue
# Check if default location matches requested
if (default_location is not None and
storage_object.get_location(return_default=True) != default_location):
continue
# Check storage is available on local node
if (available_on_local_node and
get_hostname() not in storage_object.nodes):
continue
# If nodes are specified...
if nodes:
# Determine which nodes from the list are available
available_nodes = []
for node in nodes:
if node in storage_object.nodes:
available_nodes.append(node)
# If the list of nodes is required (predefined) and not all are
# present, skip storage backend
if nodes_predefined and len(nodes) != len(available_nodes):
continue
# Otherwise, (if the nodes are not required), ensure at least one
# is available, otherwise, skip
elif (not nodes_predefined) and not len(available_nodes):
continue
# If drbd is specified, ensure storage object is suitable to run DRBD
if drbd and not storage_object.is_drbd_suitable():
continue
# If storage_type is specified, ensure hat storage matches the object
if storage_type and not self.get_class(storage_type) != storage_object.__class__:
continue
# If a shared type is defined, determine if it matches the object, otherwise skip
if shared is not None and shared != storage_object.shared:
continue
# If all checks have passed, append to list of objects to return
storage_objects.append(storage_object)
return storage_objects
def node_pre_check(cls, cluster, libvirt_config, location):
"""Ensure volume group exists on node"""
try:
cls.ensure_exists(location)
except InvalidStorageConfiguration, exc:
raise InvalidStorageConfiguration(
'%s on node %s' % (str(exc), get_hostname())
)
def ensure_available(self):
"""Ensure that the storage backend is currently available on the node"""
volume = self.get_volume(self._id_volume_name)
if (not self.check_exists()) or not volume.check_exists():
raise StorageBackendNotAvailableOnNode(
'Storage backend %s is not currently avaialble on node: %s' % (
self.name, get_hostname()))
def insert_into_stat_db(self):
"""Add statistics to statistics database"""
db_factory = self._get_registered_object('database_factory')
db_rows = [
(StatisticsDeviceType.HOST.value, get_hostname(),
StatisticsStatType.CPU_USAGE.value, self._cpu_usage,
"{:%s}".format(datetime.now())),
(StatisticsDeviceType.HOST.value, get_hostname(),
StatisticsStatType.MEMORY_USAGE.value, self._memory_usage,
"{:%s}".format(datetime.now()))
]
with db_factory.get_locking_connection() as db_inst:
db_inst.cursor.executemany(
"""INSERT INTO stats(
device_type, device_id, stat_type, stat_value, stat_date
) VALUES(?, ?, ?, ?, ?)""",
db_rows)
def _get_response_data(self):
"""Determine and return response data"""
# Return dict of node -> output, if a dict response was
# specified
if self.return_dict:
return {node: self.nodes[node]['return_val']
for node in self.nodes.keys()}
# Otherwise, default to returning data from local node
elif get_hostname() in self.nodes:
return self.nodes[get_hostname()]['return_val']
# Otherwise, return the response from the first found node.
elif self.nodes:
return self.nodes[self.nodes.keys()[0]]['return_val']
# Otherwise, if no node data, return None
return None
def generate_connection_info(self):
"""Generate required information to connect to this node from a remote node"""
# Ensure user has required permissions
self._get_registered_object('auth').assert_permission(
PERMISSIONS.MANAGE_CLUSTER
)
# Ensure that the IP address configurations has been made correctly
self.check_ip_configuration()
# Create connection user
user_factory = self._get_registered_object('user_factory')
connection_username, connection_password = user_factory.generate_user(ConnectionUser)
ssl_object = self._get_registered_object(
'certificate_generator_factory').get_cert_generator(get_hostname())
return [get_hostname(), self.get_cluster_ip_address(),
connection_username, connection_password,
ssl_object.get_ca_contents()]
def __init__(self, server=None, remote=False):
"""Store member variables and ensure that openSSL is installed"""
if not os.path.isfile(self.OPENSSL):
raise OpenSSLNotFoundException('openssl not found: %s' % self.OPENSSL)
if server == 'localhost' or server.startswith('127.') or server is None:
self.server = get_hostname()
else:
self.server = server
self.remote = remote
def __init__(self):
"""Store required object member variables and create MCVirt object"""
# Before doing ANYTHING, ensure that the hostname that MCVirt thinks the
# machine is (i.e. the hostname that the machine was already setup as)
# matches the current hostname of the machine
ensure_hostname_consistent()
# Initialise Pyro4 with flag to showing that the daemon is being started
Pyro4.current_context.STARTUP_PERIOD = True
# Store nameserver, MCVirt instance and create daemon
self.daemon_lock = DaemonLock()
self.timer_objects = []
Pyro4.config.USE_MSG_WAITALL = False
Pyro4.config.CREATE_SOCKET_METHOD = SSLSocket.create_ssl_socket
Pyro4.config.CREATE_BROADCAST_SOCKET_METHOD = SSLSocket.create_broadcast_ssl_socket
Pyro4.config.THREADPOOL_ALLOW_QUEUE = True
Pyro4.config.THREADPOOL_SIZE = 128
self.hostname = get_hostname()
# Ensure that the required SSL certificates exist
ssl_socket = CertificateGeneratorFactory().get_cert_generator('localhost')
ssl_socket.check_certificates(check_client=False)
ssl_socket = None
# Wait for nameserver
Syslogger.logger().debug('Wait for connection to nameserver')
self.obtain_connection()
Syslogger.logger().debug('Obtained nameserver connection')
RpcNSMixinDaemon.DAEMON = BaseRpcDaemon(host=self.hostname)
self.register_factories()
# Ensure libvirt is configured
Syslogger.logger().debug('Start certificate check')
cert_gen_factory = RpcNSMixinDaemon.DAEMON.registered_factories[
'certificate_generator_factory']
cert_gen = cert_gen_factory.get_cert_generator('localhost')
cert_gen.check_certificates()
cert_gen = None
cert_gen_factory = None
Syslogger.logger().debug('Register atexit')
atexit.register(self.shutdown, 'atexit', '')
for sig in (signal.SIGABRT, signal.SIGILL, signal.SIGINT,
signal.SIGSEGV, signal.SIGTERM):
signal.signal(sig, self.shutdown)
Syslogger.logger().debug('Initialising objects')
for registered_object in RpcNSMixinDaemon.DAEMON.registered_factories:
obj = RpcNSMixinDaemon.DAEMON.registered_factories[registered_object]
if type(obj) is not types.TypeType: # noqa
Syslogger.logger().debug('Initialising object %s' % registered_object)
obj.initialise()
def test_migrate_pre_migration_libvirt_failure(self):
"""Simulates a pre-migration libvirt failure"""
# Set the mcvirt libvirt failure mode to simulate a pre-migration failure
VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.PRE_MIGRATION_FAILURE
# Attempt to perform a migration
with self.assertRaises(LibvirtFailureSimulationException):
self.test_vm_object.onlineMigrate(self.local_vm_object._get_remote_nodes()[0])
VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.NORMAL_RUN
# Ensure the VM is still registered on the local node and in a running state
self.assertEqual(self.local_vm_object.getNode(), get_hostname())
self.assertEqual(self.local_vm_object.get_power_state(), PowerStates.RUNNING.value)
# Ensure that the VM is registered with the local libvirt instance and not on the remote
# libvirt instance
self.assertTrue(
self.local_vm_object.get_name() in
self.vm_factory.getAllVmNames(node=get_hostname())
)
self.assertFalse(
self.local_vm_object.get_name() in
self.vm_factory.getAllVmNames(node=self.local_vm_object._get_remote_nodes()[0])
)
# Ensure Drbd disks are in a valid state
for disk_object in self.local_vm_object.get_hard_drive_objects():
# Check that the disk is shown as not in-sync
with self.assertRaises(DrbdVolumeNotInSyncException):
disk_object._checkDrbdStatus()
# Reset disk sync status and re-check status to ensure
# the disk is otherwise in a valid state
disk_object.setSyncState(True)
disk_object._checkDrbdStatus()
# Ensure that the local and remote disks are in the correct Drbd role
local_role, remote_role = disk_object._drbdGetRole()
self.assertEqual(local_role, DrbdRoleState.PRIMARY)
self.assertEqual(remote_role, DrbdRoleState.SECONDARY)
def check_ip_configuration(self):
"""Perform various checks to ensure that the
IP configuration is such that is suitable to be part of a cluster
"""
# Ensure that the cluster IP address has been defined
cluster_ip = self.get_cluster_ip_address()
if not cluster_ip:
raise MissingConfigurationException('IP address has not yet been configured')
# Ensure that the hostname of the local machine does not resolve
# to 127.0.0.1
if socket.gethostbyname(get_hostname()).startswith('127.'):
raise MissingConfigurationException(('Node hostname %s resolves to the localhost.'
' Instead it should resolve to the cluster'
' IP address'))
resolve_ip = socket.gethostbyname(get_hostname())
if resolve_ip != cluster_ip:
raise MissingConfigurationException(('The local hostname (%s) should resolve the'
' cluster IP address (%s). Instead it resolves'
' to \'%s\'. Please correct this issue before'
' continuing.') %
(get_hostname(), cluster_ip, resolve_ip))
def get_all_drbd_hard_drive_object(self, include_remote=False):
"""Obtain all hard drive objects that are backed by DRBD"""
hard_drive_objects = []
vm_factory = self._get_registered_object('virtual_machine_factory')
for vm_object in vm_factory.getAllVirtualMachines():
if (get_hostname() in vm_object.getAvailableNodes() or include_remote):
all_hard_drive_objects = vm_object.getHardDriveObjects()
for hard_drive_object in all_hard_drive_objects:
if (hard_drive_object.get_type() is 'Drbd'):
hard_drive_objects.append(hard_drive_object)
return hard_drive_objects
def available_on_node(self, node=None, raise_on_err=True):
"""Determine if the storage volume is available on
a given node
"""
if node is None:
node = get_hostname()
available = (node in self.nodes)
if not available and raise_on_err:
raise StorageBackendNotAvailableOnNode(
'Storage not available on node: %s, %s' % (self.name, node)
)
return available
def test_migrate_libvirt_connection_failure(self):
"""Attempt to perform a migration, simulating a libvirt
connection failure"""
VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.CONNECTION_FAILURE
with self.assertRaises(libvirt.libvirtError):
self.test_vm_object.onlineMigrate(self.local_vm_object._get_remote_nodes()[0])
VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.NORMAL_RUN
# Ensure the VM is still registered on the local node and in a running state
self.assertEqual(self.local_vm_object.getNode(), get_hostname())
self.assertEqual(self.local_vm_object.get_power_state(), PowerStates.RUNNING)
def in_use(self):
"""Determine if the ISO is currently in use by a VM"""
virtual_machine_factory = self._get_registered_object('virtual_machine_factory')
for vm_name in virtual_machine_factory.getAllVmNames(node=get_hostname()):
vm_object = virtual_machine_factory.getVirtualMachineByName(vm_name)
disk_drive_object = vm_object.get_disk_drive()
vm_current_iso = disk_drive_object.getCurrentDisk()
# If the VM has an iso attached, check if the ISO is this one
if vm_current_iso and vm_current_iso.get_path() == self.get_path():
return vm_object.get_name()
return False
def _undo_function_name(self):
"""Return the name of the undo function"""
# If running on a remote node and a remote undo method
# is defined, return that
if self.current_node != get_hostname() and self.remote_undo_method:
return self.remote_undo_method
# Otherwise, if a custom undo method is defined (for all nodes)
# return that
if self.undo_method:
return self.undo_method
# Otherwise, return default undo name for method
return 'undo__%s' % self.function.__name__
def delete(self, local_only=False):
"""Remove the hard drive attachment"""
# Ensure that the user has permissions to add delete storage
self._get_registered_object('auth').assert_permission(
PERMISSIONS.MODIFY_VM,
self.virtual_machine
)
self.remove_from_virtual_machine()
cluster = self._get_registered_object('cluster')
nodes = [get_hostname()] if local_only else cluster.get_nodes(include_local=True)
self.remove_config(nodes=cluster.get_nodes(include_local=True))
del Factory.CACHED_OBJECTS[(self.virtual_machine.id_, self.attachment_id)]
self.unregister_object()
def get_config(self):
"""Create the configuration for libvirt"""
cert_gen_factory = self._get_registered_object('certificate_generator_factory')
ssl_socket = cert_gen_factory.get_cert_generator('localhost')
nodes = self._get_registered_object('cluster').get_nodes(return_all=True)
nodes.append(get_hostname())
allowed_dns = [cert_gen_factory.get_cert_generator(node).ssl_subj for node in nodes]
return {
'ip_address': MCVirtConfig().getListenAddress(),
'ssl_server_key': ssl_socket.server_key_file,
'ssl_server_cert': ssl_socket.server_pub_file,
'ssl_ca_cert': ssl_socket.ca_pub_file,
'allowed_nodes': '", "'.join(allowed_dns)
}
def test_migrate_libvirt_connection_failure(self):
"""Attempt to perform a migration, simulating a libvirt
connection failure"""
VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.CONNECTION_FAILURE
with self.assertRaises(libvirt.libvirtError):
self.test_vm_object.onlineMigrate(
self.local_vm_object._get_remote_nodes()[0])
VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.NORMAL_RUN
# Ensure the VM is still registered on the local node and in a running state
self.assertEqual(self.local_vm_object.getNode(), get_hostname())
self.assertEqual(self.local_vm_object.getPowerState(),
PowerStates.RUNNING)
def get_connection(self, server=None):
"""Obtains a Libvirt connection for a given server"""
if server is None:
server = get_hostname()
ssl_object = self._get_registered_object(
'certificate_generator_factory').get_cert_generator(server)
libvirt_url = 'qemu://%s/system?pkipath=%s' % (
ssl_object.server, ssl_object.ssl_directory)
connection = libvirt.open(libvirt_url)
if connection is None:
raise LibVirtConnectionException(
'Failed to open connection to the hypervisor on %s' %
ssl_object.server)
return connection
def get_config(self):
"""Create the configuration for libvirt"""
cert_gen_factory = self._get_registered_object('certificate_generator_factory')
ssl_socket = cert_gen_factory.get_cert_generator('localhost')
nodes = self._get_registered_object('cluster').get_nodes(return_all=True)
nodes.append(get_hostname())
allowed_dns = [cert_gen_factory.get_cert_generator(node).ssl_subj for node in nodes]
return {
'ip_address': MCVirtConfig().getListenAddress(),
'ssl_server_key': ssl_socket.server_key_file,
'ssl_server_cert': ssl_socket.server_pub_file,
'ssl_ca_cert': ssl_socket.ca_pub_file,
'allowed_nodes': '", "'.join(allowed_dns)
}
def in_use(self):
"""Determine if the ISO is currently in use by a VM"""
virtual_machine_factory = self._get_registered_object(
'virtual_machine_factory')
for vm_name in virtual_machine_factory.getAllVmNames(
node=get_hostname()):
vm_object = virtual_machine_factory.getVirtualMachineByName(
vm_name)
disk_drive_object = vm_object.get_disk_drive()
vm_current_iso = disk_drive_object.getCurrentDisk()
# If the VM has an iso attached, check if the ISO is this one
if vm_current_iso and vm_current_iso.get_path() == self.get_path():
return vm_object.get_name()
return False
def resync(self, source_node=None, auto_determine=False):
"""Perform a resync of a Drbd hard drive"""
# Ensure user has privileges to create a Drbd volume
self._get_registered_object('auth').assert_permission(
PERMISSIONS.MANAGE_DRBD, self.vm_object)
if source_node:
if source_node not in self.vm_object.getAvailableNodes():
raise InvalidNodesException('Invalid node name')
if auto_determine:
raise TooManyParametersException(
'Only one of source_node an auto_determine should be specified'
)
else:
if not auto_determine:
raise ArgumentParserException(
'Either source_node or auto_determine must be specified'
)
elif self.vm_object.getNode():
source_node = self.vm_object.getNode()
else:
raise VmNotRegistered('Cannot auto-determine node - VM is not registered')
# Check Drbd state of disk
if self._drbdGetConnectionState() != DrbdConnectionState.CONNECTED:
raise DrbdStateException(
'Drbd resource must be connected before performing a resync: %s' %
self.resource_name)
if source_node == get_hostname():
System.runCommand([NodeDrbd.DrbdADM, 'invalidate-remote',
self.resource_name])
# Monitor the Drbd status, until the VM has started syncing
while True:
if self._drbdGetConnectionState() == DrbdConnectionState.SYNC_SOURCE:
break
time.sleep(5)
# Monitor the Drbd status, until the VM has finished syncing
while True:
if self._drbdGetConnectionState() != DrbdConnectionState.SYNC_SOURCE:
break
time.sleep(5)
elif not self._cluster_disable:
remote_object = self.get_remote_object(remote_node=source_node)
remote_object.resync(source_node=source_node)
def get_location(self, node=None, return_default=False):
"""Return the location for a given node, default to local node"""
# Default node to local node
if node is None:
node = get_hostname()
# Raise exception if node is not configured for storage backend
if node not in self.nodes:
raise UnsuitableNodeException(
'Node does not support storage backend: %s, %s' % (node, self.name)
)
config = self.get_config()
return (config['nodes'][node]['location']
if 'location' in config['nodes'][node] and
config['nodes'][node]['location'] and
not return_default
else config['location'])
def gitRemove(self, message=''):
"""Remove and commits a configuration file"""
from auth.session import Session
if self._checkGitRepo():
message += "\nUser: %s\nNode: %s" % (
Session.get_current_user_object().get_username(),
get_hostname())
try:
System.runCommand(
[self.GIT, 'rm', '--cached', self.config_file],
cwd=DirectoryLocation.BASE_STORAGE_DIR)
System.runCommand([self.GIT, 'commit', '-m', message],
cwd=DirectoryLocation.BASE_STORAGE_DIR)
System.runCommand([self.GIT, 'push'],
raise_exception_on_failure=False,
cwd=DirectoryLocation.BASE_STORAGE_DIR)
except:
pass
def gitAdd(self, message=''):
"""Commit changes to an added or modified configuration file"""
from auth.session import Session
if (self._checkGitRepo()):
message += "\nUser: %s\nNode: %s" % (
Session.get_current_user_object().get_username(),
get_hostname())
try:
System.runCommand([self.GIT, 'add', self.config_file],
cwd=DirectoryLocation.BASE_STORAGE_DIR)
System.runCommand(
[self.GIT, 'commit', '-m', message, self.config_file],
cwd=DirectoryLocation.BASE_STORAGE_DIR)
System.runCommand([self.GIT, 'push'],
raise_exception_on_failure=False,
cwd=DirectoryLocation.BASE_STORAGE_DIR)
except:
pass
def _generateDrbdConfig(self):
"""Generates the Drbd resource configuration"""
# Create configuration for use with the template
raw_lv_path = self._getLogicalVolumePath(self._getLogicalVolumeName(self.Drbd_RAW_SUFFIX))
meta_lv_path = self._getLogicalVolumePath(
self._getLogicalVolumeName(
self.Drbd_META_SUFFIX))
drbd_config = \
{
'resource_name': self.resource_name,
'block_device_path': self._getDrbdDevice(),
'raw_lv_path': raw_lv_path,
'meta_lv_path': meta_lv_path,
'drbd_port': self.drbd_port,
'nodes': []
}
# Add local node to the Drbd config
cluster_object = self._get_registered_object('cluster')
node_template_conf = \
{
'name': get_hostname(),
'ip_address': cluster_object.get_cluster_ip_address()
}
drbd_config['nodes'].append(node_template_conf)
# Add remote nodes to Drbd config
for node in self.vm_object._get_remote_nodes():
node_config = cluster_object.get_node_config(node)
node_template_conf = \
{
'name': node,
'ip_address': node_config['ip_address']
}
drbd_config['nodes'].append(node_template_conf)
# Replace the variables in the template with the local Drbd configuration
config_content = Template(file=self.Drbd_CONFIG_TEMPLATE, searchList=[drbd_config])
# Write the Drbd configuration
fh = open(self._getDrbdConfigFile(), 'w')
fh.write(config_content.respond())
fh.close()
def add_from_url(self, url, name=None, node=None):
"""Download an ISO from given URL and save in ISO directory"""
self._get_registered_object('auth').assert_permission(
PERMISSIONS.MANAGE_ISO)
if node is not None and node != get_hostname():
remote_node = self._get_registered_object(
'cluster').get_remote_node(node)
remote_factory = remote_node.get_connection('iso_factory')
remote_iso = remote_factory.add_from_url(url=url, name=name)
remote_node.annotate_object(remote_iso)
return remote_iso.get_name()
# Work out name from URL if name is not supplied
if name is None:
# Parse URL to get path part
url_parse = urlparse.urlparse(url)
name = Iso.get_filename_from_path(url_parse.path)
# Get temporary directory to store ISO
temp_directory = tempfile.mkdtemp()
output_path = temp_directory + '/' + name
# Open file
iso = urllib2.urlopen(url)
# Read file in 16KB chunks
chunk_size = 16 * 1024
# Save ISO
with open(output_path, 'wb') as file:
while True:
chunk = iso.read(chunk_size)
if not chunk:
break
file.write(chunk)
iso.close()
iso_object = self.add_iso(output_path)
os.remove(output_path)
os.rmdir(temp_directory)
return iso_object.get_name()
def print_info(self):
"""Print information about the nodes in the cluster"""
table = Texttable()
table.set_deco(Texttable.HEADER | Texttable.VLINES)
table.header(('Node', 'IP Address', 'Status'))
# Add this node to the table
table.add_row((get_hostname(), self.get_cluster_ip_address(), 'Local'))
# Add remote nodes
for node in self.get_nodes(return_all=True):
node_config = self.get_node_config(node)
node_status = 'Unreachable'
try:
self.get_remote_node(node)
node_status = 'Connected'
except CouldNotConnectToNodeException:
pass
table.add_row((node, node_config['ip_address'], node_status))
return table.draw()
def test_migrate_post_migration_libvirt_failure(self):
"""Simulates a post-migration libvirt failure"""
# Set the mcvirt libvirt failure mode to simulate a post-migration failure
VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.POST_MIGRATION_FAILURE
# Attempt to perform a migration
with self.assertRaises(LibvirtFailureSimulationException):
self.test_vm_object.onlineMigrate(
self.local_vm_object._get_remote_nodes()[0])
VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.NORMAL_RUN
# Ensure the VM is still registered on the remote node and in a running state
self.assertEqual(self.local_vm_object.getNode(),
self.local_vm_object._get_remote_nodes()[0])
self.assertEqual(self.local_vm_object.getPowerState(),
PowerStates.RUNNING)
# Ensure that the VM is registered with the remote libvirt instance and not on the local
# libvirt instance
self.assertFalse(
self.local_vm_object.get_name() in self.vm_factory.getAllVmNames(
node=get_hostname()))
self.assertTrue(
self.local_vm_object.get_name() in self.vm_factory.getAllVmNames(
node=self.local_vm_object._get_remote_nodes()[0]))
# Ensure Drbd disks are in a valid state
for disk_object in self.local_vm_object.getHardDriveObjects():
# Check that the disk is shown as not in-sync
with self.assertRaises(DrbdVolumeNotInSyncException):
disk_object._checkDrbdStatus()
# Reset disk sync status and re-check status to ensure
# the disk is otherwise in a valid state
disk_object.setSyncState(True)
disk_object._checkDrbdStatus()
# Ensure that the local and remote disks are in the correct Drbd role
local_role, remote_role = disk_object._drbdGetRole()
self.assertEqual(local_role, DrbdRoleState.SECONDARY)
self.assertEqual(remote_role, DrbdRoleState.PRIMARY)
def _remove_node_ssl_certificates(self, remote_node):
"""Remove the SSL certificates relating to a node
that is being removed from the cluster
"""
if self._is_cluster_master:
def remove_auth(node_connection, remove_nodes):
# Removes the SSL certificates for the remote node
remote_cluster = node_connection.get_connection('cluster')
for remove_node in remove_nodes:
remote_cluster.remove_node_ssl_certificates(remove_node)
# For all remaining nodes in the cluster, remove all SSL certificates
# and cluster user for node being removed.
other_nodes = self.get_nodes()
other_nodes.remove(remote_node)
self.run_remote_command(callback_method=remove_auth,
nodes=other_nodes,
kwargs={'remove_nodes': [remote_node]})
# Remove Credentials for all nodes in cluster from node being removed
other_nodes.append(get_hostname())
self.run_remote_command(callback_method=remove_auth,
nodes=[remote_node],
kwargs={'remove_nodes': other_nodes})
# Remove authentication from the local node to the node to be removed
cert_generator = self._get_registered_object(
'certificate_generator_factory').get_cert_generator(remote_node)
cert_generator.remove_certificates()
# Remove local cluster user
user_factory = self._get_registered_object('user_factory')
user = user_factory.get_cluster_user_by_node(remote_node)
user.delete()
# Remove configuration for remote node from local config
self.remove_node_configuration(remote_node)
def getAllVmNames(self, node=None):
"""Returns a list of all VMs within the cluster or those registered on a specific node"""
if node is not None:
ArgumentValidator.validate_hostname(node)
# If no node was defined, check the local configuration for all VMs
if (node is None):
return MCVirtConfig().get_config()['virtual_machines']
elif node == get_hostname():
# Obtain array of all domains from libvirt
all_domains = self._get_registered_object(
'libvirt_connector').get_connection().listAllDomains()
return [vm.name() for vm in all_domains]
else:
# Return list of VMs registered on remote node
cluster = self._get_registered_object('cluster')
def remote_command(node_connection):
virtual_machine_factory = node_connection.get_connection(
'virtual_machine_factory')
return virtual_machine_factory.getAllVmNames(node=node)
return cluster.run_remote_command(callback_method=remote_command,
nodes=[node])[node]
def test_migrate(self):
"Perform an online migration using the argument parser"
# Set the mcvirt libvirt failure mode to simulate a post-migration failure
VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.POST_MIGRATION_FAILURE
# Attempt to perform a migration
self.parser.parse_arguments(
"migrate --online --node=%s %s" %
(self.local_vm_object._get_remote_nodes()[0],
self.local_vm_object.get_name()))
VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.NORMAL_RUN
# Ensure the VM is still registered on the remote node and in a running state
self.assertEqual(self.local_vm_object.getNode(),
self.local_vm_object._get_remote_nodes()[0])
self.assertEqual(self.local_vm_object.getPowerState(),
PowerStates.RUNNING.value)
# Ensure that the VM is registered with the remote libvirt instance and not on the local
# libvirt instance
self.assertFalse(
self.local_vm_object.get_name() in self.vm_factory.getAllVmNames(
node=get_hostname()))
self.assertTrue(
self.local_vm_object.get_name() in self.vm_factory.getAllVmNames(
node=self.local_vm_object._get_remote_nodes()[0]))
# Ensure Drbd disks are in a valid state
for disk_object in self.local_vm_object.getHardDriveObjects():
# Check that the disk is shown as not in-sync
disk_object._checkDrbdStatus()
# Ensure that the local and remote disks are in the correct Drbd role
local_role, remote_role = disk_object._drbdGetRole()
self.assertEqual(local_role, DrbdRoleState.SECONDARY)
self.assertEqual(remote_role, DrbdRoleState.PRIMARY)
def ssl_dn(self):
""""Return the certificate DN is openssl argument format."""
server = get_hostname() if self.remote else self.server
return '/C=GB/ST=MCVirt/L=MCVirt/O=MCVirt/CN=%s' % server
def is_local(self):
"""Determine if the server is the local machine"""
return (self.server == get_hostname())
def _ensureLogicalVolumeExists(self, name):
"""Ensures that a logical volume exists, throwing an exception if it does not"""
if not self._checkLogicalVolumeExists(name):
raise LogicalVolumeDoesNotExistException(
'Logical volume %s does not exist on %s' %
(name, get_hostname()))
def _ensureLogicalVolumeActive(self, name):
"""Ensures that a logical volume is active"""
if not self._checkLogicalVolumeActive(name):
raise LogicalVolumeIsNotActive(
'Logical volume %s is not active on %s' %
(name, get_hostname()))
def create(self, name, physical_interface):
"""Create a network on the node"""
# Ensure user has permission to manage networks
self._get_registered_object('auth').assert_permission(
PERMISSIONS.MANAGE_HOST_NETWORKS)
# Validate name
ArgumentValidator.validate_network_name(name)
# Ensure network does not already exist
if self.check_exists(name):
raise NetworkAlreadyExistsException('Network already exists: %s' %
name)
if self._is_cluster_master:
def remote_command(remote_connection):
network_factory = remote_connection.get_connection(
'network_factory')
network_factory.assert_interface_exists(physical_interface)
cluster = self._get_registered_object('cluster')
cluster.run_remote_command(remote_command)
if not self._interface_exists(physical_interface):
raise InterfaceDoesNotExist(
'Physical interface %s does not exist on local node: %s' %
(physical_interface, get_hostname()))
# Create XML for network
network_xml = ET.Element('network')
network_xml.set('ipv6', 'no')
network_name_xml = ET.SubElement(network_xml, 'name')
network_name_xml.text = name
# Create 'forward'
network_forward_xml = ET.SubElement(network_xml, 'forward')
network_forward_xml.set('mode', 'bridge')
# Set interface bridge
network_bridge_xml = ET.SubElement(network_xml, 'bridge')
network_bridge_xml.set('name', physical_interface)
# Convert XML object to string
network_xml_string = ET.tostring(network_xml,
encoding='utf8',
method='xml')
# Attempt to register network with LibVirt
try:
self._get_registered_object('libvirt_connector').get_connection(
).networkDefineXML(network_xml_string)
except:
raise LibvirtException(
'An error occurred whilst registering network with LibVirt')
# Update MCVirt config
def update_config(config):
config['networks'][name] = physical_interface
from mcvirt.mcvirt_config import MCVirtConfig
MCVirtConfig().update_config(update_config,
'Created network \'%s\'' % name)
# Obtain instance of the network object
network_instance = self.get_network_by_name(name)
# Start network
network_instance._get_libvirt_object().create()
# Set network to autostart
network_instance._get_libvirt_object().setAutostart(True)
if self._is_cluster_master:
def remote_add(node):
network_factory = node.get_connection('network_factory')
network_factory.create(name, physical_interface)
cluster = self._get_registered_object('cluster')
cluster.run_remote_command(remote_add)
return network_instance
def assert_interface_exists(self, interface):
if not self.interface_exists(interface):
raise InterfaceDoesNotExist(
'Physical interface %s does not exist on remote node: %s' %
(interface, get_hostname()))
return True
def get_connection(self, server=None):
if not (server is None or server == 'localhost'
or server == get_hostname()):
server = 'doesnnotexist.notavalidrootdomain'
return super(LibvirtConnectorUnitTest, self).get_connection(server)
def start(self):
"""Start the Pyro name server"""
# self.daemon.requestLoop()
Pyro4.config.USE_MSG_WAITALL = False
Pyro4.naming.startNSloop(host=get_hostname(), port=9090, enableBroadcast=False)
def ssl_subj(self):
"""Return the SSL DN in regular format"""
server = get_hostname() if self.remote else self.server
return 'C=GB,ST=MCVirt,L=MCVirt,O=MCVirt,CN=%s' % server
def _create(self,
name,
cpu_cores,
memory_allocation,
hard_drives=None,
network_interfaces=None,
node=None,
available_nodes=None,
storage_type=None,
hard_drive_driver=None,
graphics_driver=None,
modification_flags=None):
"""Create a VM and returns the virtual_machine object for it"""
network_interfaces = [] if network_interfaces is None else network_interfaces
hard_drives = [] if hard_drives is None else hard_drives
available_nodes = [] if available_nodes is None else available_nodes
modification_flags = [] if modification_flags is None else modification_flags
self.checkName(name)
ArgumentValidator.validate_positive_integer(cpu_cores)
ArgumentValidator.validate_positive_integer(memory_allocation)
for hard_drive in hard_drives:
ArgumentValidator.validate_positive_integer(hard_drive)
if network_interfaces:
for network_interface in network_interfaces:
ArgumentValidator.validate_network_name(network_interface)
if node is not None:
ArgumentValidator.validate_hostname(node)
for available_node in available_nodes:
ArgumentValidator.validate_hostname(available_node)
assert storage_type in [None] + [
storage_type_itx.__name__ for storage_type_itx in
self._get_registered_object('hard_drive_factory').STORAGE_TYPES
]
if hard_drive_driver is not None:
HardDriveDriver[hard_drive_driver]
# If no graphics driver has been specified, set it to the default
if graphics_driver is None:
graphics_driver = self.DEFAULT_GRAPHICS_DRIVER
# Check the driver name is valid
self.check_graphics_driver(graphics_driver)
# Ensure the cluster has not been ignored, as VMs cannot be created with MCVirt running
# in this state
if self._cluster_disabled:
raise ClusterNotInitialisedException(
'VM cannot be created whilst the cluster' +
' is not initialised')
# Determine if VM already exists
if self.check_exists(name):
raise VmAlreadyExistsException('Error: VM already exists')
# If a node has not been specified, assume the local node
if node is None:
node = get_hostname()
# If Drbd has been chosen as a storage type, ensure it is enabled on the node
node_drbd = self._get_registered_object('node_drbd')
if storage_type == 'Drbd' and not node_drbd.is_enabled():
raise DrbdNotEnabledOnNode('Drbd is not enabled on this node')
# Create directory for VM on the local and remote nodes
if os_path_exists(VirtualMachine._get_vm_dir(name)):
raise VmDirectoryAlreadyExistsException(
'Error: VM directory already exists')
# If available nodes has not been passed, assume the local machine is the only
# available node if local storage is being used. Use the machines in the cluster
# if Drbd is being used
cluster_object = self._get_registered_object('cluster')
all_nodes = cluster_object.get_nodes(return_all=True)
all_nodes.append(get_hostname())
if len(available_nodes) == 0:
if storage_type == 'Drbd':
# If the available nodes are not specified, use the
# nodes in the cluster
available_nodes = all_nodes
else:
# For local VMs, only use the local node as the available nodes
available_nodes = [get_hostname()]
# If there are more than the maximum number of Drbd machines in the cluster,
# add an option that forces the user to specify the nodes for the Drbd VM
# to be added to
if storage_type == 'Drbd' and len(
available_nodes) != node_drbd.CLUSTER_SIZE:
raise InvalidNodesException('Exactly %i nodes must be specified' %
node_drbd.CLUSTER_SIZE)
for check_node in available_nodes:
if check_node not in all_nodes:
raise NodeDoesNotExistException('Node \'%s\' does not exist' %
check_node)
if get_hostname() not in available_nodes and self._is_cluster_master:
raise InvalidNodesException(
'One of the nodes must be the local node')
# Check whether the hard drives can be created.
if self._is_cluster_master:
hard_drive_factory = self._get_registered_object(
'hard_drive_factory')
for hard_drive_size in hard_drives:
hard_drive_factory.ensure_hdd_valid(
hard_drive_size, storage_type, [
node_itx for node_itx in available_nodes
if node_itx != get_hostname()
])
# Create directory for VM
makedirs(VirtualMachine._get_vm_dir(name))
# Add VM to MCVirt configuration
def updateMCVirtConfig(config):
config['virtual_machines'].append(name)
MCVirtConfig().update_config(
updateMCVirtConfig,
'Adding new VM \'%s\' to global MCVirt configuration' % name)
# Create VM configuration file
VirtualMachineConfig.create(name, available_nodes, cpu_cores,
memory_allocation, graphics_driver)
# Add VM to remote nodes
if self._is_cluster_master:
def remote_command(remote_connection):
virtual_machine_factory = remote_connection.get_connection(
'virtual_machine_factory')
virtual_machine_factory.create(
name=name,
memory_allocation=memory_allocation,
cpu_cores=cpu_cores,
node=node,
available_nodes=available_nodes,
modification_flags=modification_flags)
cluster_object.run_remote_command(callback_method=remote_command)
# Obtain an object for the new VM, to use to create disks/network interfaces
vm_object = self.getVirtualMachineByName(name)
vm_object.get_config_object().gitAdd('Created VM \'%s\'' %
vm_object.get_name())
if node == get_hostname():
# Register VM with LibVirt. If MCVirt has not been initialised on this node,
# do not set the node in the VM configuration, as the change can't be
# replicated to remote nodes
vm_object._register(set_node=self._is_cluster_master)
elif self._is_cluster_master:
# If MCVirt has been initialised on this node and the local machine is
# not the node that the VM will be registered on, set the node on the VM
vm_object._setNode(node)
if self._is_cluster_master:
# Create disk images
hard_drive_factory = self._get_registered_object(
'hard_drive_factory')
for hard_drive_size in hard_drives:
hard_drive_factory.create(vm_object=vm_object,
size=hard_drive_size,
storage_type=storage_type,
driver=hard_drive_driver)
# If any have been specified, add a network configuration for each of the
# network interfaces to the domain XML
network_adapter_factory = self._get_registered_object(
'network_adapter_factory')
network_factory = self._get_registered_object('network_factory')
if network_interfaces is not None:
for network in network_interfaces:
network_object = network_factory.get_network_by_name(
network)
network_adapter_factory.create(vm_object, network_object)
# Add modification flags
vm_object._update_modification_flags(add_flags=modification_flags)
return vm_object
def ssl_base_directory(self):
"""Return the base SSL directory for the node."""
path = '/var/lib/mcvirt/%s/ssl' % get_hostname()
if not os.path.exists(path):
os.makedirs(path)
return path
def add_node(self, node_connection_string):
"""Connect to a remote MCVirt machine, setup shared authentication
and clusters the machines.
"""
# Ensure the user has privileges to manage the cluster
self._get_registered_object('auth').assert_permission(
PERMISSIONS.MANAGE_CLUSTER)
# Ensure that the IP address configurations has been made correctly
self.check_ip_configuration()
try:
config_json = base64.b64decode(node_connection_string)
node_config = json.loads(config_json)
assert 'username' in node_config and node_config['username']
assert 'password' in node_config and node_config['password']
assert 'ip_address' in node_config and node_config['ip_address']
assert 'hostname' in node_config and node_config['hostname']
assert 'ca_cert' in node_config and node_config['ca_cert']
except (TypeError, ValueError, AssertionError):
raise InvalidConnectionString('Connection string is invalid')
# Determine if node is already connected to cluster
if self.check_node_exists(node_config['hostname']):
raise NodeAlreadyPresent(
'Node %s is already connected to the cluster' %
node_config['hostname'])
# Create CA public key for machine
ssl_object = self._get_registered_object(
'certificate_generator_factory').get_cert_generator(
node_config['hostname'])
ssl_object.ca_pub_file = node_config['ca_cert']
# Check remote machine, to ensure it can be synced without any
# conflicts
remote = Connection(username=node_config['username'],
password=node_config['password'],
host=node_config['hostname'])
self.check_remote_machine(remote)
remote = None
original_cluster_nodes = self.get_nodes()
# Add remote node
self.add_node_configuration(
node_name=node_config['hostname'],
ip_address=node_config['ip_address'],
connection_user=node_config['username'],
connection_password=node_config['password'],
ca_key=node_config['ca_cert'])
# Obtain node connection to new node
remote_node = self.get_remote_node(node_config['hostname'])
# Generate local connection user for new remote node
local_connection_info = self.generate_connection_info()
# Add the local node to the new remote node
remote_cluster_instance = remote_node.get_connection('cluster')
remote_cluster_instance.add_node_configuration(
node_name=local_connection_info[0],
ip_address=local_connection_info[1],
connection_user=local_connection_info[2],
connection_password=local_connection_info[3],
ca_key=local_connection_info[4])
new_node_cert_gen_factory = remote_node.get_connection(
'certificate_generator_factory')
# Create client certificates for libvirt for the new node to connect to the
# current cluster node
new_node_cert_gen = new_node_cert_gen_factory.get_cert_generator(
get_hostname())
remote_node.annotate_object(new_node_cert_gen)
# Generate CSR
csr = new_node_cert_gen.generate_csr()
# Sign CSR
cert_gen_factory = self._get_registered_object(
'certificate_generator_factory')
cert_gen = cert_gen_factory.get_cert_generator(node_config['hostname'],
remote=True)
pub_key = cert_gen.sign_csr(csr)
# Add public key to new node
new_node_cert_gen.add_public_key(pub_key)
# Create client certificate for libvirt for the current cluster node to connect
# to the new node
cert_gen = cert_gen_factory.get_cert_generator(node_config['hostname'])
# Generate CSR
csr = cert_gen.generate_csr()
# Sign CSR
new_node_cert_gen = new_node_cert_gen_factory.get_cert_generator(
get_hostname(), remote=True)
remote_node.annotate_object(new_node_cert_gen)
pub_key = new_node_cert_gen.sign_csr(csr)
# Add public key to local node
cert_gen.add_public_key(pub_key)
# Sync credentials to/from old nodes in the cluster
for original_node in original_cluster_nodes:
# Share connection information between cluster node and new node
original_node_remote = self.get_remote_node(original_node)
original_cluster = original_node_remote.get_connection('cluster')
original_node_con_info = original_cluster.generate_connection_info(
)
remote_cluster_instance.add_node_configuration(
node_name=original_node_con_info[0],
ip_address=original_node_con_info[1],
connection_user=original_node_con_info[2],
connection_password=original_node_con_info[3],
ca_key=original_node_con_info[4])
new_node_con_info = remote_cluster_instance.generate_connection_info(
)
original_cluster.add_node_configuration(
node_name=new_node_con_info[0],
ip_address=new_node_con_info[1],
connection_user=new_node_con_info[2],
connection_password=new_node_con_info[3],
ca_key=new_node_con_info[4])
# Create client certificates for libvirt for the new node to connect to the
# current cluster node
new_node_cert_gen = new_node_cert_gen_factory.get_cert_generator(
original_node)
remote_node.annotate_object(new_node_cert_gen)
csr = new_node_cert_gen.generate_csr()
original_node_cert_gen_factory = original_node_remote.get_connection(
'certificate_generator_factory')
original_node_cert_gen = original_node_cert_gen_factory.get_cert_generator(
node_config['hostname'], remote=True)
original_node_remote.annotate_object(original_node_cert_gen)
pub_key = original_node_cert_gen.sign_csr(csr)
new_node_cert_gen.add_public_key(pub_key)
# Create client certificate for libvirt for the current cluster node to connect
# to the new node
original_node_cert_gen = original_node_cert_gen_factory.get_cert_generator(
node_config['hostname'])
original_node_remote.annotate_object(original_node_cert_gen)
# Generate CSR
csr = original_node_cert_gen.generate_csr()
# Sign CSR
new_node_cert_gen = new_node_cert_gen_factory.get_cert_generator(
original_node, remote=True)
remote_node.annotate_object(new_node_cert_gen)
pub_key = new_node_cert_gen.sign_csr(csr)
# Add public key to original node
original_node_cert_gen.add_public_key(pub_key)
# If Drbd is enabled on the local node, configure/enable it on the remote node
if self._get_registered_object('node_drbd').is_enabled():
remote_drbd = remote_node.get_connection('node_drbd')
remote_drbd.enable(
secret=MCVirtConfig().get_config()['drbd']['secret'])
# Sync users
self.sync_users(remote_node)
# Sync networks
self.sync_networks(remote_node)
# Sync global permissions
self.sync_permissions(remote_node)
# Sync VMs
self.sync_virtual_machines(remote_node)
