def create_id_volume(self, nodes=None): """Create identification volume on the storage backend""" # Obtain volume object for ID volume volume = self.get_volume(self._id_volume_name) # If the storage backend is shared, then only needs to be created on # a single node (prefer local host if in list of nodes). If not shared, # ID volume will be created on all nodes if nodes is None: nodes = ([get_hostname()] if get_hostname() in self.nodes else [self.nodes[0]]) if self.shared else self.nodes try: # Create ID volume volume.create(size=2 ** 20, nodes=nodes) except ExternalStorageCommandErrorException: raise ExternalStorageCommandErrorException( ('An error occured whilst adding storage backend: Either the storage ' 'backend has no free space (at least 1MB required) or ' 'shared storage is being used, but has not been specified')) try: # Ensure that ID volume exists on all nodes volume.ensure_exists(nodes=self.nodes) except VolumeDoesNotExistError: raise ExternalStorageCommandErrorException( ('An error ocurred whilst verifying the storage backend: ' 'A shared storage has been specified, but it is not'))
def delete(self, local_only=False): """Delete the logical volume for the disk""" vm_object = self.get_virtual_machine() # Ensure that the user has permissions to add delete storage self._get_registered_object('auth').assert_permission( PERMISSIONS.MODIFY_HARD_DRIVE, vm_object ) # If the storage backend is available on one to more nodes, # ensure that the hard drive exists. Prefer local host check_node = get_hostname() if get_hostname() in self.nodes else self.nodes[0] if self._is_cluster_master or check_node == get_hostname(): self.ensure_exists(nodes=[check_node]) if vm_object: vm_object.ensureUnlocked() vm_object.ensure_stopped() # Remove the hard drive from the MCVirt VM configuration self.get_attachment_object().delete(local_only=local_only) # Remove backing storage self._removeStorage(local_only=local_only) # Remove config nodes = [get_hostname()] if local_only else self._get_registered_object( 'cluster').get_nodes(include_local=True) self.remove_config(nodes=nodes)
def pre_check_network(self, name, interface): """Perform pre-limiary checks on node before determining that a network can be added""" self._get_registered_object('auth').assert_user_type('ConnectionUser', 'ClusterUser') # Ensure that the physical interface exists self.assert_interface_exists(interface) # Ensure that there are no interfaces present on the MCVirt instance # that match the network if self.check_exists(name): raise NetworkAlreadyExistsException('Network already exists on node: %s %s' % (name, get_hostname())) # Ensure that there is not already a network with the same name defined in # libvirt try: self._get_registered_object( 'libvirt_connector' ).get_connection().networkLookupByName(name) # If the libvirt connect did not throw an error that # the network does not exist, raise an exception # as the network must be pressent # @TODO: do this more nicely. Get list of networks and # assert that it's not in the list raise NetworkAlreadyExistsException( 'Network already defined in libvirt on node: %s %s' % (name, get_hostname())) except libvirtError: pass
def undo(self): """Execute the undo method for the function""" # If the local node is in the list of complete # commands, then undo it first if (get_hostname() in self.nodes and self.nodes[get_hostname()]['complete'] and hasattr(self.obj, self._undo_function_name)): # Set current node local_hostname = get_hostname() self.current_node = local_hostname Syslogger.logger().debug('Undo %s %s %s %s' % (get_hostname(), self._undo_function_name, str(self.nodes[get_hostname()]['args']), str(self.nodes[get_hostname()]['kwargs']))) getattr(self.obj, self._undo_function_name)( *self.nodes[get_hostname()]['args'], **self.nodes[get_hostname()]['kwargs']) # Iterate through nodes and undo for node in self.nodes: # Skip local node or if the function did not complete on the node if node == get_hostname() or not self.nodes[node]['complete']: continue # Run the remote undo method Syslogger.logger().debug('Undo %s %s %s %s' % (node, self.function.__name__, str(self.nodes[node]['args']), str(self.nodes[node]['kwargs']))) self._call_function_remote(node=node, undo=True)
def delete_id_volume(self, nodes=None): """Delete the ID volume for the storage backend""" # Obtain volume object for ID volume volume = self.get_volume(self._id_volume_name) # If the storage backend is shared, then only needs to be created on # a single node (prefer local host if in list of nodes). If not shared, # ID volume will be created on all nodes if nodes is None: nodes = ([get_hostname()] if get_hostname() in self.nodes else [self.nodes[0]]) if self.shared else self.nodes # Create ID volume volume.delete(nodes=nodes)
def get_remote_factory(self, node=None): if node is None or node == get_hostname(): return self node = self._get_registered_object('cluster').get_remote_node(node) remote_factory = node.get_connection('iso_factory') node.annotate_object(remote_factory) return remote_factory
def gitRemove(self, message='', custom_file=None): """Remove and commits a configuration file""" if self._checkGitRepo(): session_obj = self._get_registered_object('mcvirt_session') username = '' user = None if session_obj: try: user = session_obj.get_proxy_user_object() except UserDoesNotExistException: pass if user: username = session_obj.get_proxy_user_object().get_username() message += "\nUser: %s\nNode: %s" % (username, get_hostname()) try: System.runCommand([self.GIT, 'rm', '--cached', self.config_file], cwd=DirectoryLocation.BASE_STORAGE_DIR) System.runCommand([self.GIT, 'commit', '-m', message], cwd=DirectoryLocation.BASE_STORAGE_DIR) System.runCommand([self.GIT, 'push'], raise_exception_on_failure=False, cwd=DirectoryLocation.BASE_STORAGE_DIR) except Exception: pass
def add_node_configuration(self, node_name, ip_address, connection_user, connection_password, ca_key): """Add MCVirt node to configuration, generates a cluster user on the remote node and stores credentials against node in the MCVirt configuration. """ self._get_registered_object('auth').assert_permission(PERMISSIONS.MANAGE_CLUSTER) # Create CA file ssl_object = self._get_registered_object( 'certificate_generator_factory').get_cert_generator(node_name) ssl_object.ca_pub_file = ca_key # Connect to node and obtain cluster user remote = Connection(username=connection_user, password=connection_password, host=node_name) remote_user_factory = remote.get_connection('user_factory') connection_user = remote_user_factory.get_user_by_username(connection_user) remote.annotate_object(connection_user) username, password = connection_user.create_cluster_user(host=get_hostname()) # Add node to configuration file def add_node_config(mcvirt_config): mcvirt_config['cluster']['nodes'][node_name] = { 'ip_address': ip_address, 'username': username, 'password': password } MCVirtConfig().update_config(add_node_config)
def __init__(self): """Store required object member variables and create MCVirt object""" # Initialise Pyro4 with flag to showing that the daemon is being started Pyro4.current_context.STARTUP_PERIOD = True # Store nameserver, MCVirt instance and create daemon self.daemon_lock = DaemonLock() Pyro4.config.USE_MSG_WAITALL = False Pyro4.config.CREATE_SOCKET_METHOD = SSLSocket.create_ssl_socket Pyro4.config.CREATE_BROADCAST_SOCKET_METHOD = SSLSocket.create_broadcast_ssl_socket Pyro4.config.THREADPOOL_ALLOW_QUEUE = True Pyro4.config.THREADPOOL_SIZE = 128 self.hostname = get_hostname() # Ensure that the required SSL certificates exist ssl_socket = CertificateGeneratorFactory().get_cert_generator('localhost') ssl_socket.check_certificates(check_client=False) ssl_socket = None # Wait for nameserver self.obtain_connection() RpcNSMixinDaemon.DAEMON = BaseRpcDaemon(host=self.hostname) self.register_factories() # Ensure libvirt is configured cert_gen_factory = RpcNSMixinDaemon.DAEMON.registered_factories[ 'certificate_generator_factory'] cert_gen = cert_gen_factory.get_cert_generator('localhost') cert_gen.check_certificates() cert_gen = None cert_gen_factory = None
def getAllVmNames(self, node=None): """Returns a list of all VMs within the cluster or those registered on a specific node""" if node is not None: ArgumentValidator.validate_hostname(node) # If no node was defined, check the local configuration for all VMs if node is None: return [vm['name'] for vm in VirtualMachineConfig.get_global_config().values()] elif node == get_hostname(): # @TODO - Why is this using libvirt?! Should use # VM objects (i.e. config file to determine) # and use a seperate function to get libvirt # registered VMs # Obtain array of all domains from libvirt all_domains = self._get_registered_object( 'libvirt_connector').get_connection().listAllDomains() return [vm.name() for vm in all_domains] else: # Return list of VMs registered on remote node cluster = self._get_registered_object('cluster') def remote_command(node_connection): """Get virtual machine names from remote node""" virtual_machine_factory = node_connection.get_connection('virtual_machine_factory') return virtual_machine_factory.getAllVmNames(node=node) return cluster.run_remote_command(callback_method=remote_command, nodes=[node])[node]
def ensure_active(self): """Ensure that voljuem is active, otherwise, raise exception""" if not self.is_active(): raise VolumeIsNotActiveException( 'Volume %s is not active on %s' % (self.name, get_hostname()) )
def get_all(self, available_on_local_node=None, nodes=[], drbd=None, storage_type=None, shared=None, nodes_predefined=False, global_=None, default_location=None): """Return all storage backends, with optional filtering""" storage_objects = [] for storage_id in self.get_config().keys(): # Obtain storage object storage_object = self.get_object(storage_id) # Check if storage backend is global or not if global_ is not None and storage_object.is_global != global_: continue # Check if default location matches requested if (default_location is not None and storage_object.get_location(return_default=True) != default_location): continue # Check storage is available on local node if (available_on_local_node and get_hostname() not in storage_object.nodes): continue # If nodes are specified... if nodes: # Determine which nodes from the list are available available_nodes = [] for node in nodes: if node in storage_object.nodes: available_nodes.append(node) # If the list of nodes is required (predefined) and not all are # present, skip storage backend if nodes_predefined and len(nodes) != len(available_nodes): continue # Otherwise, (if the nodes are not required), ensure at least one # is available, otherwise, skip elif (not nodes_predefined) and not len(available_nodes): continue # If drbd is specified, ensure storage object is suitable to run DRBD if drbd and not storage_object.is_drbd_suitable(): continue # If storage_type is specified, ensure hat storage matches the object if storage_type and not self.get_class(storage_type) != storage_object.__class__: continue # If a shared type is defined, determine if it matches the object, otherwise skip if shared is not None and shared != storage_object.shared: continue # If all checks have passed, append to list of objects to return storage_objects.append(storage_object) return storage_objects
def node_pre_check(cls, cluster, libvirt_config, location): """Ensure volume group exists on node""" try: cls.ensure_exists(location) except InvalidStorageConfiguration, exc: raise InvalidStorageConfiguration( '%s on node %s' % (str(exc), get_hostname()) )
def ensure_available(self): """Ensure that the storage backend is currently available on the node""" volume = self.get_volume(self._id_volume_name) if (not self.check_exists()) or not volume.check_exists(): raise StorageBackendNotAvailableOnNode( 'Storage backend %s is not currently avaialble on node: %s' % ( self.name, get_hostname()))
def insert_into_stat_db(self): """Add statistics to statistics database""" db_factory = self._get_registered_object('database_factory') db_rows = [ (StatisticsDeviceType.HOST.value, get_hostname(), StatisticsStatType.CPU_USAGE.value, self._cpu_usage, "{:%s}".format(datetime.now())), (StatisticsDeviceType.HOST.value, get_hostname(), StatisticsStatType.MEMORY_USAGE.value, self._memory_usage, "{:%s}".format(datetime.now())) ] with db_factory.get_locking_connection() as db_inst: db_inst.cursor.executemany( """INSERT INTO stats( device_type, device_id, stat_type, stat_value, stat_date ) VALUES(?, ?, ?, ?, ?)""", db_rows)
def _get_response_data(self): """Determine and return response data""" # Return dict of node -> output, if a dict response was # specified if self.return_dict: return {node: self.nodes[node]['return_val'] for node in self.nodes.keys()} # Otherwise, default to returning data from local node elif get_hostname() in self.nodes: return self.nodes[get_hostname()]['return_val'] # Otherwise, return the response from the first found node. elif self.nodes: return self.nodes[self.nodes.keys()[0]]['return_val'] # Otherwise, if no node data, return None return None
def generate_connection_info(self): """Generate required information to connect to this node from a remote node""" # Ensure user has required permissions self._get_registered_object('auth').assert_permission( PERMISSIONS.MANAGE_CLUSTER ) # Ensure that the IP address configurations has been made correctly self.check_ip_configuration() # Create connection user user_factory = self._get_registered_object('user_factory') connection_username, connection_password = user_factory.generate_user(ConnectionUser) ssl_object = self._get_registered_object( 'certificate_generator_factory').get_cert_generator(get_hostname()) return [get_hostname(), self.get_cluster_ip_address(), connection_username, connection_password, ssl_object.get_ca_contents()]
def __init__(self, server=None, remote=False): """Store member variables and ensure that openSSL is installed""" if not os.path.isfile(self.OPENSSL): raise OpenSSLNotFoundException('openssl not found: %s' % self.OPENSSL) if server == 'localhost' or server.startswith('127.') or server is None: self.server = get_hostname() else: self.server = server self.remote = remote
def __init__(self): """Store required object member variables and create MCVirt object""" # Before doing ANYTHING, ensure that the hostname that MCVirt thinks the # machine is (i.e. the hostname that the machine was already setup as) # matches the current hostname of the machine ensure_hostname_consistent() # Initialise Pyro4 with flag to showing that the daemon is being started Pyro4.current_context.STARTUP_PERIOD = True # Store nameserver, MCVirt instance and create daemon self.daemon_lock = DaemonLock() self.timer_objects = [] Pyro4.config.USE_MSG_WAITALL = False Pyro4.config.CREATE_SOCKET_METHOD = SSLSocket.create_ssl_socket Pyro4.config.CREATE_BROADCAST_SOCKET_METHOD = SSLSocket.create_broadcast_ssl_socket Pyro4.config.THREADPOOL_ALLOW_QUEUE = True Pyro4.config.THREADPOOL_SIZE = 128 self.hostname = get_hostname() # Ensure that the required SSL certificates exist ssl_socket = CertificateGeneratorFactory().get_cert_generator('localhost') ssl_socket.check_certificates(check_client=False) ssl_socket = None # Wait for nameserver Syslogger.logger().debug('Wait for connection to nameserver') self.obtain_connection() Syslogger.logger().debug('Obtained nameserver connection') RpcNSMixinDaemon.DAEMON = BaseRpcDaemon(host=self.hostname) self.register_factories() # Ensure libvirt is configured Syslogger.logger().debug('Start certificate check') cert_gen_factory = RpcNSMixinDaemon.DAEMON.registered_factories[ 'certificate_generator_factory'] cert_gen = cert_gen_factory.get_cert_generator('localhost') cert_gen.check_certificates() cert_gen = None cert_gen_factory = None Syslogger.logger().debug('Register atexit') atexit.register(self.shutdown, 'atexit', '') for sig in (signal.SIGABRT, signal.SIGILL, signal.SIGINT, signal.SIGSEGV, signal.SIGTERM): signal.signal(sig, self.shutdown) Syslogger.logger().debug('Initialising objects') for registered_object in RpcNSMixinDaemon.DAEMON.registered_factories: obj = RpcNSMixinDaemon.DAEMON.registered_factories[registered_object] if type(obj) is not types.TypeType: # noqa Syslogger.logger().debug('Initialising object %s' % registered_object) obj.initialise()
def test_migrate_pre_migration_libvirt_failure(self): """Simulates a pre-migration libvirt failure""" # Set the mcvirt libvirt failure mode to simulate a pre-migration failure VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.PRE_MIGRATION_FAILURE # Attempt to perform a migration with self.assertRaises(LibvirtFailureSimulationException): self.test_vm_object.onlineMigrate(self.local_vm_object._get_remote_nodes()[0]) VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.NORMAL_RUN # Ensure the VM is still registered on the local node and in a running state self.assertEqual(self.local_vm_object.getNode(), get_hostname()) self.assertEqual(self.local_vm_object.get_power_state(), PowerStates.RUNNING.value) # Ensure that the VM is registered with the local libvirt instance and not on the remote # libvirt instance self.assertTrue( self.local_vm_object.get_name() in self.vm_factory.getAllVmNames(node=get_hostname()) ) self.assertFalse( self.local_vm_object.get_name() in self.vm_factory.getAllVmNames(node=self.local_vm_object._get_remote_nodes()[0]) ) # Ensure Drbd disks are in a valid state for disk_object in self.local_vm_object.get_hard_drive_objects(): # Check that the disk is shown as not in-sync with self.assertRaises(DrbdVolumeNotInSyncException): disk_object._checkDrbdStatus() # Reset disk sync status and re-check status to ensure # the disk is otherwise in a valid state disk_object.setSyncState(True) disk_object._checkDrbdStatus() # Ensure that the local and remote disks are in the correct Drbd role local_role, remote_role = disk_object._drbdGetRole() self.assertEqual(local_role, DrbdRoleState.PRIMARY) self.assertEqual(remote_role, DrbdRoleState.SECONDARY)
def check_ip_configuration(self): """Perform various checks to ensure that the IP configuration is such that is suitable to be part of a cluster """ # Ensure that the cluster IP address has been defined cluster_ip = self.get_cluster_ip_address() if not cluster_ip: raise MissingConfigurationException('IP address has not yet been configured') # Ensure that the hostname of the local machine does not resolve # to 127.0.0.1 if socket.gethostbyname(get_hostname()).startswith('127.'): raise MissingConfigurationException(('Node hostname %s resolves to the localhost.' ' Instead it should resolve to the cluster' ' IP address')) resolve_ip = socket.gethostbyname(get_hostname()) if resolve_ip != cluster_ip: raise MissingConfigurationException(('The local hostname (%s) should resolve the' ' cluster IP address (%s). Instead it resolves' ' to \'%s\'. Please correct this issue before' ' continuing.') % (get_hostname(), cluster_ip, resolve_ip))
def get_all_drbd_hard_drive_object(self, include_remote=False): """Obtain all hard drive objects that are backed by DRBD""" hard_drive_objects = [] vm_factory = self._get_registered_object('virtual_machine_factory') for vm_object in vm_factory.getAllVirtualMachines(): if (get_hostname() in vm_object.getAvailableNodes() or include_remote): all_hard_drive_objects = vm_object.getHardDriveObjects() for hard_drive_object in all_hard_drive_objects: if (hard_drive_object.get_type() is 'Drbd'): hard_drive_objects.append(hard_drive_object) return hard_drive_objects
def available_on_node(self, node=None, raise_on_err=True): """Determine if the storage volume is available on a given node """ if node is None: node = get_hostname() available = (node in self.nodes) if not available and raise_on_err: raise StorageBackendNotAvailableOnNode( 'Storage not available on node: %s, %s' % (self.name, node) ) return available
def test_migrate_libvirt_connection_failure(self): """Attempt to perform a migration, simulating a libvirt connection failure""" VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.CONNECTION_FAILURE with self.assertRaises(libvirt.libvirtError): self.test_vm_object.onlineMigrate(self.local_vm_object._get_remote_nodes()[0]) VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.NORMAL_RUN # Ensure the VM is still registered on the local node and in a running state self.assertEqual(self.local_vm_object.getNode(), get_hostname()) self.assertEqual(self.local_vm_object.get_power_state(), PowerStates.RUNNING)
def in_use(self): """Determine if the ISO is currently in use by a VM""" virtual_machine_factory = self._get_registered_object('virtual_machine_factory') for vm_name in virtual_machine_factory.getAllVmNames(node=get_hostname()): vm_object = virtual_machine_factory.getVirtualMachineByName(vm_name) disk_drive_object = vm_object.get_disk_drive() vm_current_iso = disk_drive_object.getCurrentDisk() # If the VM has an iso attached, check if the ISO is this one if vm_current_iso and vm_current_iso.get_path() == self.get_path(): return vm_object.get_name() return False
def _undo_function_name(self): """Return the name of the undo function""" # If running on a remote node and a remote undo method # is defined, return that if self.current_node != get_hostname() and self.remote_undo_method: return self.remote_undo_method # Otherwise, if a custom undo method is defined (for all nodes) # return that if self.undo_method: return self.undo_method # Otherwise, return default undo name for method return 'undo__%s' % self.function.__name__
def delete(self, local_only=False): """Remove the hard drive attachment""" # Ensure that the user has permissions to add delete storage self._get_registered_object('auth').assert_permission( PERMISSIONS.MODIFY_VM, self.virtual_machine ) self.remove_from_virtual_machine() cluster = self._get_registered_object('cluster') nodes = [get_hostname()] if local_only else cluster.get_nodes(include_local=True) self.remove_config(nodes=cluster.get_nodes(include_local=True)) del Factory.CACHED_OBJECTS[(self.virtual_machine.id_, self.attachment_id)] self.unregister_object()
def get_config(self): """Create the configuration for libvirt""" cert_gen_factory = self._get_registered_object('certificate_generator_factory') ssl_socket = cert_gen_factory.get_cert_generator('localhost') nodes = self._get_registered_object('cluster').get_nodes(return_all=True) nodes.append(get_hostname()) allowed_dns = [cert_gen_factory.get_cert_generator(node).ssl_subj for node in nodes] return { 'ip_address': MCVirtConfig().getListenAddress(), 'ssl_server_key': ssl_socket.server_key_file, 'ssl_server_cert': ssl_socket.server_pub_file, 'ssl_ca_cert': ssl_socket.ca_pub_file, 'allowed_nodes': '", "'.join(allowed_dns) }
def test_migrate_libvirt_connection_failure(self): """Attempt to perform a migration, simulating a libvirt connection failure""" VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.CONNECTION_FAILURE with self.assertRaises(libvirt.libvirtError): self.test_vm_object.onlineMigrate( self.local_vm_object._get_remote_nodes()[0]) VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.NORMAL_RUN # Ensure the VM is still registered on the local node and in a running state self.assertEqual(self.local_vm_object.getNode(), get_hostname()) self.assertEqual(self.local_vm_object.getPowerState(), PowerStates.RUNNING)
def get_connection(self, server=None): """Obtains a Libvirt connection for a given server""" if server is None: server = get_hostname() ssl_object = self._get_registered_object( 'certificate_generator_factory').get_cert_generator(server) libvirt_url = 'qemu://%s/system?pkipath=%s' % ( ssl_object.server, ssl_object.ssl_directory) connection = libvirt.open(libvirt_url) if connection is None: raise LibVirtConnectionException( 'Failed to open connection to the hypervisor on %s' % ssl_object.server) return connection
def in_use(self): """Determine if the ISO is currently in use by a VM""" virtual_machine_factory = self._get_registered_object( 'virtual_machine_factory') for vm_name in virtual_machine_factory.getAllVmNames( node=get_hostname()): vm_object = virtual_machine_factory.getVirtualMachineByName( vm_name) disk_drive_object = vm_object.get_disk_drive() vm_current_iso = disk_drive_object.getCurrentDisk() # If the VM has an iso attached, check if the ISO is this one if vm_current_iso and vm_current_iso.get_path() == self.get_path(): return vm_object.get_name() return False
def resync(self, source_node=None, auto_determine=False): """Perform a resync of a Drbd hard drive""" # Ensure user has privileges to create a Drbd volume self._get_registered_object('auth').assert_permission( PERMISSIONS.MANAGE_DRBD, self.vm_object) if source_node: if source_node not in self.vm_object.getAvailableNodes(): raise InvalidNodesException('Invalid node name') if auto_determine: raise TooManyParametersException( 'Only one of source_node an auto_determine should be specified' ) else: if not auto_determine: raise ArgumentParserException( 'Either source_node or auto_determine must be specified' ) elif self.vm_object.getNode(): source_node = self.vm_object.getNode() else: raise VmNotRegistered('Cannot auto-determine node - VM is not registered') # Check Drbd state of disk if self._drbdGetConnectionState() != DrbdConnectionState.CONNECTED: raise DrbdStateException( 'Drbd resource must be connected before performing a resync: %s' % self.resource_name) if source_node == get_hostname(): System.runCommand([NodeDrbd.DrbdADM, 'invalidate-remote', self.resource_name]) # Monitor the Drbd status, until the VM has started syncing while True: if self._drbdGetConnectionState() == DrbdConnectionState.SYNC_SOURCE: break time.sleep(5) # Monitor the Drbd status, until the VM has finished syncing while True: if self._drbdGetConnectionState() != DrbdConnectionState.SYNC_SOURCE: break time.sleep(5) elif not self._cluster_disable: remote_object = self.get_remote_object(remote_node=source_node) remote_object.resync(source_node=source_node)
def get_location(self, node=None, return_default=False): """Return the location for a given node, default to local node""" # Default node to local node if node is None: node = get_hostname() # Raise exception if node is not configured for storage backend if node not in self.nodes: raise UnsuitableNodeException( 'Node does not support storage backend: %s, %s' % (node, self.name) ) config = self.get_config() return (config['nodes'][node]['location'] if 'location' in config['nodes'][node] and config['nodes'][node]['location'] and not return_default else config['location'])
def gitRemove(self, message=''): """Remove and commits a configuration file""" from auth.session import Session if self._checkGitRepo(): message += "\nUser: %s\nNode: %s" % ( Session.get_current_user_object().get_username(), get_hostname()) try: System.runCommand( [self.GIT, 'rm', '--cached', self.config_file], cwd=DirectoryLocation.BASE_STORAGE_DIR) System.runCommand([self.GIT, 'commit', '-m', message], cwd=DirectoryLocation.BASE_STORAGE_DIR) System.runCommand([self.GIT, 'push'], raise_exception_on_failure=False, cwd=DirectoryLocation.BASE_STORAGE_DIR) except: pass
def gitAdd(self, message=''): """Commit changes to an added or modified configuration file""" from auth.session import Session if (self._checkGitRepo()): message += "\nUser: %s\nNode: %s" % ( Session.get_current_user_object().get_username(), get_hostname()) try: System.runCommand([self.GIT, 'add', self.config_file], cwd=DirectoryLocation.BASE_STORAGE_DIR) System.runCommand( [self.GIT, 'commit', '-m', message, self.config_file], cwd=DirectoryLocation.BASE_STORAGE_DIR) System.runCommand([self.GIT, 'push'], raise_exception_on_failure=False, cwd=DirectoryLocation.BASE_STORAGE_DIR) except: pass
def _generateDrbdConfig(self): """Generates the Drbd resource configuration""" # Create configuration for use with the template raw_lv_path = self._getLogicalVolumePath(self._getLogicalVolumeName(self.Drbd_RAW_SUFFIX)) meta_lv_path = self._getLogicalVolumePath( self._getLogicalVolumeName( self.Drbd_META_SUFFIX)) drbd_config = \ { 'resource_name': self.resource_name, 'block_device_path': self._getDrbdDevice(), 'raw_lv_path': raw_lv_path, 'meta_lv_path': meta_lv_path, 'drbd_port': self.drbd_port, 'nodes': [] } # Add local node to the Drbd config cluster_object = self._get_registered_object('cluster') node_template_conf = \ { 'name': get_hostname(), 'ip_address': cluster_object.get_cluster_ip_address() } drbd_config['nodes'].append(node_template_conf) # Add remote nodes to Drbd config for node in self.vm_object._get_remote_nodes(): node_config = cluster_object.get_node_config(node) node_template_conf = \ { 'name': node, 'ip_address': node_config['ip_address'] } drbd_config['nodes'].append(node_template_conf) # Replace the variables in the template with the local Drbd configuration config_content = Template(file=self.Drbd_CONFIG_TEMPLATE, searchList=[drbd_config]) # Write the Drbd configuration fh = open(self._getDrbdConfigFile(), 'w') fh.write(config_content.respond()) fh.close()
def add_from_url(self, url, name=None, node=None): """Download an ISO from given URL and save in ISO directory""" self._get_registered_object('auth').assert_permission( PERMISSIONS.MANAGE_ISO) if node is not None and node != get_hostname(): remote_node = self._get_registered_object( 'cluster').get_remote_node(node) remote_factory = remote_node.get_connection('iso_factory') remote_iso = remote_factory.add_from_url(url=url, name=name) remote_node.annotate_object(remote_iso) return remote_iso.get_name() # Work out name from URL if name is not supplied if name is None: # Parse URL to get path part url_parse = urlparse.urlparse(url) name = Iso.get_filename_from_path(url_parse.path) # Get temporary directory to store ISO temp_directory = tempfile.mkdtemp() output_path = temp_directory + '/' + name # Open file iso = urllib2.urlopen(url) # Read file in 16KB chunks chunk_size = 16 * 1024 # Save ISO with open(output_path, 'wb') as file: while True: chunk = iso.read(chunk_size) if not chunk: break file.write(chunk) iso.close() iso_object = self.add_iso(output_path) os.remove(output_path) os.rmdir(temp_directory) return iso_object.get_name()
def print_info(self): """Print information about the nodes in the cluster""" table = Texttable() table.set_deco(Texttable.HEADER | Texttable.VLINES) table.header(('Node', 'IP Address', 'Status')) # Add this node to the table table.add_row((get_hostname(), self.get_cluster_ip_address(), 'Local')) # Add remote nodes for node in self.get_nodes(return_all=True): node_config = self.get_node_config(node) node_status = 'Unreachable' try: self.get_remote_node(node) node_status = 'Connected' except CouldNotConnectToNodeException: pass table.add_row((node, node_config['ip_address'], node_status)) return table.draw()
def test_migrate_post_migration_libvirt_failure(self): """Simulates a post-migration libvirt failure""" # Set the mcvirt libvirt failure mode to simulate a post-migration failure VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.POST_MIGRATION_FAILURE # Attempt to perform a migration with self.assertRaises(LibvirtFailureSimulationException): self.test_vm_object.onlineMigrate( self.local_vm_object._get_remote_nodes()[0]) VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.NORMAL_RUN # Ensure the VM is still registered on the remote node and in a running state self.assertEqual(self.local_vm_object.getNode(), self.local_vm_object._get_remote_nodes()[0]) self.assertEqual(self.local_vm_object.getPowerState(), PowerStates.RUNNING) # Ensure that the VM is registered with the remote libvirt instance and not on the local # libvirt instance self.assertFalse( self.local_vm_object.get_name() in self.vm_factory.getAllVmNames( node=get_hostname())) self.assertTrue( self.local_vm_object.get_name() in self.vm_factory.getAllVmNames( node=self.local_vm_object._get_remote_nodes()[0])) # Ensure Drbd disks are in a valid state for disk_object in self.local_vm_object.getHardDriveObjects(): # Check that the disk is shown as not in-sync with self.assertRaises(DrbdVolumeNotInSyncException): disk_object._checkDrbdStatus() # Reset disk sync status and re-check status to ensure # the disk is otherwise in a valid state disk_object.setSyncState(True) disk_object._checkDrbdStatus() # Ensure that the local and remote disks are in the correct Drbd role local_role, remote_role = disk_object._drbdGetRole() self.assertEqual(local_role, DrbdRoleState.SECONDARY) self.assertEqual(remote_role, DrbdRoleState.PRIMARY)
def _remove_node_ssl_certificates(self, remote_node): """Remove the SSL certificates relating to a node that is being removed from the cluster """ if self._is_cluster_master: def remove_auth(node_connection, remove_nodes): # Removes the SSL certificates for the remote node remote_cluster = node_connection.get_connection('cluster') for remove_node in remove_nodes: remote_cluster.remove_node_ssl_certificates(remove_node) # For all remaining nodes in the cluster, remove all SSL certificates # and cluster user for node being removed. other_nodes = self.get_nodes() other_nodes.remove(remote_node) self.run_remote_command(callback_method=remove_auth, nodes=other_nodes, kwargs={'remove_nodes': [remote_node]}) # Remove Credentials for all nodes in cluster from node being removed other_nodes.append(get_hostname()) self.run_remote_command(callback_method=remove_auth, nodes=[remote_node], kwargs={'remove_nodes': other_nodes}) # Remove authentication from the local node to the node to be removed cert_generator = self._get_registered_object( 'certificate_generator_factory').get_cert_generator(remote_node) cert_generator.remove_certificates() # Remove local cluster user user_factory = self._get_registered_object('user_factory') user = user_factory.get_cluster_user_by_node(remote_node) user.delete() # Remove configuration for remote node from local config self.remove_node_configuration(remote_node)
def getAllVmNames(self, node=None): """Returns a list of all VMs within the cluster or those registered on a specific node""" if node is not None: ArgumentValidator.validate_hostname(node) # If no node was defined, check the local configuration for all VMs if (node is None): return MCVirtConfig().get_config()['virtual_machines'] elif node == get_hostname(): # Obtain array of all domains from libvirt all_domains = self._get_registered_object( 'libvirt_connector').get_connection().listAllDomains() return [vm.name() for vm in all_domains] else: # Return list of VMs registered on remote node cluster = self._get_registered_object('cluster') def remote_command(node_connection): virtual_machine_factory = node_connection.get_connection( 'virtual_machine_factory') return virtual_machine_factory.getAllVmNames(node=node) return cluster.run_remote_command(callback_method=remote_command, nodes=[node])[node]
def test_migrate(self): "Perform an online migration using the argument parser" # Set the mcvirt libvirt failure mode to simulate a post-migration failure VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.POST_MIGRATION_FAILURE # Attempt to perform a migration self.parser.parse_arguments( "migrate --online --node=%s %s" % (self.local_vm_object._get_remote_nodes()[0], self.local_vm_object.get_name())) VirtualMachineLibvirtFail.LIBVIRT_FAILURE_MODE = LibvirtFailureMode.NORMAL_RUN # Ensure the VM is still registered on the remote node and in a running state self.assertEqual(self.local_vm_object.getNode(), self.local_vm_object._get_remote_nodes()[0]) self.assertEqual(self.local_vm_object.getPowerState(), PowerStates.RUNNING.value) # Ensure that the VM is registered with the remote libvirt instance and not on the local # libvirt instance self.assertFalse( self.local_vm_object.get_name() in self.vm_factory.getAllVmNames( node=get_hostname())) self.assertTrue( self.local_vm_object.get_name() in self.vm_factory.getAllVmNames( node=self.local_vm_object._get_remote_nodes()[0])) # Ensure Drbd disks are in a valid state for disk_object in self.local_vm_object.getHardDriveObjects(): # Check that the disk is shown as not in-sync disk_object._checkDrbdStatus() # Ensure that the local and remote disks are in the correct Drbd role local_role, remote_role = disk_object._drbdGetRole() self.assertEqual(local_role, DrbdRoleState.SECONDARY) self.assertEqual(remote_role, DrbdRoleState.PRIMARY)
def ssl_dn(self): """"Return the certificate DN is openssl argument format.""" server = get_hostname() if self.remote else self.server return '/C=GB/ST=MCVirt/L=MCVirt/O=MCVirt/CN=%s' % server
def is_local(self): """Determine if the server is the local machine""" return (self.server == get_hostname())
def _ensureLogicalVolumeExists(self, name): """Ensures that a logical volume exists, throwing an exception if it does not""" if not self._checkLogicalVolumeExists(name): raise LogicalVolumeDoesNotExistException( 'Logical volume %s does not exist on %s' % (name, get_hostname()))
def _ensureLogicalVolumeActive(self, name): """Ensures that a logical volume is active""" if not self._checkLogicalVolumeActive(name): raise LogicalVolumeIsNotActive( 'Logical volume %s is not active on %s' % (name, get_hostname()))
def create(self, name, physical_interface): """Create a network on the node""" # Ensure user has permission to manage networks self._get_registered_object('auth').assert_permission( PERMISSIONS.MANAGE_HOST_NETWORKS) # Validate name ArgumentValidator.validate_network_name(name) # Ensure network does not already exist if self.check_exists(name): raise NetworkAlreadyExistsException('Network already exists: %s' % name) if self._is_cluster_master: def remote_command(remote_connection): network_factory = remote_connection.get_connection( 'network_factory') network_factory.assert_interface_exists(physical_interface) cluster = self._get_registered_object('cluster') cluster.run_remote_command(remote_command) if not self._interface_exists(physical_interface): raise InterfaceDoesNotExist( 'Physical interface %s does not exist on local node: %s' % (physical_interface, get_hostname())) # Create XML for network network_xml = ET.Element('network') network_xml.set('ipv6', 'no') network_name_xml = ET.SubElement(network_xml, 'name') network_name_xml.text = name # Create 'forward' network_forward_xml = ET.SubElement(network_xml, 'forward') network_forward_xml.set('mode', 'bridge') # Set interface bridge network_bridge_xml = ET.SubElement(network_xml, 'bridge') network_bridge_xml.set('name', physical_interface) # Convert XML object to string network_xml_string = ET.tostring(network_xml, encoding='utf8', method='xml') # Attempt to register network with LibVirt try: self._get_registered_object('libvirt_connector').get_connection( ).networkDefineXML(network_xml_string) except: raise LibvirtException( 'An error occurred whilst registering network with LibVirt') # Update MCVirt config def update_config(config): config['networks'][name] = physical_interface from mcvirt.mcvirt_config import MCVirtConfig MCVirtConfig().update_config(update_config, 'Created network \'%s\'' % name) # Obtain instance of the network object network_instance = self.get_network_by_name(name) # Start network network_instance._get_libvirt_object().create() # Set network to autostart network_instance._get_libvirt_object().setAutostart(True) if self._is_cluster_master: def remote_add(node): network_factory = node.get_connection('network_factory') network_factory.create(name, physical_interface) cluster = self._get_registered_object('cluster') cluster.run_remote_command(remote_add) return network_instance
def assert_interface_exists(self, interface): if not self.interface_exists(interface): raise InterfaceDoesNotExist( 'Physical interface %s does not exist on remote node: %s' % (interface, get_hostname())) return True
def get_connection(self, server=None): if not (server is None or server == 'localhost' or server == get_hostname()): server = 'doesnnotexist.notavalidrootdomain' return super(LibvirtConnectorUnitTest, self).get_connection(server)
def start(self): """Start the Pyro name server""" # self.daemon.requestLoop() Pyro4.config.USE_MSG_WAITALL = False Pyro4.naming.startNSloop(host=get_hostname(), port=9090, enableBroadcast=False)
def ssl_subj(self): """Return the SSL DN in regular format""" server = get_hostname() if self.remote else self.server return 'C=GB,ST=MCVirt,L=MCVirt,O=MCVirt,CN=%s' % server
def _create(self, name, cpu_cores, memory_allocation, hard_drives=None, network_interfaces=None, node=None, available_nodes=None, storage_type=None, hard_drive_driver=None, graphics_driver=None, modification_flags=None): """Create a VM and returns the virtual_machine object for it""" network_interfaces = [] if network_interfaces is None else network_interfaces hard_drives = [] if hard_drives is None else hard_drives available_nodes = [] if available_nodes is None else available_nodes modification_flags = [] if modification_flags is None else modification_flags self.checkName(name) ArgumentValidator.validate_positive_integer(cpu_cores) ArgumentValidator.validate_positive_integer(memory_allocation) for hard_drive in hard_drives: ArgumentValidator.validate_positive_integer(hard_drive) if network_interfaces: for network_interface in network_interfaces: ArgumentValidator.validate_network_name(network_interface) if node is not None: ArgumentValidator.validate_hostname(node) for available_node in available_nodes: ArgumentValidator.validate_hostname(available_node) assert storage_type in [None] + [ storage_type_itx.__name__ for storage_type_itx in self._get_registered_object('hard_drive_factory').STORAGE_TYPES ] if hard_drive_driver is not None: HardDriveDriver[hard_drive_driver] # If no graphics driver has been specified, set it to the default if graphics_driver is None: graphics_driver = self.DEFAULT_GRAPHICS_DRIVER # Check the driver name is valid self.check_graphics_driver(graphics_driver) # Ensure the cluster has not been ignored, as VMs cannot be created with MCVirt running # in this state if self._cluster_disabled: raise ClusterNotInitialisedException( 'VM cannot be created whilst the cluster' + ' is not initialised') # Determine if VM already exists if self.check_exists(name): raise VmAlreadyExistsException('Error: VM already exists') # If a node has not been specified, assume the local node if node is None: node = get_hostname() # If Drbd has been chosen as a storage type, ensure it is enabled on the node node_drbd = self._get_registered_object('node_drbd') if storage_type == 'Drbd' and not node_drbd.is_enabled(): raise DrbdNotEnabledOnNode('Drbd is not enabled on this node') # Create directory for VM on the local and remote nodes if os_path_exists(VirtualMachine._get_vm_dir(name)): raise VmDirectoryAlreadyExistsException( 'Error: VM directory already exists') # If available nodes has not been passed, assume the local machine is the only # available node if local storage is being used. Use the machines in the cluster # if Drbd is being used cluster_object = self._get_registered_object('cluster') all_nodes = cluster_object.get_nodes(return_all=True) all_nodes.append(get_hostname()) if len(available_nodes) == 0: if storage_type == 'Drbd': # If the available nodes are not specified, use the # nodes in the cluster available_nodes = all_nodes else: # For local VMs, only use the local node as the available nodes available_nodes = [get_hostname()] # If there are more than the maximum number of Drbd machines in the cluster, # add an option that forces the user to specify the nodes for the Drbd VM # to be added to if storage_type == 'Drbd' and len( available_nodes) != node_drbd.CLUSTER_SIZE: raise InvalidNodesException('Exactly %i nodes must be specified' % node_drbd.CLUSTER_SIZE) for check_node in available_nodes: if check_node not in all_nodes: raise NodeDoesNotExistException('Node \'%s\' does not exist' % check_node) if get_hostname() not in available_nodes and self._is_cluster_master: raise InvalidNodesException( 'One of the nodes must be the local node') # Check whether the hard drives can be created. if self._is_cluster_master: hard_drive_factory = self._get_registered_object( 'hard_drive_factory') for hard_drive_size in hard_drives: hard_drive_factory.ensure_hdd_valid( hard_drive_size, storage_type, [ node_itx for node_itx in available_nodes if node_itx != get_hostname() ]) # Create directory for VM makedirs(VirtualMachine._get_vm_dir(name)) # Add VM to MCVirt configuration def updateMCVirtConfig(config): config['virtual_machines'].append(name) MCVirtConfig().update_config( updateMCVirtConfig, 'Adding new VM \'%s\' to global MCVirt configuration' % name) # Create VM configuration file VirtualMachineConfig.create(name, available_nodes, cpu_cores, memory_allocation, graphics_driver) # Add VM to remote nodes if self._is_cluster_master: def remote_command(remote_connection): virtual_machine_factory = remote_connection.get_connection( 'virtual_machine_factory') virtual_machine_factory.create( name=name, memory_allocation=memory_allocation, cpu_cores=cpu_cores, node=node, available_nodes=available_nodes, modification_flags=modification_flags) cluster_object.run_remote_command(callback_method=remote_command) # Obtain an object for the new VM, to use to create disks/network interfaces vm_object = self.getVirtualMachineByName(name) vm_object.get_config_object().gitAdd('Created VM \'%s\'' % vm_object.get_name()) if node == get_hostname(): # Register VM with LibVirt. If MCVirt has not been initialised on this node, # do not set the node in the VM configuration, as the change can't be # replicated to remote nodes vm_object._register(set_node=self._is_cluster_master) elif self._is_cluster_master: # If MCVirt has been initialised on this node and the local machine is # not the node that the VM will be registered on, set the node on the VM vm_object._setNode(node) if self._is_cluster_master: # Create disk images hard_drive_factory = self._get_registered_object( 'hard_drive_factory') for hard_drive_size in hard_drives: hard_drive_factory.create(vm_object=vm_object, size=hard_drive_size, storage_type=storage_type, driver=hard_drive_driver) # If any have been specified, add a network configuration for each of the # network interfaces to the domain XML network_adapter_factory = self._get_registered_object( 'network_adapter_factory') network_factory = self._get_registered_object('network_factory') if network_interfaces is not None: for network in network_interfaces: network_object = network_factory.get_network_by_name( network) network_adapter_factory.create(vm_object, network_object) # Add modification flags vm_object._update_modification_flags(add_flags=modification_flags) return vm_object
def ssl_base_directory(self): """Return the base SSL directory for the node.""" path = '/var/lib/mcvirt/%s/ssl' % get_hostname() if not os.path.exists(path): os.makedirs(path) return path
def add_node(self, node_connection_string): """Connect to a remote MCVirt machine, setup shared authentication and clusters the machines. """ # Ensure the user has privileges to manage the cluster self._get_registered_object('auth').assert_permission( PERMISSIONS.MANAGE_CLUSTER) # Ensure that the IP address configurations has been made correctly self.check_ip_configuration() try: config_json = base64.b64decode(node_connection_string) node_config = json.loads(config_json) assert 'username' in node_config and node_config['username'] assert 'password' in node_config and node_config['password'] assert 'ip_address' in node_config and node_config['ip_address'] assert 'hostname' in node_config and node_config['hostname'] assert 'ca_cert' in node_config and node_config['ca_cert'] except (TypeError, ValueError, AssertionError): raise InvalidConnectionString('Connection string is invalid') # Determine if node is already connected to cluster if self.check_node_exists(node_config['hostname']): raise NodeAlreadyPresent( 'Node %s is already connected to the cluster' % node_config['hostname']) # Create CA public key for machine ssl_object = self._get_registered_object( 'certificate_generator_factory').get_cert_generator( node_config['hostname']) ssl_object.ca_pub_file = node_config['ca_cert'] # Check remote machine, to ensure it can be synced without any # conflicts remote = Connection(username=node_config['username'], password=node_config['password'], host=node_config['hostname']) self.check_remote_machine(remote) remote = None original_cluster_nodes = self.get_nodes() # Add remote node self.add_node_configuration( node_name=node_config['hostname'], ip_address=node_config['ip_address'], connection_user=node_config['username'], connection_password=node_config['password'], ca_key=node_config['ca_cert']) # Obtain node connection to new node remote_node = self.get_remote_node(node_config['hostname']) # Generate local connection user for new remote node local_connection_info = self.generate_connection_info() # Add the local node to the new remote node remote_cluster_instance = remote_node.get_connection('cluster') remote_cluster_instance.add_node_configuration( node_name=local_connection_info[0], ip_address=local_connection_info[1], connection_user=local_connection_info[2], connection_password=local_connection_info[3], ca_key=local_connection_info[4]) new_node_cert_gen_factory = remote_node.get_connection( 'certificate_generator_factory') # Create client certificates for libvirt for the new node to connect to the # current cluster node new_node_cert_gen = new_node_cert_gen_factory.get_cert_generator( get_hostname()) remote_node.annotate_object(new_node_cert_gen) # Generate CSR csr = new_node_cert_gen.generate_csr() # Sign CSR cert_gen_factory = self._get_registered_object( 'certificate_generator_factory') cert_gen = cert_gen_factory.get_cert_generator(node_config['hostname'], remote=True) pub_key = cert_gen.sign_csr(csr) # Add public key to new node new_node_cert_gen.add_public_key(pub_key) # Create client certificate for libvirt for the current cluster node to connect # to the new node cert_gen = cert_gen_factory.get_cert_generator(node_config['hostname']) # Generate CSR csr = cert_gen.generate_csr() # Sign CSR new_node_cert_gen = new_node_cert_gen_factory.get_cert_generator( get_hostname(), remote=True) remote_node.annotate_object(new_node_cert_gen) pub_key = new_node_cert_gen.sign_csr(csr) # Add public key to local node cert_gen.add_public_key(pub_key) # Sync credentials to/from old nodes in the cluster for original_node in original_cluster_nodes: # Share connection information between cluster node and new node original_node_remote = self.get_remote_node(original_node) original_cluster = original_node_remote.get_connection('cluster') original_node_con_info = original_cluster.generate_connection_info( ) remote_cluster_instance.add_node_configuration( node_name=original_node_con_info[0], ip_address=original_node_con_info[1], connection_user=original_node_con_info[2], connection_password=original_node_con_info[3], ca_key=original_node_con_info[4]) new_node_con_info = remote_cluster_instance.generate_connection_info( ) original_cluster.add_node_configuration( node_name=new_node_con_info[0], ip_address=new_node_con_info[1], connection_user=new_node_con_info[2], connection_password=new_node_con_info[3], ca_key=new_node_con_info[4]) # Create client certificates for libvirt for the new node to connect to the # current cluster node new_node_cert_gen = new_node_cert_gen_factory.get_cert_generator( original_node) remote_node.annotate_object(new_node_cert_gen) csr = new_node_cert_gen.generate_csr() original_node_cert_gen_factory = original_node_remote.get_connection( 'certificate_generator_factory') original_node_cert_gen = original_node_cert_gen_factory.get_cert_generator( node_config['hostname'], remote=True) original_node_remote.annotate_object(original_node_cert_gen) pub_key = original_node_cert_gen.sign_csr(csr) new_node_cert_gen.add_public_key(pub_key) # Create client certificate for libvirt for the current cluster node to connect # to the new node original_node_cert_gen = original_node_cert_gen_factory.get_cert_generator( node_config['hostname']) original_node_remote.annotate_object(original_node_cert_gen) # Generate CSR csr = original_node_cert_gen.generate_csr() # Sign CSR new_node_cert_gen = new_node_cert_gen_factory.get_cert_generator( original_node, remote=True) remote_node.annotate_object(new_node_cert_gen) pub_key = new_node_cert_gen.sign_csr(csr) # Add public key to original node original_node_cert_gen.add_public_key(pub_key) # If Drbd is enabled on the local node, configure/enable it on the remote node if self._get_registered_object('node_drbd').is_enabled(): remote_drbd = remote_node.get_connection('node_drbd') remote_drbd.enable( secret=MCVirtConfig().get_config()['drbd']['secret']) # Sync users self.sync_users(remote_node) # Sync networks self.sync_networks(remote_node) # Sync global permissions self.sync_permissions(remote_node) # Sync VMs self.sync_virtual_machines(remote_node)