def start(self):
self.start_time = datetime.now()
self.status = LogState.RUNNING
Syslogger.logger().debug(' Start command: %s' % ', '.join([
str(self.start_time), self.user or '', self.object_type or '', self.object_name or '',
self.method_name or ''
]))
def get_remote_node(self,
node,
ignore_cluster_master=False,
set_cluster_master=False):
"""Obtain a Remote object for a node, caching the object"""
if not self._is_cluster_master and not ignore_cluster_master:
raise ClusterNotInitialisedException(
'Cannot get remote node %s' % node +
' as the cluster is not initialised')
node_config = self.get_node_config(node)
try:
node_object = Node(node,
node_config,
cluster_master=(set_cluster_master if
set_cluster_master else None))
except:
if not self._cluster_disabled:
raise InaccessibleNodeException(
'Cannot connect to node \'%s\'' % node)
else:
Syslogger.logger().error(
'Cannot connect to node: %s (Ignored)' % node)
node_object = None
return node_object
def finish(self):
"""Mark the transaction as having been completed"""
self.comlpete = True
# Only remove transaction if it is the last
# transaction in the stack
if self.id == Transaction.transactions[-1].id:
Syslogger.logger().debug('End of transaction stack')
# Tear down all transactions
for transaction in Transaction.transactions:
# Delete each of the function objects
for func in transaction.functions:
transaction.functions.remove(func)
func.unregister(force=True)
del func
# Reset list of transactions
Transaction.transactions = []
else:
# Otherwise, remove this transaction
Syslogger.logger().debug('End of transaction')
# Delete each of the function objects
for func in self.functions:
self.functions.remove(func)
func.unregister(force=True)
del func
# @TODO HOW CAN THIS NO LONGER BE IN THE LIST?
if self in Transaction.transactions:
Transaction.transactions.remove(self)
def finish_success(self):
self.finish_time = datetime.now()
self.status = LogState.SUCCESS
Syslogger.logger().debug(' Command complete (success): %s' % ', '.join([
str(self.finish_time), self.user or '', self.object_type or '', self.object_name or '',
self.method_name or ''
]))
def _get_auth_obj(self, password=None):
"""Setup annotations for authentication"""
auth_dict = {
Annotations.USERNAME: self.__username
}
if password:
auth_dict[Annotations.PASSWORD] = password
elif self.__session_id:
auth_dict[Annotations.SESSION_ID] = self.__session_id
if self.__proxy_username:
auth_dict[Annotations.PROXY_USER] = self.__proxy_username
if self.__cluster_master is not None:
Syslogger.logger().warning('Setting cluster master to %s' % self.__cluster_master)
auth_dict[Annotations.CLUSTER_MASTER] = self.__cluster_master
if 'has_lock' in dir(Pyro4.current_context):
auth_dict[Annotations.HAS_LOCK] = Pyro4.current_context.has_lock
auth_dict[Annotations.IGNORE_CLUSTER] = self.__ignore_cluster
if 'ignore_cluster' in dir(Pyro4.current_context):
auth_dict[Annotations.IGNORE_CLUSTER] |= Pyro4.current_context.ignore_cluster
auth_dict[Annotations.IGNORE_DRBD] = self.__ignore_drbd
if 'ignore_drbd' in dir(Pyro4.current_context):
auth_dict[Annotations.IGNORE_DRBD] |= Pyro4.current_context.ignore_drbd
return auth_dict
def undo(self):
"""Execute the undo method for the function"""
# If the local node is in the list of complete
# commands, then undo it first
if (get_hostname() in self.nodes and
self.nodes[get_hostname()]['complete'] and
hasattr(self.obj, self._undo_function_name)):
# Set current node
local_hostname = get_hostname()
self.current_node = local_hostname
Syslogger.logger().debug('Undo %s %s %s %s' %
(get_hostname(),
self._undo_function_name,
str(self.nodes[get_hostname()]['args']),
str(self.nodes[get_hostname()]['kwargs'])))
getattr(self.obj, self._undo_function_name)(
*self.nodes[get_hostname()]['args'],
**self.nodes[get_hostname()]['kwargs'])
# Iterate through nodes and undo
for node in self.nodes:
# Skip local node or if the function did not complete on the node
if node == get_hostname() or not self.nodes[node]['complete']:
continue
# Run the remote undo method
Syslogger.logger().debug('Undo %s %s %s %s' %
(node,
self.function.__name__,
str(self.nodes[node]['args']),
str(self.nodes[node]['kwargs'])))
self._call_function_remote(node=node, undo=True)
def validateHandshake(self, conn, data): # Override name of upstream method # noqa
"""Perform authentication on new connections"""
self.handshake__set_defaults()
# Attempt to perform authentication sequence
try:
# Authenticate user and obtain username and session id
username, session_id = self.handshake__authenticate_user(data)
# Determine if user can provide alternative users
session_instance = self.registered_factories['mcvirt_session']
user_object = session_instance.get_current_user_object()
# Set proxy user
self.handshake__set_proxy_user(data, user_object)
# Set cluster master
self.handshake__set_cluster_master(data, user_object)
# Set has lock
self.handshake__set_has_lock(data, user_object)
# Set ignore cluster and DRBD
self.handshake__set_ignore_cluster(data, user_object)
self.handshake__set_ignore_drbd(data, user_object)
# Perform node version check
self.handshake__check_cluster_version()
# Return the session id
return session_id
except Pyro4.errors.SecurityError, e:
Syslogger.logger().exception('SecurityError during authentication: %s' % str(e))
raise
def _get_auth_obj(self, password=None):
"""Setup annotations for authentication"""
auth_dict = {Annotations.USERNAME: self.__username}
if password:
auth_dict[Annotations.PASSWORD] = password
elif self.__session_id:
auth_dict[Annotations.SESSION_ID] = self.__session_id
if self.__proxy_username:
auth_dict[Annotations.PROXY_USER] = self.__proxy_username
if self.__cluster_master is not None:
Syslogger.logger().warning('Setting cluster master to %s' %
self.__cluster_master)
auth_dict[Annotations.CLUSTER_MASTER] = self.__cluster_master
if 'has_lock' in dir(Pyro4.current_context):
auth_dict[Annotations.HAS_LOCK] = Pyro4.current_context.has_lock
auth_dict[Annotations.IGNORE_CLUSTER] = self.__ignore_cluster
if 'ignore_cluster' in dir(Pyro4.current_context):
auth_dict[Annotations.
IGNORE_CLUSTER] |= Pyro4.current_context.ignore_cluster
auth_dict[Annotations.IGNORE_Drbd] = self.__ignore_drbd
if 'ignore_drbd' in dir(Pyro4.current_context):
auth_dict[
Annotations.IGNORE_Drbd] |= Pyro4.current_context.ignore_drbd
return auth_dict
def set_state(self, new_state):
"""Set state"""
if self.state != new_state:
Syslogger.logger().debug(
'State for (%s) changed from %s to %s' %
(self.virtual_machine.get_name(),
self.state, new_state))
self.state = new_state
def initialise(self):
"""Detect running VMs on local node and create watchdog daemon"""
# Check all VMs
for virtual_machine in self._get_registered_object(
'virtual_machine_factory').get_all_virtual_machines():
Syslogger.logger().debug('Registering watchdog for: %s' % virtual_machine.get_name())
self.start_watchdog(virtual_machine)
def finish_error(self, exception):
self.finish_time = datetime.now()
self.status = LogState.FAILED
self.exception_message = str(exception)
self.exception_mcvirt = True
Syslogger.logger().error(' Command failed (MCVirt Exception): %s' % ', '.join([
str(self.finish_time), self.user or '', self.object_type or '', self.object_name or '',
self.method_name or '', self.exception_message or ''
]))
def register(self, obj_or_class, objectId, *args, **kwargs): # Override upstream # noqa
"""Override register to register object with NS."""
Syslogger.logger().debug('Registering object: %s' % objectId)
uri = RpcNSMixinDaemon.DAEMON.register(obj_or_class, *args, **kwargs)
ns = Pyro4.naming.locateNS(host=self.hostname, port=9090, broadcast=False)
ns.register(objectId, uri)
ns = None
RpcNSMixinDaemon.DAEMON.registered_factories[objectId] = obj_or_class
return uri
def interval(self):
"""Return the timer interval"""
if self.state is WATCHDOG_STATES.STARTUP:
boot_wait = self.virtual_machine.get_watchdog_boot_wait()
Syslogger.logger().debug(
'In boot period, interval is: %s' % boot_wait)
return boot_wait
else:
return self.virtual_machine.get_watchdog_interval()
def start(self, *args, **kwargs):
"""Start the Pyro daemon"""
Pyro4.current_context.STARTUP_PERIOD = False
Syslogger.logger().debug('Authentication enabled')
Syslogger.logger().debug('Obtaining lock')
with DaemonLock.LOCK:
Syslogger.logger().debug('Obtained lock')
Syslogger.logger().debug('Starting daemon request loop')
RpcNSMixinDaemon.DAEMON.requestLoop(*args, **kwargs)
Syslogger.logger().debug('Daemon request loop finished')
def run(self):
"""Obtain CPU and memory statistics"""
Pyro4.current_context.INTERNAL_REQUEST = True
Syslogger.logger().debug('Starting host stats gathering')
self._cpu_usage = OSStats.get_cpu_usage()
self._memory_usage = OSStats.get_ram_usage()
self.insert_into_stat_db()
Syslogger.logger().debug('Completed host stats gathering')
Pyro4.current_context.INTERNAL_REQUEST = False
def obtain_connection(self):
"""Attempt to obtain a connection to the name server."""
while 1:
try:
Pyro4.naming.locateNS(host=self.hostname, port=9090, broadcast=False)
return
except Exception as e:
Syslogger.logger().warn('Connecting to name server: %s' % str(e))
# Wait for 1 second for name server to come up
time.sleep(1)
def _register_object(self, local_object):
"""Register an object with the pyro daemon"""
if self._is_pyro_initialised:
try:
Syslogger.logger().debug('Registering object (dynamic): %s' % local_object)
except:
pass
self._pyroDaemon.register(local_object)
if '_pyro_server_ref' in dir(self):
local_object._pyro_server_ref = self._pyro_server_ref
def dh_params_file(self):
"""Return the path to the DH parameters file, and create it if it does not exist"""
if not self.is_local:
raise CACertificateNotFoundException('DH params file not available for remote node')
path = self._get_certificate_path('dh_params')
if not self._ensure_exists(path, assert_raise=False):
# Generate new DH parameters
Syslogger.logger().info('Generating DH parameters file')
System.runCommand([self.OPENSSL, 'dhparam', '-out', path, '2048'])
Syslogger.logger().info('DH parameters file generated')
return path
def start(self):
self.start_time = datetime.now()
self.status = LogState.RUNNING
Syslogger.logger().debug('Start command: %s' % ', '.join([
str(self.start_time), self.user or '', self.object_type or '',
self.object_name or '', self.method_name or ''
]))
for remote_log in self.remote_logs:
try:
remote_log.start()
except:
pass
def start(self):
self.start_time = datetime.now()
self.status = LogState.RUNNING
Syslogger.logger().debug('Start command: %s' % ', '.join([
str(self.start_time), self.user or '', self.object_type or '', self.object_name or '',
self.method_name or ''
]))
for remote_log in self.remote_logs:
try:
remote_log.start()
except:
pass
def register(self, obj_or_class, objectId, *args,
**kwargs): # Override upstream # noqa
"""Override register to register object with NS."""
Syslogger.logger().debug('Registering object: %s' % objectId)
uri = RpcNSMixinDaemon.DAEMON.register(obj_or_class, *args, **kwargs)
ns = Pyro4.naming.locateNS(host=self.hostname,
port=9090,
broadcast=False)
ns.register(objectId, uri)
ns = None
RpcNSMixinDaemon.DAEMON.registered_factories[objectId] = obj_or_class
return uri
def dh_params_file(self):
"""Return the path to the DH parameters file, and create it if it does not exist"""
if not self.is_local:
raise CACertificateNotFoundException(
'DH params file not available for remote node')
path = self._get_certificate_path('dh_params')
if not self._ensure_exists(path, assert_raise=False):
# Generate new DH parameters
Syslogger.logger().info('Generating DH parameters file')
System.runCommand([self.OPENSSL, 'dhparam', '-out', path, '2048'])
Syslogger.logger().info('DH parameters file generated')
return path
def obtain_connection(self):
"""Attempt to obtain a connection to the name server."""
while 1:
try:
Pyro4.naming.locateNS(host=self.hostname,
port=9090,
broadcast=False)
return
except Exception as e:
Syslogger.logger().warn('Connecting to name server: %s' %
str(e))
# Wait for 1 second for name server to come up
time.sleep(1)
def repeat_run(self):
"""Re-start timer once run has complete"""
# Restart timer, if set to repeat before run
if not self.repeat_after_run and self.repeat:
self.timer = Timer(float(self.interval), self.repeat_run)
self.timer.start()
return_output = None
try:
# Run command
return_output = self.run(*self.run_args, **self.run_kwargs)
except Exception, exc:
Syslogger.logger().error(
'Error ocurred during thread: %s\n%s' %
(self.__class__.__name__, str(exc)))
def finish_error_unknown(self, exception):
self.finish_time = datetime.now()
self.status = LogState.FAILED
self.exception_message = str(exception)
self.exception_mcvirt = False
Syslogger.logger().error('Command failed (Unknown Exception): %s' % ', '.join([
str(self.finish_time), self.user or '', self.object_type or '', self.object_name or '',
self.method_name or '', self.exception_message or ''
]))
for remote_log in self.remote_logs:
try:
remote_log.finish_error_unknown(str(exception))
except:
pass
self.unregister()
def __init__(self):
"""Setup member variables and register transaction"""
# Determine transaction ID.
self._id = len(Transaction.transactions)
# Initialise LIFO stack of functions
self.functions = []
# Initialise with an incomplete state
self.complete = False
# Only register transacstion is not in an undo-state
if not Transaction.undo_state:
# Add the transaction to the static list of transactions
Transaction.transactions.insert(0, self)
Syslogger.logger().debug('Starting new transaction')
def get_remote_node(self, node, ignore_cluster_master=False):
"""Obtain a Remote object for a node, caching the object"""
if not self._is_cluster_master and not ignore_cluster_master:
raise ClusterNotInitialisedException('Cannot get remote node %s' % node +
' as the cluster is not initialised')
node_config = self.get_node_config(node)
try:
node_object = Node(node, node_config)
except:
if not self._cluster_disabled:
raise InaccessibleNodeException('Cannot connect to node \'%s\'' % node)
else:
Syslogger.logger().error('Cannot connect to node: %s (Ignored)' % node)
node_object = None
return node_object
def authenticate_session(self, username, session):
"""Authenticate user session."""
Syslogger.logger().debug("Authenticating session for user %s: %s" % (username, session))
if (session in Session.USER_SESSIONS and
Session.USER_SESSIONS[session].username == username):
# Check session has not expired
if Session.USER_SESSIONS[session].is_valid():
Session.USER_SESSIONS[session].renew()
user_factory = self._get_registered_object('user_factory')
return user_factory.get_user_by_username(username)
else:
del Session.USER_SESSIONS[session]
raise AuthenticationError('Invalid session ID')
class Factory(PyroObject):
"""Class for obtaining virtual machine objects"""
OBJECT_TYPE = 'virtual machine'
VIRTUAL_MACHINE_CLASS = VirtualMachine
DEFAULT_GRAPHICS_DRIVER = GraphicsDriver.VMVGA.value
CACHED_OBJECTS = {}
def autostart(self, start_type=AutoStartStates.ON_POLL):
"""Autostart VMs"""
Syslogger.logger().info('Starting autostart: %s' % start_type.name)
for vm in self.getAllVirtualMachines():
if (vm.isRegisteredLocally() and vm.is_stopped
and vm._get_autostart_state()
in [AutoStartStates.ON_POLL, AutoStartStates.ON_BOOT]
and (start_type == vm._get_autostart_state()
or start_type == AutoStartStates.ON_BOOT)):
try:
Syslogger.logger().info('Autostarting: %s' % vm.get_name())
vm.start()
Syslogger.logger().info('Autostart successful: %s' %
vm.get_name())
except Exception, e:
Syslogger.logger().error('Failed to autostart: %s: %s' %
(vm.get_name(), str(e)))
Syslogger.logger().info('Finished autostsart: %s' % start_type.name)
def finish_error_unknown(self, exception):
self.finish_time = datetime.now()
self.status = LogState.FAILED
self.exception_message = str(exception)
self.exception_mcvirt = False
Syslogger.logger().error(
'Command failed (Unknown Exception): %s' % ', '.join([
str(self.finish_time), self.user or '', self.object_type or '',
self.object_name or '', self.method_name or '',
self.exception_message or ''
]))
for remote_log in self.remote_logs:
try:
remote_log.finish_error_unknown(str(exception))
except:
pass
self.unregister()
def get_cert_generator(self, server, remote=False):
"""Obtain a certificate generator object for a given server"""
if (server, remote) not in CertificateGeneratorFactory.CACHED_OBJECTS:
cert_generator = CertificateGenerator(server, remote=remote)
self._register_object(cert_generator)
if not self._is_pyro_initialised:
try:
Syslogger.logger().info(
('Obtained unregistered version of CertificateGenerator'
' for %s (Remote: %s)') % (server, remote)
)
except:
pass
return cert_generator
CertificateGeneratorFactory.CACHED_OBJECTS[(server, remote)] = cert_generator
return CertificateGeneratorFactory.CACHED_OBJECTS[(server, remote)]
def authenticate_session(self, username, session):
"""Authenticate user session."""
Syslogger.logger().debug("Authenticating session for user %s: %s" %
(username, session))
if (session in Session.USER_SESSIONS
and Session.USER_SESSIONS[session].username == username):
# Check session has not expired
if Session.USER_SESSIONS[session].is_valid():
Session.USER_SESSIONS[session].renew()
user_factory = self._get_registered_object('user_factory')
return user_factory.get_user_by_username(username)
else:
del Session.USER_SESSIONS[session]
raise AuthenticationError('Invalid session ID')
def initialise(self):
"""Delete the default libvirt network if it exists"""
libvirt = self._get_registered_object('libvirt_connector').get_connection()
try:
default = libvirt.networkLookupByName('default')
try:
default.destroy()
except:
pass
try:
default.undefine()
except:
pass
except libvirtError:
# Fail silently
Syslogger.logger().info('Failed to find default network')
def initialise(self):
"""Delete the default libvirt network if it exists"""
libvirt = self._get_registered_object(
'libvirt_connector').get_connection()
try:
default = libvirt.networkLookupByName('default')
try:
default.destroy()
except:
pass
try:
default.undefine()
except:
pass
except libvirtError:
# Fail silently
Syslogger.logger().info('Failed to find default network')
def function_failed(cls, function):
"""Called when a function fails - perform
the undo method on all previous functions on each of the
transactions
"""
# If already undoing transaction, do not undo the undo methods
if Transaction.undo_state:
return
Transaction.undo_state = True
try:
# If in a transaction
if cls.in_transaction():
# Iterate through transactions, removing each item
for transaction_ar in cls.transactions:
# Iteracte through each function in the transaction
for function in transaction_ar.functions:
# Undo the function
function.undo()
# Mark the transaction as complete, removing
# it from global list and all functions
transaction_ar.finish()
else:
# Otherwise, undo single function
function.undo()
except Exception, exc:
Syslogger.logger().error('Failed during undo: %s' % str(exc))
# If exception is thrown, remove any remaining
# transactions and reset undo_state
for transaction_ar in cls.transactions:
# Mark the transaction as complete, removing
# it from global list and all functions
transaction_ar.finish()
# Reset undo state flag
Transaction.undo_state = False
# Re-raise exception in undo
raise
def initialise(self):
"""Delete the default libvirt network if it exists"""
libvirt = self._get_registered_object('libvirt_connector').get_connection()
try:
default = libvirt.networkLookupByName(DEFAULT_LIBVIRT_NETWORK_NAME)
try:
default.destroy()
except Exception:
pass
try:
default.undefine()
except Exception:
pass
except libvirtError:
# Fail silently (ish)
Syslogger.logger().info(
'Failed to find default network (%s)' % DEFAULT_LIBVIRT_NETWORK_NAME
)
def autostart(self, start_type=AutoStartStates.ON_POLL):
"""Autostart VMs"""
Syslogger.logger().info('Starting autostart: %s' % start_type.name)
for vm in self.get_all_virtual_machines():
try:
if (vm.isRegisteredLocally() and vm.is_stopped and
vm._get_autostart_state() in
[AutoStartStates.ON_POLL, AutoStartStates.ON_BOOT] and
(start_type == vm._get_autostart_state() or
start_type == AutoStartStates.ON_BOOT)):
try:
Syslogger.logger().info('Autostarting: %s' % vm.get_name())
vm.start()
Syslogger.logger().info('Autostart successful: %s' % vm.get_name())
except Exception, exc2:
Syslogger.logger().error('Failed to autostart: %s: %s' %
(vm.get_name(), str(exc2)))
except Exception, exc:
Syslogger.logger().error('Failed to get VM state: %s: %s' % (
vm.get_name(), str(exc)))
def __init__(self):
"""Store required object member variables and create MCVirt object"""
# Initialise Pyro4 with flag to showing that the daemon is being started
Pyro4.current_context.STARTUP_PERIOD = True
# Store nameserver, MCVirt instance and create daemon
self.daemon_lock = DaemonLock()
self.timer_objects = []
Pyro4.config.USE_MSG_WAITALL = False
Pyro4.config.CREATE_SOCKET_METHOD = SSLSocket.create_ssl_socket
Pyro4.config.CREATE_BROADCAST_SOCKET_METHOD = SSLSocket.create_broadcast_ssl_socket
Pyro4.config.THREADPOOL_ALLOW_QUEUE = True
Pyro4.config.THREADPOOL_SIZE = 128
self.hostname = get_hostname()
# Ensure that the required SSL certificates exist
ssl_socket = CertificateGeneratorFactory().get_cert_generator(
'localhost')
ssl_socket.check_certificates(check_client=False)
ssl_socket = None
# Wait for nameserver
Syslogger.logger().debug('Wait for connection to nameserver')
self.obtain_connection()
RpcNSMixinDaemon.DAEMON = BaseRpcDaemon(host=self.hostname)
self.register_factories()
# Ensure libvirt is configured
Syslogger.logger().debug('Start certificate check')
cert_gen_factory = RpcNSMixinDaemon.DAEMON.registered_factories[
'certificate_generator_factory']
cert_gen = cert_gen_factory.get_cert_generator('localhost')
cert_gen.check_certificates()
cert_gen = None
cert_gen_factory = None
Syslogger.logger().debug('Register atexit')
atexit.register(self.shutdown, 'atexit', '')
for sig in (signal.SIGABRT, signal.SIGILL, signal.SIGINT,
signal.SIGSEGV, signal.SIGTERM):
signal.signal(sig, self.shutdown)
for registered_object in RpcNSMixinDaemon.DAEMON.registered_factories:
obj = RpcNSMixinDaemon.DAEMON.registered_factories[
registered_object]
if type(obj) is not types.TypeType: # noqa
Syslogger.logger().debug('Initialising object %s' %
registered_object)
obj.initialise()
def __init__(self, method, user, object_name, object_type):
# Store information about method being run
self.user = user
self.method = method
self.object_name = object_name
self.object_type = object_type
self.method_name = method.func_name
# Store method state
self.status = LogState.QUEUED
self.exception_message = None
self.exception_mcvirt = False
# Setup date objects for times
self.queue_time = datetime.now()
self.start_time = None
self.finish_time = None
Syslogger.logger().debug(' Queued command: %s' % ', '.join([
str(self.queue_time), self.user or '', self.object_type or '', self.object_name or '',
self.method_name or ''
]))
def run(self):
"""Perform watchdog check"""
Syslogger.logger().debug('Watchdog checking: %s' %
self.virtual_machine.get_name())
Pyro4.current_context.INTERNAL_REQUEST = True
# Ensure that VM is registered locally, running and watchog is enabled
if not (self.virtual_machine.is_watchdog_enabled() and
self.virtual_machine.isRegisteredLocally() and
self.virtual_machine.is_running):
self.set_state(WATCHDOG_STATES.NOT_SUITABLE)
Syslogger.logger().info(
'Watchdog not run: %s' %
self.virtual_machine.get_name())
return
if self.state in [WATCHDOG_STATES.ACTIVE, WATCHDOG_STATES.FAILING]:
self.set_state(WATCHDOG_STATES.WAITING_RESP)
agent_conn = self.virtual_machine.get_agent_connection()
resp = None
try:
resp = agent_conn.wait_lock(command='ping')
except Exception, e:
Syslogger.logger().error(e)
def start(self, *args, **kwargs):
"""Start the Pyro daemon"""
Pyro4.current_context.STARTUP_PERIOD = False
Syslogger.logger().debug('Authentication enabled')
Syslogger.logger().debug('Starting daemon request loop')
with DaemonLock.LOCK:
RpcNSMixinDaemon.DAEMON.requestLoop(*args, **kwargs)
Syslogger.logger().debug('Daemon request loop finished')
def __init__(self):
"""Store required object member variables and create MCVirt object"""
# Initialise Pyro4 with flag to showing that the daemon is being started
Pyro4.current_context.STARTUP_PERIOD = True
# Store nameserver, MCVirt instance and create daemon
self.daemon_lock = DaemonLock()
Pyro4.config.USE_MSG_WAITALL = False
Pyro4.config.CREATE_SOCKET_METHOD = SSLSocket.create_ssl_socket
Pyro4.config.CREATE_BROADCAST_SOCKET_METHOD = SSLSocket.create_broadcast_ssl_socket
Pyro4.config.THREADPOOL_ALLOW_QUEUE = True
Pyro4.config.THREADPOOL_SIZE = 128
self.hostname = get_hostname()
# Ensure that the required SSL certificates exist
ssl_socket = CertificateGeneratorFactory().get_cert_generator('localhost')
ssl_socket.check_certificates(check_client=False)
ssl_socket = None
# Wait for nameserver
Syslogger.logger().debug('Wait for connection to nameserver')
self.obtain_connection()
RpcNSMixinDaemon.DAEMON = BaseRpcDaemon(host=self.hostname)
self.register_factories()
# Ensure libvirt is configured
Syslogger.logger().debug('Start certificate check')
cert_gen_factory = RpcNSMixinDaemon.DAEMON.registered_factories[
'certificate_generator_factory']
cert_gen = cert_gen_factory.get_cert_generator('localhost')
cert_gen.check_certificates()
cert_gen = None
cert_gen_factory = None
Syslogger.logger().debug('Register atexit')
atexit.register(self.shutdown, 'atexit', '')
for sig in (signal.SIGABRT, signal.SIGILL, signal.SIGINT,
signal.SIGSEGV, signal.SIGTERM):
signal.signal(sig, self.shutdown)
for registered_object in RpcNSMixinDaemon.DAEMON.registered_factories:
obj = RpcNSMixinDaemon.DAEMON.registered_factories[registered_object]
if type(obj) is not types.TypeType: # noqa
Syslogger.logger().debug('Initialising object %s' % registered_object)
obj.initialise()
def autostart(self, start_type=AutoStartStates.ON_POLL):
"""Autostart VMs"""
Syslogger.logger().info('Starting autostart: %s' % start_type.name)
for vm in self.getAllVirtualMachines():
if (vm.isRegisteredLocally() and vm.is_stopped
and vm._get_autostart_state()
in [AutoStartStates.ON_POLL, AutoStartStates.ON_BOOT]
and (start_type == vm._get_autostart_state()
or start_type == AutoStartStates.ON_BOOT)):
try:
Syslogger.logger().info('Autostarting: %s' % vm.get_name())
vm.start()
Syslogger.logger().info('Autostart successful: %s' %
vm.get_name())
except Exception, e:
Syslogger.logger().error('Failed to autostart: %s: %s' %
(vm.get_name(), str(e)))
def shutdown(self, signum, frame):
"""Shutdown Pyro Daemon"""
Syslogger.logger().error('Received signal: %s' % signum)
for timer in self.timer_objects:
Syslogger.logger().info('Shutting down timer: %s' % timer)
try:
timer.timer.cancel()
except:
pass
RpcNSMixinDaemon.DAEMON.shutdown()
Syslogger.logger().debug('finisehd shutdown')
def shutdown(self, signum, frame):
"""Shutdown Pyro Daemon"""
Syslogger.logger().error('Received signal: %s' % signum)
for timer in self.timer_objects:
Syslogger.logger().info('Shutting down timer: %s' % timer)
try:
timer.cancel()
except Exception:
pass
RpcNSMixinDaemon.DAEMON.shutdown()
Syslogger.logger().debug('finisehd shutdown')
def runCommand(command_args, raise_exception_on_failure=True, cwd=None):
"""Runs system command, throwing an exception if the exit code is not 0"""
command_process = subprocess.Popen(
command_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=cwd)
Syslogger.logger().debug('Started system command: %s' % ', '.join(command_args))
rc = command_process.wait()
stdout = command_process.stdout.read()
stderr = command_process.stderr.read()
if rc and raise_exception_on_failure:
Syslogger.logger().error("Failed system command: %s\nRC: %s\nStdout: %s\nStderr: %s" %
(', '.join(command_args), rc, stdout, stderr))
raise MCVirtCommandException(('External command failure. '
'See MCVirt log for more information'))
Syslogger.logger().debug("Successful system command: %s\nRC: %s\nStdout: %s\nStderr: %s" %
(', '.join(command_args), rc, stdout, stderr))
return (rc, stdout, stderr)
def runCommand(command_args, raise_exception_on_failure=True, cwd=None):
"""Runs system command, throwing an exception if the exit code is not 0"""
command_process = subprocess.Popen(command_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=cwd)
Syslogger.logger().debug('Started system command: %s' %
', '.join(command_args))
rc = command_process.wait()
stdout = command_process.stdout.read()
stderr = command_process.stderr.read()
if rc and raise_exception_on_failure:
Syslogger.logger().error(
"Failed system command: %s\nRC: %s\nStdout: %s\nStderr: %s" %
(', '.join(command_args), rc, stdout, stderr))
raise MCVirtCommandException(
('External command failure. '
'See MCVirt log for more information'))
Syslogger.logger().debug(
"Successful system command: %s\nRC: %s\nStdout: %s\nStderr: %s" %
(', '.join(command_args), rc, stdout, stderr))
return (rc, stdout, stderr)
def lock_log_and_call(*args, **kwargs):
# Attempt to obtain object type and name for logging
object_name, object_type = getLogNames(callback,
wrapper.instance_method,
wrapper.object_type,
args=args,
kwargs=kwargs)
lock = MethodLock.get_lock()
# If the current Pyro connection has the lock, then do not attempt
# to lock again, as this will be caused by a locking method calling
# another locking method, which should not attempt to re-obtain the lock
requires_lock = (not ('has_lock' in dir(Pyro4.current_context)
and Pyro4.current_context.has_lock))
logger = Logger.get_logger()
if 'proxy_user' in dir(Pyro4.current_context
) and Pyro4.current_context.proxy_user:
username = Pyro4.current_context.proxy_user
elif 'username' in dir(Pyro4.current_context):
username = Pyro4.current_context.username
else:
username = ''
if requires_lock:
log = logger.create_log(callback.func_name,
user=username,
object_name=object_name,
object_type=object_type)
else:
log = None
if requires_lock:
lock.acquire()
# @TODO: lock entire cluster - raise exception if it cannot
# be obtained in short period (~5 seconds)
Pyro4.current_context.has_lock = True
if log:
log.start()
response = None
try:
response = callback(*args, **kwargs)
except MCVirtException as e:
Syslogger.logger().error(
'An internal MCVirt exception occurred in lock')
Syslogger.logger().error("".join(
Pyro4.util.getPyroTraceback()))
if log:
log.finish_error(e)
if requires_lock:
if lock.locked():
lock.release()
Pyro4.current_context.has_lock = False
raise
except Exception as e:
Syslogger.logger().error('Unknown exception occurred in lock')
Syslogger.logger().error("".join(
Pyro4.util.getPyroTraceback()))
if log:
log.finish_error_unknown(e)
if requires_lock:
if lock.locked():
lock.release()
Pyro4.current_context.has_lock = False
raise
if log:
log.finish_success()
if requires_lock:
if lock.locked():
lock.release()
Pyro4.current_context.has_lock = False
return response
def validateHandshake(self, conn,
data): # Override name of upstream method # noqa
"""Perform authentication on new connections"""
# Reset session_id for current context
Pyro4.current_context.STARTUP_PERIOD = False
Pyro4.current_context.INTERNAL_REQUEST = False
Pyro4.current_context.session_id = None
Pyro4.current_context.username = None
Pyro4.current_context.proxy_user = None
Pyro4.current_context.has_lock = False
Pyro4.current_context.cluster_master = True
# Check and store username from connection
if Annotations.USERNAME not in data:
raise Pyro4.errors.SecurityError(
'Username and password or Session must be passed')
username = str(data[Annotations.USERNAME])
# If a password has been provided
try:
# @TODO - Re-factor as the logic below is duplicated for SESSION_ID in data clause
if Annotations.PASSWORD in data:
# Store the password and perform authentication check
password = str(data[Annotations.PASSWORD])
session_instance = self.registered_factories['mcvirt_session']
session_id = session_instance.authenticate_user(
username=username, password=password)
if session_id:
Pyro4.current_context.username = username
Pyro4.current_context.session_id = session_id
# If the authenticated user can specify a proxy user, and a proxy user
# has been specified, set this in the current context
user_object = session_instance.get_current_user_object()
if user_object.allow_proxy_user and Annotations.PROXY_USER in data:
Pyro4.current_context.proxy_user = data[
Annotations.PROXY_USER]
# If the user is a cluster/connection user, treat this connection
# as a cluster client (the command as been executed on a remote node)
# unless specified otherwise
auth = self.registered_factories['auth']
if user_object.CLUSTER_USER:
if Annotations.CLUSTER_MASTER in data:
Pyro4.current_context.cluster_master = data[
Annotations.CLUSTER_MASTER]
else:
Pyro4.current_context.cluster_master = False
else:
Pyro4.current_context.cluster_master = True
if user_object.CLUSTER_USER and Annotations.HAS_LOCK in data:
Pyro4.current_context.has_lock = data[
Annotations.HAS_LOCK]
else:
Pyro4.current_context.has_lock = False
if (auth.check_permission(PERMISSIONS.CAN_IGNORE_CLUSTER,
user_object=user_object)
and Annotations.IGNORE_CLUSTER in data):
Pyro4.current_context.ignore_cluster = data[
Annotations.IGNORE_CLUSTER]
else:
Pyro4.current_context.ignore_cluster = False
if (auth.check_permission(PERMISSIONS.CAN_IGNORE_DRBD,
user_object=user_object)
and Annotations.IGNORE_Drbd in data):
Pyro4.current_context.ignore_drbd = data[
Annotations.IGNORE_Drbd]
else:
Pyro4.current_context.ignore_drbd = False
if Pyro4.current_context.cluster_master:
self.registered_factories[
'cluster'].check_node_versions()
return session_id
# If a session id has been passed, store it and check the
# session_id/username against active sessions
elif Annotations.SESSION_ID in data:
session_id = str(data[Annotations.SESSION_ID])
session_instance = self.registered_factories['mcvirt_session']
if session_instance.authenticate_session(username=username,
session=session_id):
Pyro4.current_context.username = username
Pyro4.current_context.session_id = session_id
# Determine if user can provide alternative users
user_object = session_instance.get_current_user_object()
if user_object.allow_proxy_user and Annotations.PROXY_USER in data:
Pyro4.current_context.proxy_user = data[
Annotations.PROXY_USER]
# If the user is a cluster/connection user, treat this connection
# as a cluster client (the command as been executed on a remote node)
# unless specified otherwise
auth = self.registered_factories['auth']
if user_object.CLUSTER_USER:
if Annotations.CLUSTER_MASTER in data:
Pyro4.current_context.cluster_master = data[
Annotations.CLUSTER_MASTER]
else:
Pyro4.current_context.cluster_master = False
else:
Pyro4.current_context.cluster_master = True
if user_object.CLUSTER_USER and Annotations.HAS_LOCK in data:
Pyro4.current_context.has_lock = data[
Annotations.HAS_LOCK]
else:
Pyro4.current_context.has_lock = False
if (auth.check_permission(PERMISSIONS.CAN_IGNORE_CLUSTER,
user_object=user_object)
and Annotations.IGNORE_CLUSTER in data):
Pyro4.current_context.ignore_cluster = data[
Annotations.IGNORE_CLUSTER]
else:
Pyro4.current_context.ignore_cluster = False
if (auth.check_permission(PERMISSIONS.CAN_IGNORE_DRBD,
user_object=user_object)
and Annotations.IGNORE_Drbd in data):
Pyro4.current_context.ignore_drbd = data[
Annotations.IGNORE_Drbd]
else:
Pyro4.current_context.ignore_drbd = False
if Pyro4.current_context.cluster_master:
self.registered_factories[
'cluster'].check_node_versions()
return session_id
except Pyro4.errors.SecurityError:
raise
except Exception, e:
Syslogger.logger().exception('Error during authentication: %s' %
str(e))
def _register_object(self, local_object):
"""Register an object with the pyro daemon"""
if self._is_pyro_initialised:
Syslogger.logger().debug('Registering object (dynamic): %s' %
local_object)
self._pyroDaemon.register(local_object)
def shutdown(self, signum, frame):
"""Shutdown Pyro Daemon"""
Syslogger.logger().error('Received signal: %s' % signum)
RpcNSMixinDaemon.DAEMON.shutdown()
Syslogger.logger().debug('finisehd shutdown')