def load_helper_modules(self):
"""
Loads helper modules for post-processing of results
"""
if not self.config.get('helpers'):
return
for helper in self.config.get('helpers'):
helper = helper.strip()
logger.info('Loading helper module %s', helper)
try:
module = importlib.import_module(helper)
except ImportError as e:
logger.warning('Cannot import helper module: %s', e)
continue
if not hasattr(module, 'HelperAgent'):
logger.warning(
'Module %s does not provide a HelperAgent interface',
helper)
continue
if not hasattr(module.HelperAgent, 'run'):
logger.warning(
'In module %s HelperAgent class does not provide a run() method',
helper)
continue
self.helper_modules[helper] = module
def load_helper_modules(self):
"""
Loads helper modules for post-processing of results
"""
if not self.config.get('helpers'):
return
for helper in self.config.get('helpers'):
helper = helper.strip()
logger.info('Loading helper module %s', helper)
try:
module = importlib.import_module(helper)
except ImportError as e:
logger.warning(
'Cannot import helper module: %s',
e
)
continue
if not hasattr(module, 'HelperAgent'):
logger.warning(
'Module %s does not provide a HelperAgent interface',
helper
)
continue
if not hasattr(module.HelperAgent, 'run'):
logger.warning(
'In module %s HelperAgent class does not provide a run() method',
helper
)
continue
self.helper_modules[helper] = module
def stop(self):
"""
Stop the vPoller Proxy process
"""
logger.info('Proxy process is shutting down')
self.close_sockets()
def create_agents(self):
"""
Prepares the vSphere Agents used by the vPoller Worker
Raises:
VPollerException
"""
logger.debug('Creating vSphere Agents')
db = VConnectorDatabase(self.config.get('db'))
agents = db.get_agents(only_enabled=True)
if not agents:
logger.warning('No registered or enabled vSphere Agents found')
raise VPollerException(
'No registered or enabled vSphere Agents found')
for agent in agents:
a = VConnector(
user=agent['user'],
pwd=agent['pwd'],
host=agent['host'],
cache_enabled=self.config.get('cache_enabled'),
cache_maxsize=self.config.get('cache_maxsize'),
cache_ttl=self.config.get('cache_ttl'),
cache_housekeeping=self.config.get('cache_housekeeping'))
self.agents[a.host] = a
logger.info('Created vSphere Agent for %s', agent['host'])
def create_agents(self):
"""
Prepares the vSphere Agents used by the vPoller Worker
Raises:
VPollerException
"""
logger.debug('Creating vSphere Agents')
db = VConnectorDatabase(self.config.get('db'))
agents = db.get_agents(only_enabled=True)
if not agents:
logger.warning('No registered or enabled vSphere Agents found')
raise VPollerException(
'No registered or enabled vSphere Agents found'
)
for agent in agents:
a = VConnector(
user=agent['user'],
pwd=agent['pwd'],
host=agent['host'],
cache_enabled=self.config.get('cache_enabled'),
cache_maxsize=self.config.get('cache_maxsize'),
cache_ttl=self.config.get('cache_ttl'),
cache_housekeeping=self.config.get('cache_housekeeping')
)
self.agents[a.host] = a
logger.info('Created vSphere Agent for %s', agent['host'])
def stop(self):
"""
Stop the vPoller Proxy process
"""
logger.info('Proxy process is shutting down')
self.close_sockets()
def start_workers(self):
"""
Start the vPoller Worker processes
"""
logger.info('Starting Worker processes')
if self.num_workers <= 0:
self.num_workers = multiprocessing.cpu_count()
logger.info('Concurrency: %d (processes)', self.num_workers)
for i in range(self.num_workers):
worker = VPollerWorker(
db=self.config.get('db'),
proxy=self.config.get('proxy'),
helpers=self.config.get('helpers'),
tasks=self.config.get('tasks'),
cache_enabled=self.config.get('cache_enabled'),
cache_maxsize=self.config.get('cache_maxsize'),
cache_ttl=self.config.get('cache_ttl'),
cache_housekeeping=self.config.get('cache_housekeeping'))
worker.daemon = True
self.workers.append(worker)
worker.start()
def start_workers(self):
"""
Start the vPoller Worker processes
"""
logger.info('Starting Worker processes')
if self.num_workers <= 0:
self.num_workers = multiprocessing.cpu_count()
logger.info(
'Concurrency: %d (processes)',
self.num_workers
)
for i in range(self.num_workers):
worker = VPollerWorker(
db=self.config.get('db'),
proxy=self.config.get('proxy'),
helpers=self.config.get('helpers'),
tasks=self.config.get('tasks'),
cache_enabled=self.config.get('cache_enabled'),
cache_maxsize=self.config.get('cache_maxsize'),
cache_ttl=self.config.get('cache_ttl'),
cache_housekeeping=self.config.get('cache_housekeeping')
)
worker.daemon = True
self.workers.append(worker)
worker.start()
def stop(self):
"""
Stop the vPoller Manager and Workers
"""
logger.info('Worker Manager is shutting down')
self.close_sockets()
self.stop_workers()
def stop(self):
"""
Stop the vPoller Proxy processes
"""
logger.info('Proxy Manager is shutting down')
self.close_sockets()
self.stop_proxy_process()
def stop(self):
"""
Stop vPoller Worker process
"""
logger.info('Worker process is shutting down')
self.close_sockets()
self.stop_agents()
def stop(self):
"""
Stop the vPoller Manager and Workers
"""
logger.info('Worker Manager is shutting down')
self.close_sockets()
self.stop_workers()
def stop(self):
"""
Stop vPoller Worker process
"""
logger.info('Worker process is shutting down')
self.close_sockets()
self.stop_agents()
def stop(self):
"""
Stop the vPoller Proxy processes
"""
logger.info('Proxy Manager is shutting down')
self.close_sockets()
self.stop_proxy_process()
def signal_stop(self):
"""
Signal the vPoller Worker Manager that shutdown time has arrived
"""
logger.info('Received shutdown signal')
self.time_to_die.set()
return {'success': 0, 'msg': 'Shutdown time has arrived'}
def stop_proxy_process(self):
"""
Stop the vPoller Proxy process
"""
logger.info('Stopping Proxy process')
self.proxy.signal_stop()
self.proxy.join(3)
def signal_stop(self):
"""
Signal the vPoller Worker Manager that shutdown time has arrived
"""
logger.info('Received shutdown signal')
self.time_to_die.set()
return {'success': 0, 'msg': 'Shutdown time has arrived'}
def close_sockets(self):
"""
Closes the ZeroMQ sockets used by the vPoller Proxy Manager
"""
logger.info('Closing Proxy Manager sockets')
self.zpoller.unregister(self.mgmt_socket)
self.mgmt_socket.close()
self.zcontext.term()
def close_sockets(self):
"""
Closes the ZeroMQ sockets used by the vPoller Proxy Manager
"""
logger.info('Closing Proxy Manager sockets')
self.zpoller.unregister(self.mgmt_socket)
self.mgmt_socket.close()
self.zcontext.term()
def stop_proxy_process(self):
"""
Stop the vPoller Proxy process
"""
logger.info('Stopping Proxy process')
self.proxy.signal_stop()
self.proxy.join(3)
def close_sockets(self):
"""
Closes the ZeroMQ sockets used by the vPoller Worker
"""
logger.info('Closing Worker process sockets')
self.zpoller.unregister(self.worker_socket)
self.worker_socket.close()
self.zcontext.term()
def stop_workers(self):
"""
Stop the vPoller Worker processes
"""
logger.info('Stopping Worker processes')
for worker in self.workers:
worker.signal_stop()
worker.join(3)
def close_sockets(self):
"""
Closes the ZeroMQ sockets used by the vPoller Worker
"""
logger.info('Closing Worker process sockets')
self.zpoller.unregister(self.worker_socket)
self.worker_socket.close()
self.zcontext.term()
def stop_workers(self):
"""
Stop the vPoller Worker processes
"""
logger.info('Stopping Worker processes')
for worker in self.workers:
worker.signal_stop()
worker.join(3)
def start_proxy_process(self):
"""
Start the vPoller Proxy process
"""
logger.info('Starting Proxy process')
self.proxy = VPollerProxy(frontend=self.config.get('frontend'),
backend=self.config.get('backend'))
self.proxy.daemon = True
self.proxy.start()
def create_sockets(self):
"""
Creates the ZeroMQ sockets used by the vPoller Proxy Manager
"""
logger.info('Creating Proxy Manager sockets')
self.zcontext = zmq.Context()
self.mgmt_socket = self.zcontext.socket(zmq.REP)
self.mgmt_socket.bind(self.config.get('mgmt'))
self.zpoller = zmq.Poller()
self.zpoller.register(self.mgmt_socket, zmq.POLLIN)
def create_sockets(self):
"""
Creates the ZeroMQ sockets used by the vPoller Proxy Manager
"""
logger.info('Creating Proxy Manager sockets')
self.zcontext = zmq.Context()
self.mgmt_socket = self.zcontext.socket(zmq.REP)
self.mgmt_socket.bind(self.config.get('mgmt'))
self.zpoller = zmq.Poller()
self.zpoller.register(self.mgmt_socket, zmq.POLLIN)
def close_sockets(self):
"""
Closes the ZeroMQ sockets used by vPoller Proxy process
"""
logger.info('Closing Proxy process sockets')
self.zpoller.unregister(self.frontend)
self.zpoller.unregister(self.backend)
self.frontend.close()
self.backend.close()
self.zcontext.term()
def close_sockets(self):
"""
Closes the ZeroMQ sockets used by vPoller Proxy process
"""
logger.info('Closing Proxy process sockets')
self.zpoller.unregister(self.frontend)
self.zpoller.unregister(self.backend)
self.frontend.close()
self.backend.close()
self.zcontext.term()
def run(self):
logger.info('Proxy process is starting')
self.create_sockets()
logger.info('Proxy process is ready and running')
while not self.time_to_die.is_set():
try:
self.distribute_tasks()
except KeyboardInterrupt:
self.signal_stop()
self.stop()
def run(self):
logger.info('Proxy process is starting')
self.create_sockets()
logger.info('Proxy process is ready and running')
while not self.time_to_die.is_set():
try:
self.distribute_tasks()
except KeyboardInterrupt:
self.signal_stop()
self.stop()
def start_proxy_process(self):
"""
Start the vPoller Proxy process
"""
logger.info('Starting Proxy process')
self.proxy = VPollerProxy(
frontend=self.config.get('frontend'),
backend=self.config.get('backend')
)
self.proxy.daemon = True
self.proxy.start()
def create_sockets(self):
"""
Creates the ZeroMQ sockets used by the vPoller Worker
Creates two sockets:
"""
logger.info('Creating Worker sockets')
self.zcontext = zmq.Context()
self.worker_socket = self.zcontext.socket(zmq.DEALER)
self.worker_socket.connect(self.config.get('proxy'))
self.zpoller = zmq.Poller()
self.zpoller.register(self.worker_socket, zmq.POLLIN)
def create_sockets(self):
"""
Creates the ZeroMQ sockets used by the vPoller Worker
Creates two sockets:
"""
logger.info('Creating Worker sockets')
self.zcontext = zmq.Context()
self.worker_socket = self.zcontext.socket(zmq.DEALER)
self.worker_socket.connect(self.config.get('proxy'))
self.zpoller = zmq.Poller()
self.zpoller.register(self.worker_socket, zmq.POLLIN)
def create_sockets(self):
"""
Creates the ZeroMQ sockets used by the vPoller Proxy process
"""
logger.info('Creating Proxy process sockets')
self.zcontext = zmq.Context()
self.frontend = self.zcontext.socket(zmq.ROUTER)
self.backend = self.zcontext.socket(zmq.DEALER)
self.frontend.bind(self.config.get('frontend'))
self.backend.bind(self.config.get('backend'))
self.zpoller = zmq.Poller()
self.zpoller.register(self.frontend, zmq.POLLIN)
self.zpoller.register(self.backend, zmq.POLLIN)
def create_sockets(self):
"""
Creates the ZeroMQ sockets used by the vPoller Proxy process
"""
logger.info('Creating Proxy process sockets')
self.zcontext = zmq.Context()
self.frontend = self.zcontext.socket(zmq.ROUTER)
self.backend = self.zcontext.socket(zmq.DEALER)
self.frontend.bind(self.config.get('frontend'))
self.backend.bind(self.config.get('backend'))
self.zpoller = zmq.Poller()
self.zpoller.register(self.frontend, zmq.POLLIN)
self.zpoller.register(self.backend, zmq.POLLIN)
def start(self):
"""
Start the vPoller Worker Manager and processes
"""
logger.info('Starting Worker Manager [%s release]', __version__)
self.load_config()
self.create_sockets()
self.start_workers()
logger.info('Worker Manager is ready and running')
while not self.time_to_die.is_set():
try:
self.wait_for_mgmt_task()
except KeyboardInterrupt:
self.signal_stop()
self.stop()
def start(self):
"""
Start the vPoller Worker Manager and processes
"""
logger.info('Starting Worker Manager [%s release]', __version__)
self.load_config()
self.create_sockets()
self.start_workers()
logger.info('Worker Manager is ready and running')
while not self.time_to_die.is_set():
try:
self.wait_for_mgmt_task()
except KeyboardInterrupt:
self.signal_stop()
self.stop()
def load_task_modules(self):
"""
Loads the task modules
"""
if not self.config.get('tasks'):
raise VPollerException('No task modules provided')
for task in self.config.get('tasks'):
task = task.strip()
logger.info('Loading task module %s', task)
try:
module = importlib.import_module(task)
except ImportError as e:
logger.warning('Cannot import task module: %s', e.message)
continue
self.task_modules[task] = module
if not self.task_modules:
raise VPollerException('No task modules loaded')
def run(self):
"""
The main worker method.
Args:
config (str): Path to the confuguration file for vPoller Worker
"""
logger.info('Worker process is starting')
self.load_task_modules()
self.load_helper_modules()
self.create_sockets()
self.create_agents()
logger.info('Worker process is ready and running')
while not self.time_to_die.is_set():
try:
self.wait_for_tasks()
except KeyboardInterrupt:
self.signal_stop()
self.stop()
def load_task_modules(self):
"""
Loads the task modules
"""
if not self.config.get('tasks'):
raise VPollerException('No task modules provided')
for task in self.config.get('tasks'):
task = task.strip()
logger.info('Loading task module %s', task)
try:
module = importlib.import_module(task)
except ImportError as e:
logger.warning(
'Cannot import task module: %s',
e.message
)
continue
self.task_modules[task] = module
if not self.task_modules:
raise VPollerException('No task modules loaded')
def run(self):
"""
The main worker method.
Args:
config (str): Path to the confuguration file for vPoller Worker
"""
logger.info('Worker process is starting')
self.load_task_modules()
self.load_helper_modules()
self.create_sockets()
self.create_agents()
logger.info('Worker process is ready and running')
while not self.time_to_die.is_set():
try:
self.wait_for_tasks()
except KeyboardInterrupt:
self.signal_stop()
self.stop()
