def load_helper_modules(self): """ Loads helper modules for post-processing of results """ if not self.config.get('helpers'): return for helper in self.config.get('helpers'): helper = helper.strip() logger.info('Loading helper module %s', helper) try: module = importlib.import_module(helper) except ImportError as e: logger.warning('Cannot import helper module: %s', e) continue if not hasattr(module, 'HelperAgent'): logger.warning( 'Module %s does not provide a HelperAgent interface', helper) continue if not hasattr(module.HelperAgent, 'run'): logger.warning( 'In module %s HelperAgent class does not provide a run() method', helper) continue self.helper_modules[helper] = module
def load_helper_modules(self): """ Loads helper modules for post-processing of results """ if not self.config.get('helpers'): return for helper in self.config.get('helpers'): helper = helper.strip() logger.info('Loading helper module %s', helper) try: module = importlib.import_module(helper) except ImportError as e: logger.warning( 'Cannot import helper module: %s', e ) continue if not hasattr(module, 'HelperAgent'): logger.warning( 'Module %s does not provide a HelperAgent interface', helper ) continue if not hasattr(module.HelperAgent, 'run'): logger.warning( 'In module %s HelperAgent class does not provide a run() method', helper ) continue self.helper_modules[helper] = module
def stop(self): """ Stop the vPoller Proxy process """ logger.info('Proxy process is shutting down') self.close_sockets()
def create_agents(self): """ Prepares the vSphere Agents used by the vPoller Worker Raises: VPollerException """ logger.debug('Creating vSphere Agents') db = VConnectorDatabase(self.config.get('db')) agents = db.get_agents(only_enabled=True) if not agents: logger.warning('No registered or enabled vSphere Agents found') raise VPollerException( 'No registered or enabled vSphere Agents found') for agent in agents: a = VConnector( user=agent['user'], pwd=agent['pwd'], host=agent['host'], cache_enabled=self.config.get('cache_enabled'), cache_maxsize=self.config.get('cache_maxsize'), cache_ttl=self.config.get('cache_ttl'), cache_housekeeping=self.config.get('cache_housekeeping')) self.agents[a.host] = a logger.info('Created vSphere Agent for %s', agent['host'])
def create_agents(self): """ Prepares the vSphere Agents used by the vPoller Worker Raises: VPollerException """ logger.debug('Creating vSphere Agents') db = VConnectorDatabase(self.config.get('db')) agents = db.get_agents(only_enabled=True) if not agents: logger.warning('No registered or enabled vSphere Agents found') raise VPollerException( 'No registered or enabled vSphere Agents found' ) for agent in agents: a = VConnector( user=agent['user'], pwd=agent['pwd'], host=agent['host'], cache_enabled=self.config.get('cache_enabled'), cache_maxsize=self.config.get('cache_maxsize'), cache_ttl=self.config.get('cache_ttl'), cache_housekeeping=self.config.get('cache_housekeeping') ) self.agents[a.host] = a logger.info('Created vSphere Agent for %s', agent['host'])
def start_workers(self): """ Start the vPoller Worker processes """ logger.info('Starting Worker processes') if self.num_workers <= 0: self.num_workers = multiprocessing.cpu_count() logger.info('Concurrency: %d (processes)', self.num_workers) for i in range(self.num_workers): worker = VPollerWorker( db=self.config.get('db'), proxy=self.config.get('proxy'), helpers=self.config.get('helpers'), tasks=self.config.get('tasks'), cache_enabled=self.config.get('cache_enabled'), cache_maxsize=self.config.get('cache_maxsize'), cache_ttl=self.config.get('cache_ttl'), cache_housekeeping=self.config.get('cache_housekeeping')) worker.daemon = True self.workers.append(worker) worker.start()
def start_workers(self): """ Start the vPoller Worker processes """ logger.info('Starting Worker processes') if self.num_workers <= 0: self.num_workers = multiprocessing.cpu_count() logger.info( 'Concurrency: %d (processes)', self.num_workers ) for i in range(self.num_workers): worker = VPollerWorker( db=self.config.get('db'), proxy=self.config.get('proxy'), helpers=self.config.get('helpers'), tasks=self.config.get('tasks'), cache_enabled=self.config.get('cache_enabled'), cache_maxsize=self.config.get('cache_maxsize'), cache_ttl=self.config.get('cache_ttl'), cache_housekeeping=self.config.get('cache_housekeeping') ) worker.daemon = True self.workers.append(worker) worker.start()
def stop(self): """ Stop the vPoller Manager and Workers """ logger.info('Worker Manager is shutting down') self.close_sockets() self.stop_workers()
def stop(self): """ Stop the vPoller Proxy processes """ logger.info('Proxy Manager is shutting down') self.close_sockets() self.stop_proxy_process()
def stop(self): """ Stop vPoller Worker process """ logger.info('Worker process is shutting down') self.close_sockets() self.stop_agents()
def signal_stop(self): """ Signal the vPoller Worker Manager that shutdown time has arrived """ logger.info('Received shutdown signal') self.time_to_die.set() return {'success': 0, 'msg': 'Shutdown time has arrived'}
def stop_proxy_process(self): """ Stop the vPoller Proxy process """ logger.info('Stopping Proxy process') self.proxy.signal_stop() self.proxy.join(3)
def close_sockets(self): """ Closes the ZeroMQ sockets used by the vPoller Proxy Manager """ logger.info('Closing Proxy Manager sockets') self.zpoller.unregister(self.mgmt_socket) self.mgmt_socket.close() self.zcontext.term()
def close_sockets(self): """ Closes the ZeroMQ sockets used by the vPoller Worker """ logger.info('Closing Worker process sockets') self.zpoller.unregister(self.worker_socket) self.worker_socket.close() self.zcontext.term()
def stop_workers(self): """ Stop the vPoller Worker processes """ logger.info('Stopping Worker processes') for worker in self.workers: worker.signal_stop() worker.join(3)
def start_proxy_process(self): """ Start the vPoller Proxy process """ logger.info('Starting Proxy process') self.proxy = VPollerProxy(frontend=self.config.get('frontend'), backend=self.config.get('backend')) self.proxy.daemon = True self.proxy.start()
def create_sockets(self): """ Creates the ZeroMQ sockets used by the vPoller Proxy Manager """ logger.info('Creating Proxy Manager sockets') self.zcontext = zmq.Context() self.mgmt_socket = self.zcontext.socket(zmq.REP) self.mgmt_socket.bind(self.config.get('mgmt')) self.zpoller = zmq.Poller() self.zpoller.register(self.mgmt_socket, zmq.POLLIN)
def close_sockets(self): """ Closes the ZeroMQ sockets used by vPoller Proxy process """ logger.info('Closing Proxy process sockets') self.zpoller.unregister(self.frontend) self.zpoller.unregister(self.backend) self.frontend.close() self.backend.close() self.zcontext.term()
def run(self): logger.info('Proxy process is starting') self.create_sockets() logger.info('Proxy process is ready and running') while not self.time_to_die.is_set(): try: self.distribute_tasks() except KeyboardInterrupt: self.signal_stop() self.stop()
def start_proxy_process(self): """ Start the vPoller Proxy process """ logger.info('Starting Proxy process') self.proxy = VPollerProxy( frontend=self.config.get('frontend'), backend=self.config.get('backend') ) self.proxy.daemon = True self.proxy.start()
def create_sockets(self): """ Creates the ZeroMQ sockets used by the vPoller Worker Creates two sockets: """ logger.info('Creating Worker sockets') self.zcontext = zmq.Context() self.worker_socket = self.zcontext.socket(zmq.DEALER) self.worker_socket.connect(self.config.get('proxy')) self.zpoller = zmq.Poller() self.zpoller.register(self.worker_socket, zmq.POLLIN)
def create_sockets(self): """ Creates the ZeroMQ sockets used by the vPoller Proxy process """ logger.info('Creating Proxy process sockets') self.zcontext = zmq.Context() self.frontend = self.zcontext.socket(zmq.ROUTER) self.backend = self.zcontext.socket(zmq.DEALER) self.frontend.bind(self.config.get('frontend')) self.backend.bind(self.config.get('backend')) self.zpoller = zmq.Poller() self.zpoller.register(self.frontend, zmq.POLLIN) self.zpoller.register(self.backend, zmq.POLLIN)
def start(self): """ Start the vPoller Worker Manager and processes """ logger.info('Starting Worker Manager [%s release]', __version__) self.load_config() self.create_sockets() self.start_workers() logger.info('Worker Manager is ready and running') while not self.time_to_die.is_set(): try: self.wait_for_mgmt_task() except KeyboardInterrupt: self.signal_stop() self.stop()
def load_task_modules(self): """ Loads the task modules """ if not self.config.get('tasks'): raise VPollerException('No task modules provided') for task in self.config.get('tasks'): task = task.strip() logger.info('Loading task module %s', task) try: module = importlib.import_module(task) except ImportError as e: logger.warning('Cannot import task module: %s', e.message) continue self.task_modules[task] = module if not self.task_modules: raise VPollerException('No task modules loaded')
def run(self): """ The main worker method. Args: config (str): Path to the confuguration file for vPoller Worker """ logger.info('Worker process is starting') self.load_task_modules() self.load_helper_modules() self.create_sockets() self.create_agents() logger.info('Worker process is ready and running') while not self.time_to_die.is_set(): try: self.wait_for_tasks() except KeyboardInterrupt: self.signal_stop() self.stop()
def load_task_modules(self): """ Loads the task modules """ if not self.config.get('tasks'): raise VPollerException('No task modules provided') for task in self.config.get('tasks'): task = task.strip() logger.info('Loading task module %s', task) try: module = importlib.import_module(task) except ImportError as e: logger.warning( 'Cannot import task module: %s', e.message ) continue self.task_modules[task] = module if not self.task_modules: raise VPollerException('No task modules loaded')