def test_get_host_status(self): """ Verify that KuberContainerManager().test_get_host_status() works as expected. """ kube_container_mgr = KubeContainerManager(CONFIG) status_struct = {'status': {'use': 'kube'}} for test_data in ( (200, status_struct, False), (200, status_struct['status'], True)): kube_container_mgr.con.get = MagicMock(return_value=MagicMock( status_code=test_data[0], json=MagicMock(return_value=test_data[1]))) status_code, data = kube_container_mgr.get_host_status('10.2.0.2') self.assertEquals(test_data[0], status_code) self.assertEquals(test_data[1], data)
def test_node_registered(self): """ Verify that KubeContainerManager().node_registered() works as expected. """ kube_container_mgr = KubeContainerManager(CONFIG) # First call should return True. The rest should be False. kube_container_mgr.con = MagicMock() kube_container_mgr.con.get = MagicMock(side_effect=( MagicMock(status_code=200), MagicMock(status_code=404), MagicMock(status_code=500))) self.assertTrue(kube_container_mgr.node_registered('test')) self.assertFalse(kube_container_mgr.node_registered('test')) self.assertFalse(kube_container_mgr.node_registered('test'))
def test_host_status_retrieve_with_container_manager(self): """ Verify retrieving Host status when it is in a container manager cluster. """ with mock.patch('cherrypy.engine.publish') as _publish: manager = mock.MagicMock(StoreHandlerManager) kube_container_mgr = KubeContainerManager({ 'server_url': 'http://127.0.0.1:8080', 'token': 'token' }) # A dummy requests.Response response = requests.Response() response.status_code = 200 response._content='{"use": "kube"}' kube_container_mgr._get = mock.MagicMock(return_value=response) manager.list_container_managers.return_value = [kube_container_mgr] _publish.return_value = [manager] test_host = make_new(HOST) test_cluster = make_new(CLUSTER) test_cluster.type = C.CLUSTER_TYPE_KUBERNETES test_cluster.hostset = [test_host.address] # Verify if the host exists the data is returned manager.get.side_effect = ( test_host, test_cluster) manager.list.return_value = Clusters.new(clusters=[test_cluster]) body = self.simulate_request('/api/v0/host/10.2.0.2/status') self.assertEqual(self.srmock.status, falcon.HTTP_200) result = json.loads(body[0]) self.assertEquals(C.CLUSTER_TYPE_KUBERNETES, result['type']) self.assertEquals('available', result['host']['status']) self.assertEquals({'use': 'kube'}, result['container_manager'])
def test_node_registered(self): """ Verify that KuberContainerManager().node_registered() works as expected. """ config = Config( etcd={ 'uri': urlparse('http://127.0.0.1:2379'), }, kubernetes={ 'uri': urlparse('http://127.0.0.1:8080'), 'token': 'token', } ) kube_container_mgr = KubeContainerManager(config) # First call should return True. The rest should be False. kube_container_mgr.con = MagicMock() kube_container_mgr.con.get = MagicMock(side_effect=( MagicMock(status_code=200), MagicMock(status_code=404), MagicMock(status_code=500))) self.assertTrue(kube_container_mgr.node_registered('test')) self.assertFalse(kube_container_mgr.node_registered('test')) self.assertFalse(kube_container_mgr.node_registered('test'))
def investigator(queue, config, run_once=False): """ Investigates new hosts to retrieve and store facts. :param queue: Queue to pull work from. :type queue: Queue.Queue :param config: Configuration information. :type config: commissaire.config.Config """ logger = logging.getLogger('investigator') logger.info('Investigator started') while True: # Statuses follow: # http://commissaire.readthedocs.org/en/latest/enums.html#host-statuses transport = ansibleapi.Transport() to_investigate, ssh_priv_key = queue.get() address = to_investigate['address'] logger.info('{0} is now in investigating.'.format(address)) logger.debug('Investigation details: key={0}, data={1}'.format( to_investigate, ssh_priv_key)) f = tempfile.NamedTemporaryFile(prefix='key', delete=False) key_file = f.name logger.debug('Using {0} as the temporary key location for {1}'.format( key_file, address)) f.write(base64.decodestring(ssh_priv_key)) logger.info('Wrote key for {0}'.format(address)) f.close() key = '/commissaire/hosts/{0}'.format(address) etcd_resp, error = cherrypy.engine.publish('store-get', key)[0] if error: logger.warn('Unable to continue for {0} due to ' '{1}: {2}. Returning...'.format( address, type(error), error)) clean_up_key(key_file) continue data = json.loads(etcd_resp.value) try: result, facts = transport.get_info(address, key_file) data.update(facts) data['last_check'] = datetime.datetime.utcnow().isoformat() data['status'] = 'bootstrapping' logger.info('Facts for {0} retrieved'.format(address)) logger.debug('Data: {0}'.format(data)) except: exc_type, exc_msg, tb = sys.exc_info() logger.warn('Getting info failed for {0}: {1}'.format( address, exc_msg)) data['status'] = 'failed' cherrypy.engine.publish('store-save', key, json.dumps(data))[0] clean_up_key(key_file) if run_once: break continue cherrypy.engine.publish('store-save', key, json.dumps(data))[0] logger.info( 'Finished and stored investigation data for {0}'.format(address)) logger.debug('Finished investigation update for {0}: {1}'.format( address, data)) logger.info('{0} is now in bootstrapping'.format(address)) oscmd = get_oscmd(data['os']) try: result, facts = transport.bootstrap(address, key_file, config, oscmd) data['status'] = 'inactive' cherrypy.engine.publish('store-save', key, json.dumps(data))[0] except: exc_type, exc_msg, tb = sys.exc_info() logger.warn('Unable to start bootstraping for {0}: {1}'.format( address, exc_msg)) data['status'] = 'disassociated' cherrypy.engine.publish('store-save', key, json.dumps(data))[0] clean_up_key(key_file) if run_once: break continue # Verify association with the container manager try: container_mgr = KubeContainerManager(config) # Try 3 times waiting 5 seconds each time before giving up for cnt in range(0, 3): if container_mgr.node_registered(address): logger.info( '{0} has been registered with the container manager.') data['status'] = 'active' break if cnt == 3: msg = 'Could not register with the container manager' logger.warn(msg) raise Exception(msg) logger.debug( '{0} has not been registered with the container manager. ' 'Checking again in 5 seconds...'.format(address)) sleep(5) except: _, exc_msg, _ = sys.exc_info() logger.warn( 'Unable to finish bootstrap for {0} while associating with ' 'the container manager: {1}'.format(address, exc_msg)) data['status'] = 'inactive' cherrypy.engine.publish('store-save', key, json.dumps(data))[0] logger.info('Finished bootstrapping for {0}'.format(address)) logging.debug('Finished bootstrapping for {0}: {1}'.format( address, data)) clean_up_key(key_file) if run_once: logger.info('Exiting due to run_once request.') break logger.info('Investigator stopping')
def investigator(queue, config, store_kwargs={}, run_once=False): """ Investigates new hosts to retrieve and store facts. :param queue: Queue to pull work from. :type queue: Queue.Queue :param config: Configuration information. :type config: commissaire.config.Config :param store_kwargs: Keyword arguments used to make the etcd client. :type store_kwargs: dict """ logger = logging.getLogger('investigator') logger.info('Investigator started') store = etcd.Client(**store_kwargs) while True: # Statuses follow: # http://commissaire.readthedocs.org/en/latest/enums.html#host-statuses to_investigate, ssh_priv_key, remote_user = queue.get() address = to_investigate['address'] logger.info('{0} is now in investigating.'.format(address)) logger.debug( 'Investigation details: key={0}, data={1}, remote_user={2}'.format( to_investigate, ssh_priv_key, remote_user)) transport = ansibleapi.Transport(remote_user) f = tempfile.NamedTemporaryFile(prefix='key', delete=False) key_file = f.name logger.debug( 'Using {0} as the temporary key location for {1}'.format( key_file, address)) f.write(base64.decodestring(ssh_priv_key)) logger.info('Wrote key for {0}'.format(address)) f.close() try: key = '/commissaire/hosts/{0}'.format(address) etcd_resp = store.get(key) except Exception as error: logger.warn( 'Unable to continue for {0} due to ' '{1}: {2}. Returning...'.format(address, type(error), error)) clean_up_key(key_file) continue data = json.loads(etcd_resp.value) try: result, facts = transport.get_info(address, key_file) data.update(facts) data['last_check'] = datetime.datetime.utcnow().isoformat() data['status'] = 'bootstrapping' logger.info('Facts for {0} retrieved'.format(address)) logger.debug('Data: {0}'.format(data)) except: exc_type, exc_msg, tb = sys.exc_info() logger.warn('Getting info failed for {0}: {1}'.format( address, exc_msg)) data['status'] = 'failed' store.write(key, json.dumps(data)) clean_up_key(key_file) if run_once: break continue store.write(key, json.dumps(data)) logger.info( 'Finished and stored investigation data for {0}'.format(address)) logger.debug('Finished investigation update for {0}: {1}'.format( address, data)) logger.info('{0} is now in bootstrapping'.format(address)) oscmd = get_oscmd(data['os']) try: result, facts = transport.bootstrap( address, key_file, config, oscmd) data['status'] = 'inactive' store.write(key, json.dumps(data)) except: exc_type, exc_msg, tb = sys.exc_info() logger.warn('Unable to start bootstraping for {0}: {1}'.format( address, exc_msg)) data['status'] = 'disassociated' store.write(key, json.dumps(data)) clean_up_key(key_file) if run_once: break continue # Verify association with the container manager try: container_mgr = KubeContainerManager(config) # Try 3 times waiting 5 seconds each time before giving up for cnt in range(0, 3): if container_mgr.node_registered(address): logger.info( '{0} has been registered with the container manager.') data['status'] = 'active' break if cnt == 3: msg = 'Could not register with the container manager' logger.warn(msg) raise Exception(msg) logger.debug( '{0} has not been registered with the container manager. ' 'Checking again in 5 seconds...'.format(address)) sleep(5) except: _, exc_msg, _ = sys.exc_info() logger.warn( 'Unable to finish bootstrap for {0} while associating with ' 'the container manager: {1}'.format(address, exc_msg)) data['status'] = 'inactive' store.write(key, json.dumps(data)) logger.info( 'Finished bootstrapping for {0}'.format(address)) logging.debug('Finished bootstrapping for {0}: {1}'.format( address, data)) clean_up_key(key_file) if run_once: logger.info('Exiting due to run_once request.') break logger.info('Investigator stopping')
def investigator(queue, config, store, run_once=False): """ Investigates new hosts to retrieve and store facts. :param queue: Queue to pull work from. :type queue: gevent.queue.Queue :param config: Configuration information. :type config: commissaire.config.Config :param store: Data store to place results. :type store: etcd.Client """ # TODO: Change this to be watch and etcd "queue" and kick off a function # similar to clusterpoolexec logger = logging.getLogger("investigator") logger.info("Investigator started") transport = ansibleapi.Transport() while True: # Statuses follow: # http://commissaire.readthedocs.org/en/latest/enums.html#host-statuses to_investigate, ssh_priv_key = queue.get() address = to_investigate["address"] logger.info("{0} is now in investigating.".format(address)) logger.debug("Investigation details: key={0}, data={1}".format(to_investigate, ssh_priv_key)) f = tempfile.NamedTemporaryFile(prefix="key", delete=False) key_file = f.name logger.debug("Using {0} as the temporary key location for {1}".format(key_file, address)) f.write(base64.decodestring(ssh_priv_key)) logger.debug("Wrote key for {0}".format(address)) f.close() key = "/commissaire/hosts/{0}".format(address) data = json.loads(store.get(key).value) try: result, facts = transport.get_info(address, key_file) data.update(facts) data["last_check"] = datetime.datetime.utcnow().isoformat() data["status"] = "bootstrapping" logger.info("Facts for {0} retrieved".format(address)) except: logger.warn("Getting info failed for {0}".format(address)) data["status"] = "failed" store.set(key, json.dumps(data)) exc_type, exc_msg, tb = sys.exc_info() logger.debug("{0} Exception: {1}".format(address, exc_msg)) clean_up_key(key_file) if run_once: break continue store.set(key, json.dumps(data)) logger.info("Finished and stored investigation data for {0}".format(address)) logger.debug("Finished investigation update for {0}: {1}".format(address, data)) # -- logger.info("{0} is now in bootstrapping".format(address)) oscmd = get_oscmd(data["os"])() try: result, facts = transport.bootstrap(address, key_file, config, oscmd) data["status"] = "inactive" store.set(key, json.dumps(data)) except: logger.warn("Unable to bootstrap {0}".format(address)) exc_type, exc_msg, tb = sys.exc_info() logger.debug("{0} Exception: {1}".format(address, exc_msg)) data["status"] = "disassociated" store.set(key, json.dumps(data)) clean_up_key(key_file) if run_once: break continue # Verify association with the container manager try: container_mgr = KubeContainerManager(config) # Try 3 times waiting 5 seconds each time before giving up for cnt in range(0, 3): if container_mgr.node_registered(address): logger.info("{0} has been registered with the container manager.") data["status"] = "active" break if cnt == 3: raise Exception("Could not register with the container manager") logger.debug( "{0} has not been registered with the container manager. " "Checking again in 5 seconds...".format(address) ) gevent.sleep(5) except: logger.warn("Unable to bootstrap {0}".format(address)) exc = sys.exc_info()[0] logger.debug("{0} Exception: {1}".format(address, exc)) data["status"] = "inactive" store.set(key, json.dumps(data)) logger.info("Finished bootstrapping for {0}".format(address)) logging.debug("Finished bootstrapping for {0}: {1}".format(address, data)) clean_up_key(key_file) if run_once: logger.info("Exiting due to run_once request.") break logger.info("Investigator stopping")
def investigator(queue, config, run_once=False): """ Investigates new hosts to retrieve and store facts. :param queue: Queue to pull work from. :type queue: Queue.Queue :param config: Configuration information. :type config: commissaire.config.Config """ logger = logging.getLogger('investigator') logger.info('Investigator started') while True: # Statuses follow: # http://commissaire.readthedocs.org/en/latest/enums.html#host-statuses store_manager, to_investigate, ssh_priv_key, remote_user = queue.get() address = to_investigate['address'] logger.info('{0} is now in investigating.'.format(address)) logger.debug( 'Investigation details: key={0}, data={1}, remote_user={2}'.format( to_investigate, ssh_priv_key, remote_user)) transport = ansibleapi.Transport(remote_user) try: host = store_manager.get( Host( address=address, status='', os='', cpus=0, memory=0, space=0, last_check='', ssh_priv_key='', remote_user='')) key = TemporarySSHKey(host, logger) key.create() except Exception as error: logger.warn( 'Unable to continue for {0} due to ' '{1}: {2}. Returning...'.format(address, type(error), error)) key.remove() continue try: result, facts = transport.get_info(address, key.path) # recreate the host instance with new data data = json.loads(host.to_json(secure=True)) data.update(facts) host = Host(**data) host.last_check = datetime.datetime.utcnow().isoformat() host.status = 'bootstrapping' logger.info('Facts for {0} retrieved'.format(address)) logger.debug('Data: {0}'.format(host.to_json())) except: exc_type, exc_msg, tb = sys.exc_info() logger.warn('Getting info failed for {0}: {1}'.format( address, exc_msg)) host.status = 'failed' store_manager.save(host) key.remove() if run_once: break continue store_manager.save(host) logger.info( 'Finished and stored investigation data for {0}'.format(address)) logger.debug('Finished investigation update for {0}: {1}'.format( address, host.to_json())) logger.info('{0} is now in bootstrapping'.format(address)) oscmd = get_oscmd(host.os) try: result, facts = transport.bootstrap( address, key.path, config, oscmd, store_manager) host.status = 'inactive' store_manager.save(host) except: exc_type, exc_msg, tb = sys.exc_info() logger.warn('Unable to start bootstraping for {0}: {1}'.format( address, exc_msg)) host.status = 'disassociated' store_manager.save(host) key.remove() if run_once: break continue host.status = cluster_type = C.CLUSTER_TYPE_HOST try: cluster = util.cluster_for_host(address, store_manager) cluster_type = cluster.type except KeyError: # Not part of a cluster pass # Verify association with the container manager if cluster_type == C.CLUSTER_TYPE_KUBERNETES: try: container_mgr = KubeContainerManager(config) # Try 3 times waiting 5 seconds each time before giving up for cnt in range(0, 3): if container_mgr.node_registered(address): logger.info( '{0} has been registered with the ' 'container manager.'.format(address)) host.status = 'active' break if cnt == 3: msg = 'Could not register with the container manager' logger.warn(msg) raise Exception(msg) logger.debug( '{0} has not been registered with the container ' ' manager. Checking again in 5 seconds...'.format( address)) sleep(5) except: _, exc_msg, _ = sys.exc_info() logger.warn( 'Unable to finish bootstrap for {0} while associating ' 'with the container manager: {1}'.format( address, exc_msg)) host.status = 'inactive' store_manager.save(host) logger.info( 'Finished bootstrapping for {0}'.format(address)) logging.debug('Finished bootstrapping for {0}: {1}'.format( address, host.to_json())) key.remove() if run_once: logger.info('Exiting due to run_once request.') break logger.info('Investigator stopping')