def on_get(self, req, resp, address):
"""
Handles retrieval of an existing Host.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param address: The address of the Host being requested.
:type address: str
"""
# If the host is still bootstrapping, the store handler
# won't find it yet. So respond with a fake host status.
if cherrypy.engine.publish('investigator-is-pending', address)[0]:
host = Host.new(address=address, status='investigating')
resp.status = falcon.HTTP_200
req.context['model'] = host
return
# TODO: Verify input
try:
store_manager = cherrypy.engine.publish('get-store-manager')[0]
# TODO: use some kind of global default for Hosts
host = store_manager.get(Host.new(address=address))
resp.status = falcon.HTTP_200
req.context['model'] = host
except:
resp.status = falcon.HTTP_404
return
def on_get(self, req, resp, address):
"""
Handles retrieval of existing Host credentials.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param address: The address of the Host being requested.
:type address: str
"""
# TODO: Verify input
# TODO: Decide if this should be a model or if it makes sense to
# stay a subset off of Host and bypass the req.context
# middleware system.
try:
store_manager = cherrypy.engine.publish('get-store-manager')[0]
host = store_manager.get(Host.new(address=address))
resp.status = falcon.HTTP_200
body = {
'ssh_priv_key': host.ssh_priv_key,
'remote_user': host.remote_user or 'root',
}
resp.body = json.dumps(body)
except:
resp.status = falcon.HTTP_404
return
def _calculate_hosts(self, cluster):
"""
Calculates the hosts metadata for the cluster.
:param cluster: The name of the cluster.
:type cluster: str
"""
# XXX: Not sure which wil be more efficient: fetch all
# the host data in one etcd call and sort through
# them, or fetch the ones we need individually.
# For the MVP phase, fetch all is better.
etcd_resp, error = cherrypy.engine.publish('store-get',
'/commissaire/hosts')[0]
if error:
self.logger.warn('Etcd does not have any hosts. '
'Cannot determine cluster stats.')
return
available = unavailable = total = 0
for child in etcd_resp._children:
host = Host(**json.loads(child['value']))
if host.address in cluster.hostset:
total += 1
if host.status == 'active':
available += 1
else:
unavailable += 1
cluster.hosts['total'] = total
cluster.hosts['available'] = available
cluster.hosts['unavailable'] = unavailable
def on_get(self, req, resp, address):
"""
Handles retrieval of existing Host credentials.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param address: The address of the Host being requested.
:type address: str
"""
# TODO: Verify input
# TODO: Decide if this should be a model or if it makes sense to
# stay a subset off of Host and bypass the req.context
# middleware system.
try:
store_manager = cherrypy.engine.publish('get-store-manager')[0]
host = store_manager.get(Host.new(address=address))
resp.status = falcon.HTTP_200
body = {
'ssh_priv_key': host.ssh_priv_key,
'remote_user': host.remote_user or 'root',
}
resp.body = json.dumps(body)
except:
resp.status = falcon.HTTP_404
return
def on_get(self, req, resp):
"""
Handles GET requests for Hosts.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
"""
try:
hosts_dir = self.store.get('/commissaire/hosts/')
self.logger.debug('Etcd Response: {0}'.format(hosts_dir))
except etcd.EtcdKeyNotFound:
self.logger.warn(
'Etcd does not have any hosts. Returning [] and 404.')
resp.status = falcon.HTTP_404
req.context['model'] = None
return
results = []
# Don't let an empty host directory through
if len(hosts_dir._children):
for host in hosts_dir.leaves:
results.append(Host(**json.loads(host.value)))
resp.status = falcon.HTTP_200
req.context['model'] = Hosts(hosts=results)
else:
self.logger.debug(
'Etcd has a hosts directory but no content.')
resp.status = falcon.HTTP_200
req.context['model'] = None
def test__format_key_with_primary_key(self):
"""
Verify etcd keys are generated correctly whith a primary key.
"""
self.assertEquals(
'/commissaire/hosts/10.0.0.1',
self.instance._format_key(
Host.new(
address='10.0.0.1', status='', os='', cpus=2,
memory=1024, space=1000, last_check='',
ssh_priv_key='', remote_user='')))
def on_delete(self, req, resp, address):
"""
Handles the Deletion of a Host.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param address: The address of the Host being requested.
:type address: str
"""
resp.body = "{}"
try:
Host.delete(address)
resp.status = falcon.HTTP_200
except:
resp.status = falcon.HTTP_404
# Also remove the host from all clusters.
# Note: We've done all we need to for the host deletion,
# so if an error occurs from here just log it and
# return.
try:
clusters = Clusters.retrieve()
except:
self.logger.warn("Etcd does not have any clusters")
return
try:
for cluster_name in clusters.clusters:
self.logger.debug("Checking cluster {0}".format(cluster_name))
cluster = Cluster.retrieve(cluster_name)
if address in cluster.hostset:
self.logger.info("Removing {0} from cluster {1}".format(address, cluster_name))
cluster.hostset.remove(address)
cluster.save(cluster_name)
self.logger.info("{0} has been removed from cluster {1}".format(address, cluster_name))
except:
self.logger.warn("Failed to remove {0} from cluster {1}".format(address, cluster_name))
def test__dispatch(self):
"""
Verify dispatching of operations works properly.
"""
# Test namespace
self.instance._save_on_namespace = mock.MagicMock()
self.instance._dispatch('save', Cluster.new(name='test'))
self.instance._save_on_namespace.assert_called_once()
self.instance._get_on_namespace = mock.MagicMock()
self.instance._dispatch('get', Cluster.new(name='test'))
self.instance._get_on_namespace.assert_called_once()
self.instance._delete_on_namespace = mock.MagicMock()
self.instance._dispatch('delete', Cluster.new(name='test'))
self.instance._delete_on_namespace.assert_called_once()
self.instance._list_on_namespace = mock.MagicMock()
self.instance._dispatch('list', Cluster.new(name='test'))
self.instance._list_on_namespace.assert_called_once()
# Test host
self.instance._save_host = mock.MagicMock()
self.instance._dispatch('save', Host.new(name='test'))
self.instance._save_host.assert_called_once()
self.instance._get_host = mock.MagicMock()
self.instance._dispatch('get', Host.new(name='test'))
self.instance._get_host.assert_called_once()
self.instance._delete_host = mock.MagicMock()
self.instance._dispatch('delete', Host.new(name='test'))
self.instance._delete_host.assert_called_once()
self.instance._list_host = mock.MagicMock()
self.instance._dispatch('list', Host.new(name='test'))
self.instance._list_host.assert_called_once()
def test__dispatch(self):
"""
Verify dispatching of operations works properly.
"""
# Test namespace
self.instance._save_on_namespace = mock.MagicMock()
self.instance._dispatch('save', Cluster.new(name='test'))
self.instance._save_on_namespace.assert_called_once()
self.instance._get_on_namespace = mock.MagicMock()
self.instance._dispatch('get', Cluster.new(name='test'))
self.instance._get_on_namespace.assert_called_once()
self.instance._delete_on_namespace = mock.MagicMock()
self.instance._dispatch('delete', Cluster.new(name='test'))
self.instance._delete_on_namespace.assert_called_once()
self.instance._list_on_namespace = mock.MagicMock()
self.instance._dispatch('list', Cluster.new(name='test'))
self.instance._list_on_namespace.assert_called_once()
# Test host
self.instance._save_host = mock.MagicMock()
self.instance._dispatch('save', Host.new(name='test'))
self.instance._save_host.assert_called_once()
self.instance._get_host = mock.MagicMock()
self.instance._dispatch('get', Host.new(name='test'))
self.instance._get_host.assert_called_once()
self.instance._delete_host = mock.MagicMock()
self.instance._dispatch('delete', Host.new(name='test'))
self.instance._delete_host.assert_called_once()
self.instance._list_host = mock.MagicMock()
self.instance._dispatch('list', Host.new(name='test'))
self.instance._list_host.assert_called_once()
def on_delete(self, req, resp, address):
"""
Handles the Deletion of a Host.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param address: The address of the Host being requested.
:type address: str
"""
resp.body = '{}'
store_manager = cherrypy.engine.publish('get-store-manager')[0]
try:
host = Host.new(address=address)
WATCHER_QUEUE.dequeue(host)
store_manager.delete(host)
self.logger.debug(
'Deleted host {0} and dequeued it from the watcher.'.format(
host.address))
resp.status = falcon.HTTP_200
except:
resp.status = falcon.HTTP_404
# Also remove the host from all clusters.
# Note: We've done all we need to for the host deletion,
# so if an error occurs from here just log it and
# return.
try:
clusters = store_manager.list(Clusters(clusters=[]))
except:
self.logger.warn('Store does not have any clusters')
return
for cluster in clusters.clusters:
try:
self.logger.debug('Checking cluster {0}'.format(cluster.name))
if address in cluster.hostset:
self.logger.info('Removing {0} from cluster {1}'.format(
address, cluster.name))
cluster.hostset.remove(address)
store_manager.save(cluster)
self.logger.info(
'{0} has been removed from cluster {1}'.format(
address, cluster.name))
except:
self.logger.warn(
'Failed to remove {0} from cluster {1}'.format(
address, cluster.name))
def test__format_key_with_primary_key(self):
"""
Verify etcd keys are generated correctly whith a primary key.
"""
self.assertEquals(
'/commissaire/hosts/10.0.0.1',
self.instance._format_key(
Host.new(address='10.0.0.1',
status='',
os='',
cpus=2,
memory=1024,
space=1000,
last_check='',
ssh_priv_key='',
remote_user='')))
def on_delete(self, req, resp, address):
"""
Handles the Deletion of a Host.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param address: The address of the Host being requested.
:type address: str
"""
resp.body = '{}'
store_manager = cherrypy.engine.publish('get-store-manager')[0]
try:
# TODO: use some kind of global default for Hosts
store_manager.delete(Host.new(address=address))
resp.status = falcon.HTTP_200
except:
resp.status = falcon.HTTP_404
# Also remove the host from all clusters.
# Note: We've done all we need to for the host deletion,
# so if an error occurs from here just log it and
# return.
try:
clusters = store_manager.list(Clusters(clusters=[]))
except:
self.logger.warn('Store does not have any clusters')
return
for cluster in clusters.clusters:
try:
self.logger.debug(
'Checking cluster {0}'.format(cluster.name))
if address in cluster.hostset:
self.logger.info(
'Removing {0} from cluster {1}'.format(
address, cluster.name))
cluster.hostset.remove(address)
store_manager.save(cluster)
self.logger.info(
'{0} has been removed from cluster {1}'.format(
address, cluster.name))
except:
self.logger.warn(
'Failed to remove {0} from cluster {1}'.format(
address, cluster.name))
def on_get(self, req, resp, address):
"""
Handles retrieval of an existing Host.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param address: The address of the Host being requested.
:type address: str
"""
# TODO: Verify input
try:
host = Host.retrieve(address)
resp.status = falcon.HTTP_200
req.context["model"] = host
except:
resp.status = falcon.HTTP_404
return
def _list_host(self, model_instance):
"""
Lists data at a location in a store and returns back model instances.
:param model_instance: Model instance to search for and list
:type model_instance: commissaire.model.Model
:returns: A list of models
:rtype: list
"""
hosts = []
path = _model_mapper[model_instance.__class__.__name__]
items = self._store.get(self._endpoint + path).json()
for item in items.get('items'):
try:
hosts.append(self._format_model(item, Host.new(), True))
except (TypeError, KeyError):
# TODO: Add logging
pass
return Hosts.new(hosts=hosts)
def _list_host(self, model_instance):
"""
Lists data at a location in a store and returns back model instances.
:param model_instance: Model instance to search for and list
:type model_instance: commissaire.model.Model
:returns: A list of models
:rtype: list
"""
hosts = []
path = _model_mapper[model_instance.__class__.__name__]
items = self._store.get(self._endpoint + path).json()
for item in items.get('items'):
try:
hosts.append(self._format_model(item, Host.new(), True))
except (TypeError, KeyError):
# TODO: Add logging
pass
return Hosts.new(hosts=hosts)
def test_investigator(self):
"""
Verify the investigator.
"""
with mock.patch('commissaire.transport.ansibleapi.Transport') as _tp:
_tp().get_info.return_value = (
0,
{
'os': 'fedora',
'cpus': 2,
'memory': 11989228,
'space': 487652,
}
)
_tp().bootstrap.return_value = (0, {})
request_queue = Queue()
response_queue = MagicMock(Queue)
to_investigate = {
'address': '10.0.0.2',
'ssh_priv_key': 'dGVzdAo=',
'remote_user': '******'
}
manager = MagicMock(StoreHandlerManager)
manager.get.return_value = Host(**json.loads(self.etcd_host))
request_queue.put_nowait((
manager, to_investigate, Cluster.new().__dict__))
investigator(request_queue, response_queue, run_once=True)
# Investigator saves *after* bootstrapping.
self.assertEquals(0, manager.save.call_count)
self.assertEquals(1, response_queue.put.call_count)
host, error = response_queue.put.call_args[0][0]
self.assertEquals(host.status, 'inactive')
self.assertIsNone(error)
def on_get(self, req, resp, address):
"""
Handles retrieval of an existing Host.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param address: The address of the Host being requested.
:type address: str
"""
# TODO: Verify input
try:
etcd_resp = self.store.get(util.etcd_host_key(address))
self.logger.debug('Etcd Response: {0}'.format(etcd_resp))
except etcd.EtcdKeyNotFound:
resp.status = falcon.HTTP_404
return
resp.status = falcon.HTTP_200
req.context['model'] = Host(**json.loads(etcd_resp.value))
def on_get(self, req, resp, address):
"""
Handles retrieval of an existing Host.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param address: The address of the Host being requested.
:type address: str
"""
# TODO: Verify input
try:
store_manager = cherrypy.engine.publish('get-store-manager')[0]
# TODO: use some kind of global default for Hosts
host = store_manager.get(Host.new(address=address))
resp.status = falcon.HTTP_200
req.context['model'] = host
except:
resp.status = falcon.HTTP_404
return
def on_get(self, req, resp, address):
"""
Handles retrieval of an existing Host.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param address: The address of the Host being requested.
:type address: str
"""
# TODO: Verify input
etcd_resp, error = cherrypy.engine.publish(
'store-get', util.etcd_host_key(address))[0]
self.logger.debug('Etcd Response: {0}'.format(etcd_resp))
if error:
resp.status = falcon.HTTP_404
return
resp.status = falcon.HTTP_200
req.context['model'] = Host(**json.loads(etcd_resp.value))
def on_put(self, req, resp, address):
"""
Handles the creation of a new Host.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param address: The address of the Host being requested.
:type address: str
"""
try:
# Extract what we need from the input data.
# Don't treat it as a skeletal host record.
req_data = req.stream.read()
req_body = json.loads(req_data.decode())
ssh_priv_key = req_body['ssh_priv_key']
# Cluster member is optional.
cluster_name = req_body.get('cluster', None)
except (KeyError, ValueError):
self.logger.info(
'Bad client PUT request for host {0}: {1}'.
format(address, req_data))
resp.status = falcon.HTTP_400
return
key = util.etcd_host_key(address)
try:
etcd_resp = self.store.get(key)
self.logger.debug('Etcd Response: {0}'.format(etcd_resp))
# Check if the request conflicts with the existing host.
existing_host = Host(**json.loads(etcd_resp.value))
if existing_host.ssh_priv_key != ssh_priv_key:
resp.status = falcon.HTTP_409
return
if cluster_name:
try:
assert util.etcd_cluster_has_host(
self.store, cluster_name, address)
except (AssertionError, KeyError):
resp.status = falcon.HTTP_409
return
# Request is compatible with the existing host, so
# we're done. (Not using HTTP_201 since we didn't
# actually create anything.)
resp.status = falcon.HTTP_200
req.context['model'] = existing_host
return
except etcd.EtcdKeyNotFound:
pass
host_creation = {
'address': address,
'ssh_priv_key': ssh_priv_key,
'os': '',
'status': 'investigating',
'cpus': -1,
'memory': -1,
'space': -1,
'last_check': None
}
# Verify the cluster exists, if given. Do it now
# so we can fail before writing anything to etcd.
if cluster_name:
if not util.etcd_cluster_exists(self.store, cluster_name):
resp.status = falcon.HTTP_409
return
host = Host(**host_creation)
new_host = self.store.set(key, host.to_json(secure=True))
INVESTIGATE_QUEUE.put((host_creation, ssh_priv_key))
# Add host to the requested cluster.
if cluster_name:
util.etcd_cluster_add_host(self.store, cluster_name, address)
resp.status = falcon.HTTP_201
req.context['model'] = Host(**json.loads(new_host.value))
def etcd_host_create(address, ssh_priv_key, cluster_name=None):
"""
Creates a new host record in etcd and optionally adds the host to
the specified cluster. Returns a (status, host) tuple where status
is the Falcon HTTP status and host is a Host model instance, which
may be None if an error occurred.
This function is idempotent so long as the host parameters agree
with an existing host record and cluster membership.
:param address: Host address.
:type address: str
:param ssh_priv_key: Host's SSH key, base64-encoded.
:type ssh_priv_key: str
:param cluster_name: Name of the cluster to join, or None
:type cluster_name: str or None
:return: (status, host)
:rtype: tuple
"""
key = etcd_host_key(address)
etcd_resp, error = cherrypy.engine.publish('store-get', key)[0]
if not error:
# Check if the request conflicts with the existing host.
existing_host = Host(**json.loads(etcd_resp.value))
if existing_host.ssh_priv_key != ssh_priv_key:
return (falcon.HTTP_409, None)
if cluster_name:
try:
assert etcd_cluster_has_host(cluster_name, address)
except (AssertionError, KeyError):
return (falcon.HTTP_409, None)
# Request is compatible with the existing host, so
# we're done. (Not using HTTP_201 since we didn't
# actually create anything.)
return (falcon.HTTP_200, existing_host)
host_creation = {
'address': address,
'ssh_priv_key': ssh_priv_key,
'os': '',
'status': 'investigating',
'cpus': -1,
'memory': -1,
'space': -1,
'last_check': None
}
# Verify the cluster exists, if given. Do it now
# so we can fail before writing anything to etcd.
if cluster_name:
if not etcd_cluster_exists(cluster_name):
return (falcon.HTTP_409, None)
host = Host(**host_creation)
new_host, _ = cherrypy.engine.publish(
'store-save', key, host.to_json(secure=True))[0]
# Add host to the requested cluster.
if cluster_name:
etcd_cluster_add_host(cluster_name, address)
INVESTIGATE_QUEUE.put((host_creation, ssh_priv_key))
return (falcon.HTTP_201, Host(**json.loads(new_host.value)))
# Copyright (C) 2016 Red Hat, Inc
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
Host(s) handlers.
"""
import falcon
import etcd
import json
from commissaire.queues import INVESTIGATE_QUEUE
from commissaire.resource import Resource
from commissaire.handlers.models import Cluster, Host, Hosts
import commissaire.handlers.util as util
class HostsResource(Resource):
"""
Resource for working with Hosts.
def etcd_host_create(address, ssh_priv_key, remote_user, cluster_name=None):
"""
Creates a new host record in etcd and optionally adds the host to
the specified cluster. Returns a (status, host) tuple where status
is the Falcon HTTP status and host is a Host model instance, which
may be None if an error occurred.
This function is idempotent so long as the host parameters agree
with an existing host record and cluster membership.
:param address: Host address.
:type address: str
:param ssh_priv_key: Host's SSH key, base64-encoded.
:type ssh_priv_key: str
:param remote_user: The user to use with SSH.
:type remote_user: str
:param cluster_name: Name of the cluster to join, or None
:type cluster_name: str or None
:return: (status, host)
:rtype: tuple
"""
key = etcd_host_key(address)
etcd_resp, error = cherrypy.engine.publish('store-get', key)[0]
if not error:
# Check if the request conflicts with the existing host.
existing_host = Host(**json.loads(etcd_resp.value))
if existing_host.ssh_priv_key != ssh_priv_key:
return (falcon.HTTP_409, None)
if cluster_name:
try:
assert etcd_cluster_has_host(cluster_name, address)
except (AssertionError, KeyError):
return (falcon.HTTP_409, None)
# Request is compatible with the existing host, so
# we're done. (Not using HTTP_201 since we didn't
# actually create anything.)
return (falcon.HTTP_200, existing_host)
host_creation = {
'address': address,
'ssh_priv_key': ssh_priv_key,
'os': '',
'status': 'investigating',
'cpus': -1,
'memory': -1,
'space': -1,
'last_check': None,
'remote_user': remote_user,
}
# Verify the cluster exists, if given. Do it now
# so we can fail before writing anything to etcd.
if cluster_name:
if not etcd_cluster_exists(cluster_name):
return (falcon.HTTP_409, None)
host = Host(**host_creation)
new_host, _ = cherrypy.engine.publish(
'store-save', key, host.to_json(secure=True))[0]
# Add host to the requested cluster.
if cluster_name:
etcd_cluster_add_host(cluster_name, address)
INVESTIGATE_QUEUE.put((host_creation, ssh_priv_key, remote_user))
return (falcon.HTTP_201, Host(**json.loads(new_host.value)))
def investigator(queue, config, run_once=False):
"""
Investigates new hosts to retrieve and store facts.
:param queue: Queue to pull work from.
:type queue: Queue.Queue
:param config: Configuration information.
:type config: commissaire.config.Config
"""
logger = logging.getLogger('investigator')
logger.info('Investigator started')
while True:
# Statuses follow:
# http://commissaire.readthedocs.org/en/latest/enums.html#host-statuses
store_manager, to_investigate, ssh_priv_key, remote_user = queue.get()
address = to_investigate['address']
logger.info('{0} is now in investigating.'.format(address))
logger.debug(
'Investigation details: key={0}, data={1}, remote_user={2}'.format(
to_investigate, ssh_priv_key, remote_user))
transport = ansibleapi.Transport(remote_user)
try:
host = store_manager.get(
Host(
address=address,
status='',
os='',
cpus=0,
memory=0,
space=0,
last_check='',
ssh_priv_key='',
remote_user=''))
key = TemporarySSHKey(host, logger)
key.create()
except Exception as error:
logger.warn(
'Unable to continue for {0} due to '
'{1}: {2}. Returning...'.format(address, type(error), error))
key.remove()
continue
try:
result, facts = transport.get_info(address, key.path)
# recreate the host instance with new data
data = json.loads(host.to_json(secure=True))
data.update(facts)
host = Host(**data)
host.last_check = datetime.datetime.utcnow().isoformat()
host.status = 'bootstrapping'
logger.info('Facts for {0} retrieved'.format(address))
logger.debug('Data: {0}'.format(host.to_json()))
except:
exc_type, exc_msg, tb = sys.exc_info()
logger.warn('Getting info failed for {0}: {1}'.format(
address, exc_msg))
host.status = 'failed'
store_manager.save(host)
key.remove()
if run_once:
break
continue
store_manager.save(host)
logger.info(
'Finished and stored investigation data for {0}'.format(address))
logger.debug('Finished investigation update for {0}: {1}'.format(
address, host.to_json()))
logger.info('{0} is now in bootstrapping'.format(address))
oscmd = get_oscmd(host.os)
try:
result, facts = transport.bootstrap(
address, key.path, config, oscmd, store_manager)
host.status = 'inactive'
store_manager.save(host)
except:
exc_type, exc_msg, tb = sys.exc_info()
logger.warn('Unable to start bootstraping for {0}: {1}'.format(
address, exc_msg))
host.status = 'disassociated'
store_manager.save(host)
key.remove()
if run_once:
break
continue
host.status = cluster_type = C.CLUSTER_TYPE_HOST
try:
cluster = util.cluster_for_host(address, store_manager)
cluster_type = cluster.type
except KeyError:
# Not part of a cluster
pass
# Verify association with the container manager
if cluster_type == C.CLUSTER_TYPE_KUBERNETES:
try:
container_mgr = KubeContainerManager(config)
# Try 3 times waiting 5 seconds each time before giving up
for cnt in range(0, 3):
if container_mgr.node_registered(address):
logger.info(
'{0} has been registered with the '
'container manager.'.format(address))
host.status = 'active'
break
if cnt == 3:
msg = 'Could not register with the container manager'
logger.warn(msg)
raise Exception(msg)
logger.debug(
'{0} has not been registered with the container '
' manager. Checking again in 5 seconds...'.format(
address))
sleep(5)
except:
_, exc_msg, _ = sys.exc_info()
logger.warn(
'Unable to finish bootstrap for {0} while associating '
'with the container manager: {1}'.format(
address, exc_msg))
host.status = 'inactive'
store_manager.save(host)
logger.info(
'Finished bootstrapping for {0}'.format(address))
logging.debug('Finished bootstrapping for {0}: {1}'.format(
address, host.to_json()))
key.remove()
if run_once:
logger.info('Exiting due to run_once request.')
break
logger.info('Investigator stopping')
def on_get(self, req, resp, address):
"""
Handles retrieval of existing Host status.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param address: The address of the Host being requested.
:type address: str
"""
try:
store_manager = cherrypy.engine.publish('get-store-manager')[0]
host = store_manager.get(Host.new(address=address))
self.logger.debug('StatusHost found host {0}'.format(host.address))
status = HostStatus.new(host={
'last_check': host.last_check,
'status': host.status,
})
try:
resp.status = falcon.HTTP_200
cluster = util.cluster_for_host(host.address, store_manager)
status.type = cluster.type
self.logger.debug('Cluster type for {0} is {1}'.format(
host.address, status.type))
if status.type != C.CLUSTER_TYPE_HOST:
try:
container_mgr = store_manager.list_container_managers(
cluster.type)[0]
except Exception as error:
self.logger.error(
'StatusHost for host {0} did not find a '
'container_mgr: {1}: {2}'.format(
host.address, type(error), error))
raise error
self.logger.debug(
'StatusHost for host {0} got container_mgr '
'instance {1}'.format(host.address,
type(container_mgr)))
is_raw = req.get_param_as_bool('raw') or False
self.logger.debug(
'StatusHost raw={0} found host {0} will'.format(
is_raw, host.address))
status_code, result = container_mgr.get_host_status(
host.address, is_raw)
# If we have a raw request ..
if is_raw:
# .. forward the http status as well or fall back to
# service unavailable
resp.status = getattr(
falcon.status_codes,
'HTTP_{0}'.format(status_code),
falcon.status_codes.HTTP_SERVICE_UNAVAILABLE)
status.container_manager = result
else:
# Raise to be caught in host only type
raise KeyError
except KeyError:
# The host is not in a cluster.
self.logger.info(
'Host {0} is not in a cluster. Defaulting to {1}'.format(
host.address, C.CLUSTER_TYPE_HOST))
status.type = C.CLUSTER_TYPE_HOST
self.logger.debug(
'StatusHost end status code: {0} json={1}'.format(
resp.status, status.to_json()))
except Exception as ex:
self.logger.debug(
'Host Status exception caught for {0}: {1}:{2}'.format(
host.address, type(ex), ex))
resp.status = falcon.HTTP_404
return
self.logger.debug('Status for {0}: {1}'.format(host.address,
status.to_json()))
req.context['model'] = status
def etcd_host_create(address, ssh_priv_key, remote_user, cluster_name=None):
"""
Creates a new host record in etcd and optionally adds the host to
the specified cluster. Returns a (status, host) tuple where status
is the Falcon HTTP status and host is a Host model instance, which
may be None if an error occurred.
This function is idempotent so long as the host parameters agree
with an existing host record and cluster membership.
:param address: Host address.
:type address: str
:param ssh_priv_key: Host's SSH key, base64-encoded.
:type ssh_priv_key: str
:param remote_user: The user to use with SSH.
:type remote_user: str
:param cluster_name: Name of the cluster to join, or None
:type cluster_name: str or None
:return: (status, host)
:rtype: tuple
"""
store_manager = cherrypy.engine.publish('get-store-manager')[0]
try:
# Check if the request conflicts with the existing host.
existing_host = store_manager.get(Host.new(address=address))
if existing_host.ssh_priv_key != ssh_priv_key:
return (falcon.HTTP_409, None)
if cluster_name:
try:
assert etcd_cluster_has_host(cluster_name, address)
except (AssertionError, KeyError):
return (falcon.HTTP_409, None)
# Request is compatible with the existing host, so
# we're done. (Not using HTTP_201 since we didn't
# actually create anything.)
return (falcon.HTTP_200, existing_host)
except:
pass
host_creation = Host.new(
address=address,
ssh_priv_key=ssh_priv_key,
status='investigating',
remote_user=remote_user
).__dict__
# Verify the cluster exists, if given. Do it now
# so we can fail before writing anything to etcd.
if cluster_name:
if not etcd_cluster_exists(cluster_name):
return (falcon.HTTP_409, None)
host = Host(**host_creation)
new_host = store_manager.save(host)
# Add host to the requested cluster.
if cluster_name:
etcd_cluster_add_host(cluster_name, address)
manager_clone = store_manager.clone()
job_request = (manager_clone, host_creation, ssh_priv_key, remote_user)
INVESTIGATE_QUEUE.put(job_request)
return (falcon.HTTP_201, new_host)
def etcd_host_create(address, ssh_priv_key, remote_user, cluster_name=None):
"""
Creates a new host record in etcd and optionally adds the host to
the specified cluster. Returns a (status, host) tuple where status
is the Falcon HTTP status and host is a Host model instance, which
may be None if an error occurred.
This function is idempotent so long as the host parameters agree
with an existing host record and cluster membership.
:param address: Host address.
:type address: str
:param ssh_priv_key: Host's SSH key, base64-encoded.
:type ssh_priv_key: str
:param remote_user: The user to use with SSH.
:type remote_user: str
:param cluster_name: Name of the cluster to join, or None
:type cluster_name: str or None
:return: (status, host)
:rtype: tuple
"""
store_manager = cherrypy.engine.publish('get-store-manager')[0]
try:
# Check if the request conflicts with the existing host.
existing_host = store_manager.get(Host.new(address=address))
if existing_host.ssh_priv_key != ssh_priv_key:
return (falcon.HTTP_409, None)
if cluster_name:
try:
assert etcd_cluster_has_host(cluster_name, address)
except (AssertionError, KeyError):
return (falcon.HTTP_409, None)
# Request is compatible with the existing host, so
# we're done. (Not using HTTP_201 since we didn't
# actually create anything.)
return (falcon.HTTP_200, existing_host)
except:
pass
# Verify the cluster exists, if given. Do it now
# so we can fail before writing anything to etcd.
if cluster_name:
cluster = get_cluster_model(cluster_name)
if cluster is None:
return (falcon.HTTP_409, None)
else:
cluster = None
host = Host.new(
address=address,
ssh_priv_key=ssh_priv_key,
status='investigating',
remote_user=remote_user)
def callback(store_manager, host, exception):
if exception is None:
store_manager.save(host)
# Add host to the requested cluster.
if cluster_name:
etcd_cluster_add_host(cluster_name, host.address)
cherrypy.engine.publish(
'investigator-submit', store_manager, host, cluster, callback)
return (falcon.HTTP_201, host)
def on_put(self, req, resp, address):
"""
Handles the creation of a new Host.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param address: The address of the Host being requested.
:type address: str
"""
# TODO: Verify input
try:
host = self.store.get("/commissaire/hosts/{0}".format(address))
resp.status = falcon.HTTP_409
return
except etcd.EtcdKeyNotFound:
pass
data = req.stream.read().decode()
host_creation = json.loads(data)
ssh_priv_key = host_creation["ssh_priv_key"]
host_creation["address"] = address
host_creation["os"] = ""
host_creation["status"] = "investigating"
host_creation["cpus"] = -1
host_creation["memory"] = -1
host_creation["space"] = -1
host_creation["last_check"] = None
# Don't store the cluster name in etcd.
cluster_name = host_creation.pop("cluster", None)
# Verify the cluster exists, if given. Do it now
# so we can fail before writing anything to etcd.
if cluster_name:
# XXX: Based on ClusterSingleHostResource.on_put().
# Add a util module to share common operations.
cluster_key = "/commissaire/clusters/{0}".format(cluster_name)
try:
etcd_resp = self.store.get(cluster_key)
self.logger.info("Request for cluster {0}".format(cluster_name))
self.logger.debug("{0}".format(etcd_resp))
except etcd.EtcdKeyNotFound:
self.logger.info("Request for non-existent cluster {0}.".format(cluster_name))
resp.status = falcon.HTTP_409
return
cluster = Cluster(**json.loads(etcd_resp.value))
hostset = set(cluster.hostset)
hostset.add(address) # Ensures no duplicates
cluster.hostset = list(hostset)
host = Host(**host_creation)
new_host = self.store.set("/commissaire/hosts/{0}".format(address), host.to_json(secure=True))
INVESTIGATE_QUEUE.put((host_creation, ssh_priv_key))
# Add host to the requested cluster.
if cluster_name:
# FIXME: Should guard against races here, since we're fetching
# the cluster record and writing it back with some parts
# unmodified. Use either locking or a conditional write
# with the etcd 'modifiedIndex'. Deferring for now.
self.store.set(cluster_key, cluster.to_json(secure=True))
resp.status = falcon.HTTP_201
req.context["model"] = Host(**json.loads(new_host.value))
' "cpus": 0, "memory": 0, "space": 0,'
' "last_check": ""}')
#: Response JSON for a newly created implicit host (no address given)
INITIAL_IMPLICIT_HOST_JSON = ('{"address": "127.0.0.1",'
' "status": "investigating", "os": "",'
' "cpus": 0, "memory": 0, "space": 0,'
' "last_check": ""}')
#: Credential JSON for tests
HOST_CREDS_JSON = '{"remote_user": "******", "ssh_priv_key": "dGVzdAo="}'
#: HostStatus JSON for tests
HOST_STATUS_JSON = (
'{"type": "host_only", "container_manager": {}, "commissaire": '
'{"status": "available", "last_check": "2016-07-29T20:39:50.529454"}}')
#: Host model for most tests
HOST = Host.new(ssh_priv_key='dGVzdAo=',
remote_user='******',
**json.loads(HOST_JSON))
#: HostStatus model for most tests
HOST_STATUS = HostStatus.new(**json.loads(HOST_STATUS_JSON))
#: Hosts model for most tests
HOSTS = Hosts.new(hosts=[HOST])
#: Cluster model for most tests
CLUSTER = Cluster.new(
name='cluster',
status='ok',
hostset=[],
)
#: Cluster model with HOST for most tests
CLUSTER_WITH_HOST = Cluster.new(
name='cluster',
status='ok',
Returns a new deepcopy of an instance.
"""
return copy.deepcopy(instance)
#: Response JSON for a single host
HOST_JSON = (
'{"address": "10.2.0.2",'
' "status": "available", "os": "atomic",'
' "cpus": 2, "memory": 11989228, "space": 487652,'
' "last_check": "2015-12-17T15:48:18.710454"}')
#: Credential JSON for tests
HOST_CREDS_JSON = '{"remote_user": "******", "ssh_priv_key": "dGVzdAo="}'
#: Host model for most tests
HOST = Host.new(
ssh_priv_key='dGVzdAo=',
remote_user='******',
**json.loads(HOST_JSON))
#: Hosts model for most tests
HOSTS = Hosts.new(
hosts=[HOST]
)
#: Cluster model for most tests
CLUSTER = Cluster.new(
name='cluster',
status='ok',
hostset=[],
)
#: Cluster model with HOST for most tests
CLUSTER_WITH_HOST = Cluster.new(
name='cluster',
status='ok',
