def on_put(self, req, resp, address):
"""
Handles the creation of a new Host.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param address: The address of the Host being requested.
:type address: str
"""
try:
# Extract what we need from the input data.
# Don't treat it as a skeletal host record.
req_data = req.stream.read()
req_body = json.loads(req_data.decode())
ssh_priv_key = req_body['ssh_priv_key']
# Cluster member is optional.
cluster_name = req_body.get('cluster', None)
except (KeyError, ValueError):
self.logger.info(
'Bad client PUT request for host {0}: {1}'.
format(address, req_data))
resp.status = falcon.HTTP_400
return
key = util.etcd_host_key(address)
try:
etcd_resp = self.store.get(key)
self.logger.debug('Etcd Response: {0}'.format(etcd_resp))
# Check if the request conflicts with the existing host.
existing_host = Host(**json.loads(etcd_resp.value))
if existing_host.ssh_priv_key != ssh_priv_key:
resp.status = falcon.HTTP_409
return
if cluster_name:
try:
assert util.etcd_cluster_has_host(
self.store, cluster_name, address)
except (AssertionError, KeyError):
resp.status = falcon.HTTP_409
return
# Request is compatible with the existing host, so
# we're done. (Not using HTTP_201 since we didn't
# actually create anything.)
resp.status = falcon.HTTP_200
req.context['model'] = existing_host
return
except etcd.EtcdKeyNotFound:
pass
host_creation = {
'address': address,
'ssh_priv_key': ssh_priv_key,
'os': '',
'status': 'investigating',
'cpus': -1,
'memory': -1,
'space': -1,
'last_check': None
}
# Verify the cluster exists, if given. Do it now
# so we can fail before writing anything to etcd.
if cluster_name:
if not util.etcd_cluster_exists(self.store, cluster_name):
resp.status = falcon.HTTP_409
return
host = Host(**host_creation)
new_host = self.store.set(key, host.to_json(secure=True))
INVESTIGATE_QUEUE.put((host_creation, ssh_priv_key))
# Add host to the requested cluster.
if cluster_name:
util.etcd_cluster_add_host(self.store, cluster_name, address)
resp.status = falcon.HTTP_201
req.context['model'] = Host(**json.loads(new_host.value))
def etcd_host_create(address, ssh_priv_key, remote_user, cluster_name=None):
"""
Creates a new host record in etcd and optionally adds the host to
the specified cluster. Returns a (status, host) tuple where status
is the Falcon HTTP status and host is a Host model instance, which
may be None if an error occurred.
This function is idempotent so long as the host parameters agree
with an existing host record and cluster membership.
:param address: Host address.
:type address: str
:param ssh_priv_key: Host's SSH key, base64-encoded.
:type ssh_priv_key: str
:param remote_user: The user to use with SSH.
:type remote_user: str
:param cluster_name: Name of the cluster to join, or None
:type cluster_name: str or None
:return: (status, host)
:rtype: tuple
"""
key = etcd_host_key(address)
etcd_resp, error = cherrypy.engine.publish('store-get', key)[0]
if not error:
# Check if the request conflicts with the existing host.
existing_host = Host(**json.loads(etcd_resp.value))
if existing_host.ssh_priv_key != ssh_priv_key:
return (falcon.HTTP_409, None)
if cluster_name:
try:
assert etcd_cluster_has_host(cluster_name, address)
except (AssertionError, KeyError):
return (falcon.HTTP_409, None)
# Request is compatible with the existing host, so
# we're done. (Not using HTTP_201 since we didn't
# actually create anything.)
return (falcon.HTTP_200, existing_host)
host_creation = {
'address': address,
'ssh_priv_key': ssh_priv_key,
'os': '',
'status': 'investigating',
'cpus': -1,
'memory': -1,
'space': -1,
'last_check': None,
'remote_user': remote_user,
}
# Verify the cluster exists, if given. Do it now
# so we can fail before writing anything to etcd.
if cluster_name:
if not etcd_cluster_exists(cluster_name):
return (falcon.HTTP_409, None)
host = Host(**host_creation)
new_host, _ = cherrypy.engine.publish(
'store-save', key, host.to_json(secure=True))[0]
# Add host to the requested cluster.
if cluster_name:
etcd_cluster_add_host(cluster_name, address)
INVESTIGATE_QUEUE.put((host_creation, ssh_priv_key, remote_user))
return (falcon.HTTP_201, Host(**json.loads(new_host.value)))
def investigator(queue, config, run_once=False):
"""
Investigates new hosts to retrieve and store facts.
:param queue: Queue to pull work from.
:type queue: Queue.Queue
:param config: Configuration information.
:type config: commissaire.config.Config
"""
logger = logging.getLogger('investigator')
logger.info('Investigator started')
while True:
# Statuses follow:
# http://commissaire.readthedocs.org/en/latest/enums.html#host-statuses
store_manager, to_investigate, ssh_priv_key, remote_user = queue.get()
address = to_investigate['address']
logger.info('{0} is now in investigating.'.format(address))
logger.debug(
'Investigation details: key={0}, data={1}, remote_user={2}'.format(
to_investigate, ssh_priv_key, remote_user))
transport = ansibleapi.Transport(remote_user)
try:
host = store_manager.get(
Host(
address=address,
status='',
os='',
cpus=0,
memory=0,
space=0,
last_check='',
ssh_priv_key='',
remote_user=''))
key = TemporarySSHKey(host, logger)
key.create()
except Exception as error:
logger.warn(
'Unable to continue for {0} due to '
'{1}: {2}. Returning...'.format(address, type(error), error))
key.remove()
continue
try:
result, facts = transport.get_info(address, key.path)
# recreate the host instance with new data
data = json.loads(host.to_json(secure=True))
data.update(facts)
host = Host(**data)
host.last_check = datetime.datetime.utcnow().isoformat()
host.status = 'bootstrapping'
logger.info('Facts for {0} retrieved'.format(address))
logger.debug('Data: {0}'.format(host.to_json()))
except:
exc_type, exc_msg, tb = sys.exc_info()
logger.warn('Getting info failed for {0}: {1}'.format(
address, exc_msg))
host.status = 'failed'
store_manager.save(host)
key.remove()
if run_once:
break
continue
store_manager.save(host)
logger.info(
'Finished and stored investigation data for {0}'.format(address))
logger.debug('Finished investigation update for {0}: {1}'.format(
address, host.to_json()))
logger.info('{0} is now in bootstrapping'.format(address))
oscmd = get_oscmd(host.os)
try:
result, facts = transport.bootstrap(
address, key.path, config, oscmd, store_manager)
host.status = 'inactive'
store_manager.save(host)
except:
exc_type, exc_msg, tb = sys.exc_info()
logger.warn('Unable to start bootstraping for {0}: {1}'.format(
address, exc_msg))
host.status = 'disassociated'
store_manager.save(host)
key.remove()
if run_once:
break
continue
host.status = cluster_type = C.CLUSTER_TYPE_HOST
try:
cluster = util.cluster_for_host(address, store_manager)
cluster_type = cluster.type
except KeyError:
# Not part of a cluster
pass
# Verify association with the container manager
if cluster_type == C.CLUSTER_TYPE_KUBERNETES:
try:
container_mgr = KubeContainerManager(config)
# Try 3 times waiting 5 seconds each time before giving up
for cnt in range(0, 3):
if container_mgr.node_registered(address):
logger.info(
'{0} has been registered with the '
'container manager.'.format(address))
host.status = 'active'
break
if cnt == 3:
msg = 'Could not register with the container manager'
logger.warn(msg)
raise Exception(msg)
logger.debug(
'{0} has not been registered with the container '
' manager. Checking again in 5 seconds...'.format(
address))
sleep(5)
except:
_, exc_msg, _ = sys.exc_info()
logger.warn(
'Unable to finish bootstrap for {0} while associating '
'with the container manager: {1}'.format(
address, exc_msg))
host.status = 'inactive'
store_manager.save(host)
logger.info(
'Finished bootstrapping for {0}'.format(address))
logging.debug('Finished bootstrapping for {0}: {1}'.format(
address, host.to_json()))
key.remove()
if run_once:
logger.info('Exiting due to run_once request.')
break
logger.info('Investigator stopping')
def on_put(self, req, resp, address):
"""
Handles the creation of a new Host.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param address: The address of the Host being requested.
:type address: str
"""
# TODO: Verify input
try:
host = self.store.get("/commissaire/hosts/{0}".format(address))
resp.status = falcon.HTTP_409
return
except etcd.EtcdKeyNotFound:
pass
data = req.stream.read().decode()
host_creation = json.loads(data)
ssh_priv_key = host_creation["ssh_priv_key"]
host_creation["address"] = address
host_creation["os"] = ""
host_creation["status"] = "investigating"
host_creation["cpus"] = -1
host_creation["memory"] = -1
host_creation["space"] = -1
host_creation["last_check"] = None
# Don't store the cluster name in etcd.
cluster_name = host_creation.pop("cluster", None)
# Verify the cluster exists, if given. Do it now
# so we can fail before writing anything to etcd.
if cluster_name:
# XXX: Based on ClusterSingleHostResource.on_put().
# Add a util module to share common operations.
cluster_key = "/commissaire/clusters/{0}".format(cluster_name)
try:
etcd_resp = self.store.get(cluster_key)
self.logger.info("Request for cluster {0}".format(cluster_name))
self.logger.debug("{0}".format(etcd_resp))
except etcd.EtcdKeyNotFound:
self.logger.info("Request for non-existent cluster {0}.".format(cluster_name))
resp.status = falcon.HTTP_409
return
cluster = Cluster(**json.loads(etcd_resp.value))
hostset = set(cluster.hostset)
hostset.add(address) # Ensures no duplicates
cluster.hostset = list(hostset)
host = Host(**host_creation)
new_host = self.store.set("/commissaire/hosts/{0}".format(address), host.to_json(secure=True))
INVESTIGATE_QUEUE.put((host_creation, ssh_priv_key))
# Add host to the requested cluster.
if cluster_name:
# FIXME: Should guard against races here, since we're fetching
# the cluster record and writing it back with some parts
# unmodified. Use either locking or a conditional write
# with the etcd 'modifiedIndex'. Deferring for now.
self.store.set(cluster_key, cluster.to_json(secure=True))
resp.status = falcon.HTTP_201
req.context["model"] = Host(**json.loads(new_host.value))
def etcd_host_create(address, ssh_priv_key, cluster_name=None):
"""
Creates a new host record in etcd and optionally adds the host to
the specified cluster. Returns a (status, host) tuple where status
is the Falcon HTTP status and host is a Host model instance, which
may be None if an error occurred.
This function is idempotent so long as the host parameters agree
with an existing host record and cluster membership.
:param address: Host address.
:type address: str
:param ssh_priv_key: Host's SSH key, base64-encoded.
:type ssh_priv_key: str
:param cluster_name: Name of the cluster to join, or None
:type cluster_name: str or None
:return: (status, host)
:rtype: tuple
"""
key = etcd_host_key(address)
etcd_resp, error = cherrypy.engine.publish('store-get', key)[0]
if not error:
# Check if the request conflicts with the existing host.
existing_host = Host(**json.loads(etcd_resp.value))
if existing_host.ssh_priv_key != ssh_priv_key:
return (falcon.HTTP_409, None)
if cluster_name:
try:
assert etcd_cluster_has_host(cluster_name, address)
except (AssertionError, KeyError):
return (falcon.HTTP_409, None)
# Request is compatible with the existing host, so
# we're done. (Not using HTTP_201 since we didn't
# actually create anything.)
return (falcon.HTTP_200, existing_host)
host_creation = {
'address': address,
'ssh_priv_key': ssh_priv_key,
'os': '',
'status': 'investigating',
'cpus': -1,
'memory': -1,
'space': -1,
'last_check': None
}
# Verify the cluster exists, if given. Do it now
# so we can fail before writing anything to etcd.
if cluster_name:
if not etcd_cluster_exists(cluster_name):
return (falcon.HTTP_409, None)
host = Host(**host_creation)
new_host, _ = cherrypy.engine.publish(
'store-save', key, host.to_json(secure=True))[0]
# Add host to the requested cluster.
if cluster_name:
etcd_cluster_add_host(cluster_name, address)
INVESTIGATE_QUEUE.put((host_creation, ssh_priv_key))
return (falcon.HTTP_201, Host(**json.loads(new_host.value)))
