def on_put(self, req, resp, name):
"""
Handles PUT (or "initiate") requests for a Cluster restart.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param name: The name of the Cluster being restarted.
:type name: str
"""
POOLS['clusterexecpool'].spawn(
clusterexec.clusterexec, name, 'restart', self.store)
key = '/commissaire/cluster/{0}/restart'.format(name)
cluster_restart_default = {
'status': 'in_process',
'restarted': [],
'in_process': [],
'started_at': datetime.datetime.utcnow().isoformat(),
'finished_at': None
}
cluster_restart = ClusterRestart(**cluster_restart_default)
self.store.set(key, cluster_restart.to_json())
resp.status = falcon.HTTP_201
req.context['model'] = cluster_restart
def on_put(self, req, resp, name):
"""
Handles PUT (or "initiate") requests for a Cluster restart.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param name: The name of the Cluster being restarted.
:type name: str
"""
# Make sure the cluster name is valid.
if not util.etcd_cluster_exists(name):
self.logger.info(
'Restart PUT requested for nonexistent cluster {0}'.format(
name))
resp.status = falcon.HTTP_404
return
# If the operation is already in progress, return the current
# status with response code 200 OK.
try:
store_manager = cherrypy.engine.publish('get-store-manager')[0]
cluster_restart = store_manager.get(ClusterRestart.new(name=name))
self.logger.debug('Found a ClusterRestart for {0}'.format(name))
if not cluster_restart.finished_at:
self.logger.debug(
'Cluster {0} restart already in progress'.format(name))
resp.status = falcon.HTTP_200
req.context['model'] = cluster_restart
return
except:
# This means one doesn't already exist
pass
# TODO: Move to a poll?
store_manager = cherrypy.engine.publish('get-store-manager')[0]
args = (store_manager.clone(), name, 'restart')
p = Process(target=clusterexec, args=args)
p.start()
self.logger.debug('Started restart in clusterexecpool for {0}'.format(
name))
cluster_restart = ClusterRestart.new(
name=name,
status='in_process',
started_at=datetime.datetime.utcnow().isoformat()
)
store_manager = cherrypy.engine.publish('get-store-manager')[0]
store_manager.save(cluster_restart)
resp.status = falcon.HTTP_201
req.context['model'] = cluster_restart
def on_put(self, req, resp, name):
"""
Handles PUT (or "initiate") requests for a Cluster restart.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param name: The name of the Cluster being restarted.
:type name: str
"""
# Make sure the cluster name is valid.
if not util.etcd_cluster_exists(name):
self.logger.info(
'Restart PUT requested for nonexistent cluster {0}'.format(
name))
resp.status = falcon.HTTP_404
return
# If the operation is already in progress, return the current
# status with response code 200 OK.
try:
store_manager = cherrypy.engine.publish('get-store-manager')[0]
cluster_restart = store_manager.get(ClusterRestart.new(name=name))
self.logger.debug('Found a ClusterRestart for {0}'.format(name))
if not cluster_restart.finished_at:
self.logger.debug(
'Cluster {0} restart already in progress'.format(name))
resp.status = falcon.HTTP_200
req.context['model'] = cluster_restart
return
except:
# This means one doesn't already exist
pass
# TODO: Move to a poll?
store_manager = cherrypy.engine.publish('get-store-manager')[0]
args = (store_manager.clone(), name, 'restart')
p = Process(target=clusterexec, args=args)
p.start()
self.logger.debug('Started restart in clusterexecpool for {0}'.format(
name))
cluster_restart = ClusterRestart.new(
name=name,
status='in_process',
started_at=datetime.datetime.utcnow().isoformat()
)
store_manager = cherrypy.engine.publish('get-store-manager')[0]
store_manager.save(cluster_restart)
resp.status = falcon.HTTP_201
req.context['model'] = cluster_restart
def on_put(self, req, resp, name):
"""
Handles PUT (or "initiate") requests for a Cluster restart.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param name: The name of the Cluster being restarted.
:type name: str
"""
# Make sure the cluster name is valid.
if not util.etcd_cluster_exists(name):
self.logger.info(
'Restart PUT requested for nonexistent cluster {0}'.format(
name))
resp.status = falcon.HTTP_404
return
# If the operation is already in progress, return the current
# status with response code 200 OK.
try:
cluster_restart = ClusterRestart.retrieve(name)
self.logger.debug('Found a ClusterRestart for {0}'.format(name))
if not cluster_restart.finished_at:
self.logger.debug(
'Cluster {0} restart already in progress'.format(name))
resp.status = falcon.HTTP_200
req.context['model'] = cluster_restart
return
except:
# This means one doesn't already exist
pass
# TODO: Move to a poll?
p = Process(target=clusterexec, args=(name, 'restart'))
p.start()
self.logger.debug('Started restart in clusterexecpool for {0}'.format(
name))
cluster_restart_default = {
'status': 'in_process',
'restarted': [],
'in_process': [],
'started_at': datetime.datetime.utcnow().isoformat(),
'finished_at': None
}
cluster_restart = ClusterRestart(**cluster_restart_default)
cluster_restart.save(name)
resp.status = falcon.HTTP_201
req.context['model'] = cluster_restart
def on_put(self, req, resp, name):
"""
Handles PUT (or "initiate") requests for a Cluster restart.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param name: The name of the Cluster being restarted.
:type name: str
"""
# Make sure the cluster name is valid.
if not util.etcd_cluster_exists(self.store, name):
self.logger.info(
'Restart PUT requested for nonexistent cluster {0}'.format(
name))
resp.status = falcon.HTTP_404
return
# If the operation is already in progress, return the current
# status with response code 200 OK.
key = '/commissaire/cluster/{0}/restart'.format(name)
try:
etcd_resp = self.store.get(key)
self.logger.debug('Etcd Response: {0}'.format(etcd_resp))
cluster_restart = ClusterRestart(**json.loads(etcd_resp.value))
if not cluster_restart.finished_at:
self.logger.debug(
'Cluster {0} restart already in progress'.format(name))
resp.status = falcon.HTTP_200
req.context['model'] = cluster_restart
return
except etcd.EtcdKeyNotFound:
pass
POOLS['clusterexecpool'].spawn(
clusterexec.clusterexec, name, 'restart', self.store)
self.logger.debug('Started restart in clusterexecpool for {0}'.format(
name))
cluster_restart_default = {
'status': 'in_process',
'restarted': [],
'in_process': [],
'started_at': datetime.datetime.utcnow().isoformat(),
'finished_at': None
}
cluster_restart = ClusterRestart(**cluster_restart_default)
self.store.set(key, cluster_restart.to_json())
resp.status = falcon.HTTP_201
req.context['model'] = cluster_restart
def on_put(self, req, resp, name):
"""
Handles PUT (or "initiate") requests for a Cluster restart.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param name: The name of the Cluster being restarted.
:type name: str
"""
# Make sure the cluster name is valid.
if not util.etcd_cluster_exists(name):
self.logger.info(
'Restart PUT requested for nonexistent cluster {0}'.format(
name))
resp.status = falcon.HTTP_404
return
# If the operation is already in progress, return the current
# status with response code 200 OK.
key = '/commissaire/cluster/{0}/restart'.format(name)
etcd_resp, error = cherrypy.engine.publish('store-get', key)[0]
if not error:
self.logger.debug('Etcd Response: {0}'.format(etcd_resp))
cluster_restart = ClusterRestart(**json.loads(etcd_resp.value))
if not cluster_restart.finished_at:
self.logger.debug(
'Cluster {0} restart already in progress'.format(name))
resp.status = falcon.HTTP_200
req.context['model'] = cluster_restart
return
# TODO: Move to a poll?
p = Process(target=clusterexec, args=(name, 'restart'))
p.start()
self.logger.debug('Started restart in clusterexecpool for {0}'.format(
name))
cluster_restart_default = {
'status': 'in_process',
'restarted': [],
'in_process': [],
'started_at': datetime.datetime.utcnow().isoformat(),
'finished_at': None
}
cluster_restart = ClusterRestart(**cluster_restart_default)
cherrypy.engine.publish('store-save', key, cluster_restart.to_json())
resp.status = falcon.HTTP_201
req.context['model'] = cluster_restart
def on_put(self, req, resp, name):
"""
Handles PUT (or "initiate") requests for a Cluster restart.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param name: The name of the Cluster being restarted.
:type name: str
"""
# Make sure the cluster name is valid.
if not util.etcd_cluster_exists(name):
self.logger.info(
'Restart PUT requested for nonexistent cluster {0}'.format(
name))
resp.status = falcon.HTTP_404
return
# If the operation is already in progress, return the current
# status with response code 200 OK.
key = '/commissaire/cluster/{0}/restart'.format(name)
etcd_resp, error = cherrypy.engine.publish('store-get', key)[0]
if not error:
self.logger.debug('Etcd Response: {0}'.format(etcd_resp))
cluster_restart = ClusterRestart(**json.loads(etcd_resp.value))
if not cluster_restart.finished_at:
self.logger.debug(
'Cluster {0} restart already in progress'.format(name))
resp.status = falcon.HTTP_200
req.context['model'] = cluster_restart
return
# TODO: Move to a poll?
p = Process(target=clusterexec, args=(name, 'restart'))
p.start()
self.logger.debug(
'Started restart in clusterexecpool for {0}'.format(name))
cluster_restart_default = {
'status': 'in_process',
'restarted': [],
'in_process': [],
'started_at': datetime.datetime.utcnow().isoformat(),
'finished_at': None
}
cluster_restart = ClusterRestart(**cluster_restart_default)
cherrypy.engine.publish('store-save', key, cluster_restart.to_json())
resp.status = falcon.HTTP_201
req.context['model'] = cluster_restart
def on_put(self, req, resp, name):
"""
Handles PUT (or "initiate") requests for a Cluster restart.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param name: The name of the Cluster being restarted.
:type name: str
"""
# Make sure the cluster name is valid.
if not util.etcd_cluster_exists(self.store, name):
self.logger.info(
'Restart PUT requested for nonexistent cluster {0}'.format(
name))
resp.status = falcon.HTTP_404
return
# If the operation is already in progress, return the current
# status with response code 200 OK.
key = '/commissaire/cluster/{0}/restart'.format(name)
try:
etcd_resp = self.store.get(key)
self.logger.debug('Etcd Response: {0}'.format(etcd_resp))
cluster_restart = ClusterRestart(**json.loads(etcd_resp.value))
if not cluster_restart.finished_at:
self.logger.debug(
'Cluster {0} restart already in progress'.format(name))
resp.status = falcon.HTTP_200
req.context['model'] = cluster_restart
return
except etcd.EtcdKeyNotFound:
pass
POOLS['clusterexecpool'].spawn(clusterexec.clusterexec, name,
'restart', self.store)
self.logger.debug(
'Started restart in clusterexecpool for {0}'.format(name))
cluster_restart_default = {
'status': 'in_process',
'restarted': [],
'in_process': [],
'started_at': datetime.datetime.utcnow().isoformat(),
'finished_at': None
}
cluster_restart = ClusterRestart(**cluster_restart_default)
self.store.set(key, cluster_restart.to_json())
resp.status = falcon.HTTP_201
req.context['model'] = cluster_restart
def on_get(self, req, resp, name):
"""
Handles GET (or "status") requests for a Cluster restart.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param name: The name of the Cluster being restarted.
:type name: str
"""
if not util.etcd_cluster_exists(name):
self.logger.info(
'Restart GET requested for nonexistent cluster {0}'.format(
name))
resp.status = falcon.HTTP_404
return
try:
store_manager = cherrypy.engine.publish('get-store-manager')[0]
cluster_restart = store_manager.get(ClusterRestart.new(name=name))
except:
# Return "204 No Content" if we have no status,
# meaning no restart is in progress. The client
# can't be expected to know that, so it's not a
# client error (4xx).
self.logger.debug((
'Restart GET requested for {0} but no restart '
'has ever been executed.').format(name))
resp.status = falcon.HTTP_204
return
resp.status = falcon.HTTP_200
req.context['model'] = cluster_restart
def on_get(self, req, resp, name):
"""
Handles GET (or "status") requests for a Cluster restart.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param name: The name of the Cluster being restarted.
:type name: str
"""
key = '/commissaire/cluster/{0}/restart'.format(name)
if not util.etcd_cluster_exists(name):
self.logger.info(
'Restart GET requested for nonexistent cluster {0}'.format(
name))
resp.status = falcon.HTTP_404
return
status, error = cherrypy.engine.publish('store-get', key)[0]
self.logger.debug('Etcd Response: {0}'.format(status))
if error:
# Return "204 No Content" if we have no status,
# meaning no restart is in progress. The client
# can't be expected to know that, so it's not a
# client error (4xx).
self.logger.debug(('Restart GET requested for {0} but no restart '
'has ever been executed.').format(name))
resp.status = falcon.HTTP_204
return
resp.status = falcon.HTTP_200
req.context['model'] = ClusterRestart(**json.loads(status.value))
def on_get(self, req, resp, name):
"""
Handles GET (or "status") requests for a Cluster restart.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param name: The name of the Cluster being restarted.
:type name: str
"""
if not util.etcd_cluster_exists(name):
self.logger.info(
'Restart GET requested for nonexistent cluster {0}'.format(
name))
resp.status = falcon.HTTP_404
return
try:
store_manager = cherrypy.engine.publish('get-store-manager')[0]
cluster_restart = store_manager.get(ClusterRestart.new(name=name))
except:
# Return "204 No Content" if we have no status,
# meaning no restart is in progress. The client
# can't be expected to know that, so it's not a
# client error (4xx).
self.logger.debug((
'Restart GET requested for {0} but no restart '
'has ever been executed.').format(name))
resp.status = falcon.HTTP_204
return
resp.status = falcon.HTTP_200
req.context['model'] = cluster_restart
#: Cluster model with HOST for most tests
CLUSTER_WITH_HOST = Cluster.new(
name='cluster',
status='ok',
hostset=[HOST],
)
#: Cluster model with flattened HOST for tests
CLUSTER_WITH_FLAT_HOST = Cluster.new(
name='cluster',
status='ok',
hostset=[HOST.address],
)
#: ClusterRestart model for most tests
CLUSTER_RESTART = ClusterRestart.new(name='cluster',
status='ok',
restarted=[],
in_process=[],
started_at='',
finished_at='')
#: ClusterUpgrade model for most tests
CLUSTER_UPGRADE = ClusterUpgrade.new(name='cluster',
status='ok',
upgraded=[],
in_process=[],
started_at='',
finished_at='')
#: ClusterDeploy model for most tests
CLUSTER_DEPLOY = ClusterDeploy.new(name='cluster',
status='ok',
version='1.0',
deployed=[],
in_process=[],
def clusterexec(store_manager, cluster_name, command, kwargs={}):
"""
Remote executes a shell commands across a cluster.
:param store_manager: Proxy object for remtote stores
:type store_manager: commissaire.store.StoreHandlerManager
:param cluster_name: Name of the cluster to act on
:type cluster_name: str
:param command: Top-level command to execute
:type command: str
:param kwargs: Keyword arguments for the command
:type kwargs: dict
"""
logger = logging.getLogger('clusterexec')
# TODO: This is a hack and should really be done elsewhere
command_args = ()
if command == 'upgrade':
finished_hosts_key = 'upgraded'
model_instance = ClusterUpgrade.new(
name=cluster_name,
status='in_process',
started_at=datetime.datetime.utcnow().isoformat(),
upgraded=[],
in_process=[],
)
elif command == 'restart':
finished_hosts_key = 'restarted'
model_instance = ClusterRestart.new(
name=cluster_name,
status='in_process',
started_at=datetime.datetime.utcnow().isoformat(),
restarted=[],
in_process=[],
)
elif command == 'deploy':
finished_hosts_key = 'deployed'
version = kwargs.get('version', '')
command_args = (version,)
model_instance = ClusterDeploy.new(
name=cluster_name,
status='in_process',
started_at=datetime.datetime.utcnow().isoformat(),
version=version,
deployed=[],
in_process=[],
)
end_status = 'finished'
try:
# Set the initial status in the store
logger.info('Setting initial status.')
logger.debug('Status={0}'.format(model_instance.to_json()))
store_manager.save(model_instance)
except Exception as error:
logger.error(
'Unable to save initial state for "{0}" clusterexec due to '
'{1}: {2}'.format(cluster_name, type(error), error))
return
# Collect all host addresses in the cluster
try:
cluster = store_manager.get(Cluster.new(
name=cluster_name, status='', hostset=[]))
except Exception as error:
logger.warn(
'Unable to continue for cluster "{0}" due to '
'{1}: {2}. Returning...'.format(cluster_name, type(error), error))
return
if cluster.hostset:
logger.debug(
'{0} hosts in cluster "{1}"'.format(
len(cluster.hostset), cluster_name))
else:
logger.warn('No hosts in cluster "{0}"'.format(cluster_name))
# TODO: Find better way to do this
try:
hosts = store_manager.list(Hosts(hosts=[]))
except Exception as error:
logger.warn(
'No hosts in the cluster. Error: {0}. Exiting clusterexec'.format(
error))
return
for host in hosts.hosts:
if host.address not in cluster.hostset:
logger.debug(
'Skipping {0} as it is not in this cluster.'.format(
host.address))
continue # Move on to the next one
oscmd = get_oscmd(host.os)
# command_list is only used for logging
command_list = getattr(oscmd, command)(*command_args)
logger.info('Executing {0} on {1}...'.format(
command_list, host.address))
model_instance.in_process.append(host.address)
try:
store_manager.save(model_instance)
except Exception as error:
logger.error(
'Unable to save in_process state for "{0}" clusterexec due to '
'{1}: {2}'.format(cluster_name, type(error), error))
return
key = TemporarySSHKey(host, logger)
key.create()
try:
transport = ansibleapi.Transport(host.remote_user)
exe = getattr(transport, command)
result, facts = exe(
host.address, key.path, oscmd, kwargs)
# XXX: ansibleapi explicitly raises Exception()
except Exception as ex:
# If there was a failure set the end_status and break out
end_status = 'failed'
logger.error('Clusterexec {0} for {1} failed: {2}: {3}'.format(
command, host.address, type(ex), ex))
break
finally:
try:
key.remove()
logger.debug('Removed temporary key file {0}'.format(key.path))
except:
logger.warn(
'Unable to remove the temporary key file: {0}'.format(
key.path))
# Set the finished hosts
new_finished_hosts = getattr(
model_instance, finished_hosts_key) + [host.address]
setattr(
model_instance,
finished_hosts_key,
new_finished_hosts)
try:
idx = model_instance.in_process.index(host.address)
model_instance.in_process.pop(idx)
except ValueError:
logger.warn('Host {0} was not in_process for {1} {2}'.format(
host['address'], command, cluster_name))
try:
store_manager.save(model_instance)
logger.info('Finished executing {0} for {1} in {2}'.format(
command, host.address, cluster_name))
except Exception as error:
logger.error(
'Unable to save cluster state for "{0}" clusterexec due to '
'{1}: {2}'.format(cluster_name, type(error), error))
return
# Final set of command result
model_instance.finished_at = datetime.datetime.utcnow().isoformat()
model_instance.status = end_status
logger.info('Cluster {0} final {1} status: {2}'.format(
cluster_name, command, model_instance.to_json()))
try:
store_manager.save(model_instance)
except Exception as error:
logger.error(
'Unable to save final state for "{0}" clusterexec due to '
'{1}: {2}'.format(cluster_name, type(error), error))
logger.info('Clusterexec stopping')
CLUSTER_WITH_HOST = Cluster.new(
name='cluster',
status='ok',
hostset=[HOST],
)
#: Cluster model with flattened HOST for tests
CLUSTER_WITH_FLAT_HOST = Cluster.new(
name='cluster',
status='ok',
hostset=[HOST.address],
)
#: ClusterRestart model for most tests
CLUSTER_RESTART = ClusterRestart.new(
name='cluster',
status='ok',
restarted=[],
in_process=[],
started_at='',
finished_at= ''
)
#: ClusterUpgrade model for most tests
CLUSTER_UPGRADE = ClusterUpgrade.new(
name='cluster',
status='ok',
upgraded=[],
in_process=[],
started_at='',
finished_at= ''
)
#: ClusterDeploy model for most tests
CLUSTER_DEPLOY = ClusterDeploy.new(
name='cluster',