def on_get(self, req, resp, name):
"""
Handles GET (or "status") requests for a tree image deployment
across a Cluster.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param name: The name of the Cluster undergoing deployment.
:type name: str
"""
if not util.etcd_cluster_exists(name):
self.logger.info(
'Deploy GET requested for nonexistent cluster {0}'.format(
name))
resp.status = falcon.HTTP_404
return
try:
store_manager = cherrypy.engine.publish('get-store-manager')[0]
cluster_deploy = store_manager.get(ClusterDeploy.new(name=name))
self.logger.debug('Found ClusterDeploy for {0}'.format(name))
except:
# Return "204 No Content" if we have no status,
# meaning no deployment is in progress. The client
# can't be expected to know that, so it's not a
# client error (4xx).
self.logger.debug((
'Deploy GET requested for {0} but no deployment '
'has ever been executed.').format(name))
resp.status = falcon.HTTP_204
return
resp.status = falcon.HTTP_200
req.context['model'] = cluster_deploy
def on_get(self, req, resp, name):
"""
Handles GET (or "status") requests for a tree image deployment
across a Cluster.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param name: The name of the Cluster undergoing deployment.
:type name: str
"""
if not util.etcd_cluster_exists(name):
self.logger.info(
'Deploy GET requested for nonexistent cluster {0}'.format(
name))
resp.status = falcon.HTTP_404
return
try:
store_manager = cherrypy.engine.publish('get-store-manager')[0]
cluster_deploy = store_manager.get(ClusterDeploy.new(name=name))
self.logger.debug('Found ClusterDeploy for {0}'.format(name))
except:
# Return "204 No Content" if we have no status,
# meaning no deployment is in progress. The client
# can't be expected to know that, so it's not a
# client error (4xx).
self.logger.debug((
'Deploy GET requested for {0} but no deployment '
'has ever been executed.').format(name))
resp.status = falcon.HTTP_204
return
resp.status = falcon.HTTP_200
req.context['model'] = cluster_deploy
def on_put(self, req, resp, name):
"""
Handles PUT (or "initiate") requests for a tree image deployment
across a Cluster.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param name: The name of the Cluster undergoing deployment.
:type name: str
"""
data = req.stream.read().decode()
try:
args = json.loads(data)
version = args['version']
except (KeyError, ValueError):
resp.status = falcon.HTTP_400
return
# Make sure the cluster name is valid.
if not util.etcd_cluster_exists(name):
self.logger.info(
'Deploy PUT requested for nonexistent cluster {0}'.format(
name))
resp.status = falcon.HTTP_404
return
store_manager = cherrypy.engine.publish('get-store-manager')[0]
# If the operation is already in progress and the requested version
# matches, return the current status with response code 200 OK.
# If the requested version conflicts with the operation in progress,
# return the current status with response code 409 Conflict.
try:
cluster_deploy = store_manager.get(ClusterDeploy.new(name=name))
self.logger.debug('Found ClusterDeploy for {0}'.format(name))
if not cluster_deploy.finished_at:
if cluster_deploy.version == version:
self.logger.debug(
'Cluster {0} deployment to {1} already in progress'.
format(name, version))
resp.status = falcon.HTTP_200
else:
self.logger.debug(
'Cluster deployment to {0} requested while '
'deployment to {1} was already in progress'.
format(version, cluster_deploy.version))
resp.status = falcon.HTTP_409
req.context['model'] = cluster_deploy
return
except:
pass
args = (store_manager.clone(), name, 'deploy', {'version': version})
p = Process(target=clusterexec, args=args)
p.start()
self.logger.debug(
'Started deployment to {0} in clusterexecpool for {1}'.format(
version, name))
cluster_deploy = ClusterDeploy.new(
name=name,
status='in_process',
started_at=datetime.datetime.utcnow().isoformat()
)
store_manager.save(cluster_deploy)
resp.status = falcon.HTTP_201
req.context['model'] = cluster_deploy
hostset=[HOST],
)
#: Cluster model with flattened HOST for tests
CLUSTER_WITH_FLAT_HOST = Cluster.new(
name='cluster',
status='ok',
hostset=[HOST.address],
)
#: ClusterRestart model for most tests
CLUSTER_RESTART = ClusterRestart.new(name='cluster',
status='ok',
restarted=[],
in_process=[],
started_at='',
finished_at='')
#: ClusterUpgrade model for most tests
CLUSTER_UPGRADE = ClusterUpgrade.new(name='cluster',
status='ok',
upgraded=[],
in_process=[],
started_at='',
finished_at='')
#: ClusterDeploy model for most tests
CLUSTER_DEPLOY = ClusterDeploy.new(name='cluster',
status='ok',
version='1.0',
deployed=[],
in_process=[],
started_at='',
finished_at='')
def on_put(self, req, resp, name):
"""
Handles PUT (or "initiate") requests for a tree image deployment
across a Cluster.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param name: The name of the Cluster undergoing deployment.
:type name: str
"""
data = req.stream.read().decode()
try:
args = json.loads(data)
version = args['version']
except (KeyError, ValueError):
resp.status = falcon.HTTP_400
return
# Make sure the cluster name is valid.
if not util.etcd_cluster_exists(name):
self.logger.info(
'Deploy PUT requested for nonexistent cluster {0}'.format(
name))
resp.status = falcon.HTTP_404
return
store_manager = cherrypy.engine.publish('get-store-manager')[0]
# If the operation is already in progress and the requested version
# matches, return the current status with response code 200 OK.
# If the requested version conflicts with the operation in progress,
# return the current status with response code 409 Conflict.
try:
cluster_deploy = store_manager.get(ClusterDeploy.new(name=name))
self.logger.debug('Found ClusterDeploy for {0}'.format(name))
if not cluster_deploy.finished_at:
if cluster_deploy.version == version:
self.logger.debug(
'Cluster {0} deployment to {1} already in progress'.
format(name, version))
resp.status = falcon.HTTP_200
else:
self.logger.debug(
'Cluster deployment to {0} requested while '
'deployment to {1} was already in progress'.
format(version, cluster_deploy.version))
resp.status = falcon.HTTP_409
req.context['model'] = cluster_deploy
return
except:
pass
args = (store_manager.clone(), name, 'deploy', {'version': version})
p = Process(target=clusterexec, args=args)
p.start()
self.logger.debug(
'Started deployment to {0} in clusterexecpool for {1}'.format(
version, name))
cluster_deploy = ClusterDeploy.new(
name=name,
status='in_process',
started_at=datetime.datetime.utcnow().isoformat()
)
store_manager.save(cluster_deploy)
resp.status = falcon.HTTP_201
req.context['model'] = cluster_deploy
def on_put(self, req, resp, name):
"""
Handles PUT (or "initiate") requests for a tree image deployment
across a Cluster.
:param req: Request instance that will be passed through.
:type req: falcon.Request
:param resp: Response instance that will be passed through.
:type resp: falcon.Response
:param name: The name of the Cluster undergoing deployment.
:type name: str
"""
data = req.stream.read().decode()
try:
args = json.loads(data)
version = args['version']
except (KeyError, ValueError):
resp.status = falcon.HTTP_400
return
# Make sure the cluster name is valid.
if not util.etcd_cluster_exists(name):
self.logger.info(
'Upgrade PUT requested for nonexistent cluster {0}'.format(
name))
resp.status = falcon.HTTP_404
return
# If the operation is already in progress and the requested version
# matches, return the current status with response code 200 OK.
# If the requested version conflicts with the operation in progress,
# return the current status with response code 409 Conflict.
try:
cluster_deploy = ClusterDeploy.retrieve(name)
self.logger.debug('Found ClusterDeploy for {0}'.format(name))
if not cluster_deploy.finished_at:
if cluster_deploy.version == version:
self.logger.debug(
'Cluster {0} deployment to {1} already in progress'.
format(name, version))
resp.status = falcon.HTTP_200
else:
self.logger.debug(
'Cluster deployment to {0} requested while '
'upgrade to {1} was already in progress'.
format(version, cluster_deploy.version))
resp.status = falcon.HTTP_409
req.context['model'] = cluster_deploy
return
except:
pass
p = Process(target=clusterexec,
args=(name, 'upgrade', {'version': version}))
p.start()
self.logger.debug('Started upgrade in clusterexecpool for {0}'.format(
name))
cluster_deploy_default = {
'status': 'in_process',
'version': version,
'deployed': [],
'in_process': [],
'started_at': datetime.datetime.utcnow().isoformat(),
'finished_at': None
}
cluster_deploy = ClusterDeploy(**cluster_deploy_default)
cluster_deploy.save(name)
resp.status = falcon.HTTP_201
req.context['model'] = cluster_deploy
def clusterexec(store_manager, cluster_name, command, kwargs={}):
"""
Remote executes a shell commands across a cluster.
:param store_manager: Proxy object for remtote stores
:type store_manager: commissaire.store.StoreHandlerManager
:param cluster_name: Name of the cluster to act on
:type cluster_name: str
:param command: Top-level command to execute
:type command: str
:param kwargs: Keyword arguments for the command
:type kwargs: dict
"""
logger = logging.getLogger('clusterexec')
# TODO: This is a hack and should really be done elsewhere
command_args = ()
if command == 'upgrade':
finished_hosts_key = 'upgraded'
model_instance = ClusterUpgrade.new(
name=cluster_name,
status='in_process',
started_at=datetime.datetime.utcnow().isoformat(),
upgraded=[],
in_process=[],
)
elif command == 'restart':
finished_hosts_key = 'restarted'
model_instance = ClusterRestart.new(
name=cluster_name,
status='in_process',
started_at=datetime.datetime.utcnow().isoformat(),
restarted=[],
in_process=[],
)
elif command == 'deploy':
finished_hosts_key = 'deployed'
version = kwargs.get('version', '')
command_args = (version,)
model_instance = ClusterDeploy.new(
name=cluster_name,
status='in_process',
started_at=datetime.datetime.utcnow().isoformat(),
version=version,
deployed=[],
in_process=[],
)
end_status = 'finished'
try:
# Set the initial status in the store
logger.info('Setting initial status.')
logger.debug('Status={0}'.format(model_instance.to_json()))
store_manager.save(model_instance)
except Exception as error:
logger.error(
'Unable to save initial state for "{0}" clusterexec due to '
'{1}: {2}'.format(cluster_name, type(error), error))
return
# Collect all host addresses in the cluster
try:
cluster = store_manager.get(Cluster.new(
name=cluster_name, status='', hostset=[]))
except Exception as error:
logger.warn(
'Unable to continue for cluster "{0}" due to '
'{1}: {2}. Returning...'.format(cluster_name, type(error), error))
return
if cluster.hostset:
logger.debug(
'{0} hosts in cluster "{1}"'.format(
len(cluster.hostset), cluster_name))
else:
logger.warn('No hosts in cluster "{0}"'.format(cluster_name))
# TODO: Find better way to do this
try:
hosts = store_manager.list(Hosts(hosts=[]))
except Exception as error:
logger.warn(
'No hosts in the cluster. Error: {0}. Exiting clusterexec'.format(
error))
return
for host in hosts.hosts:
if host.address not in cluster.hostset:
logger.debug(
'Skipping {0} as it is not in this cluster.'.format(
host.address))
continue # Move on to the next one
oscmd = get_oscmd(host.os)
# command_list is only used for logging
command_list = getattr(oscmd, command)(*command_args)
logger.info('Executing {0} on {1}...'.format(
command_list, host.address))
model_instance.in_process.append(host.address)
try:
store_manager.save(model_instance)
except Exception as error:
logger.error(
'Unable to save in_process state for "{0}" clusterexec due to '
'{1}: {2}'.format(cluster_name, type(error), error))
return
key = TemporarySSHKey(host, logger)
key.create()
try:
transport = ansibleapi.Transport(host.remote_user)
exe = getattr(transport, command)
result, facts = exe(
host.address, key.path, oscmd, kwargs)
# XXX: ansibleapi explicitly raises Exception()
except Exception as ex:
# If there was a failure set the end_status and break out
end_status = 'failed'
logger.error('Clusterexec {0} for {1} failed: {2}: {3}'.format(
command, host.address, type(ex), ex))
break
finally:
try:
key.remove()
logger.debug('Removed temporary key file {0}'.format(key.path))
except:
logger.warn(
'Unable to remove the temporary key file: {0}'.format(
key.path))
# Set the finished hosts
new_finished_hosts = getattr(
model_instance, finished_hosts_key) + [host.address]
setattr(
model_instance,
finished_hosts_key,
new_finished_hosts)
try:
idx = model_instance.in_process.index(host.address)
model_instance.in_process.pop(idx)
except ValueError:
logger.warn('Host {0} was not in_process for {1} {2}'.format(
host['address'], command, cluster_name))
try:
store_manager.save(model_instance)
logger.info('Finished executing {0} for {1} in {2}'.format(
command, host.address, cluster_name))
except Exception as error:
logger.error(
'Unable to save cluster state for "{0}" clusterexec due to '
'{1}: {2}'.format(cluster_name, type(error), error))
return
# Final set of command result
model_instance.finished_at = datetime.datetime.utcnow().isoformat()
model_instance.status = end_status
logger.info('Cluster {0} final {1} status: {2}'.format(
cluster_name, command, model_instance.to_json()))
try:
store_manager.save(model_instance)
except Exception as error:
logger.error(
'Unable to save final state for "{0}" clusterexec due to '
'{1}: {2}'.format(cluster_name, type(error), error))
logger.info('Clusterexec stopping')
hostset=[HOST.address],
)
#: ClusterRestart model for most tests
CLUSTER_RESTART = ClusterRestart.new(
name='cluster',
status='ok',
restarted=[],
in_process=[],
started_at='',
finished_at= ''
)
#: ClusterUpgrade model for most tests
CLUSTER_UPGRADE = ClusterUpgrade.new(
name='cluster',
status='ok',
upgraded=[],
in_process=[],
started_at='',
finished_at= ''
)
#: ClusterDeploy model for most tests
CLUSTER_DEPLOY = ClusterDeploy.new(
name='cluster',
status='ok',
version='1.0',
deployed=[],
in_process=[],
started_at='',
finished_at= ''
)