def check_before_deployment(self, supertask):
# checking admin intersection with untagged
network_info = self.serialize_network_cfg(self.cluster)
network_info["networks"] = [n for n in network_info["networks"] if n["name"] != "fuelweb_admin"]
check_networks = supertask.create_subtask(TASK_NAMES.check_networks)
self._call_silently(check_networks, tasks.CheckNetworksTask, data=network_info, check_admin_untagged=True)
if check_networks.status == TASK_STATUSES.error:
logger.warning("Checking networks failed: %s", check_networks.message)
raise errors.CheckBeforeDeploymentError(check_networks.message)
TaskHelper.set_ready_if_not_finished(check_networks)
db().delete(check_networks)
db().refresh(supertask)
db().flush()
# checking prerequisites
check_before = supertask.create_subtask(TASK_NAMES.check_before_deployment)
logger.debug("Checking prerequisites task: %s", check_before.uuid)
self._call_silently(check_before, tasks.CheckBeforeDeploymentTask)
# if failed to check prerequisites
# then task is already set to error
if check_before.status == TASK_STATUSES.error:
logger.warning("Checking prerequisites failed: %s", check_before.message)
raise errors.CheckBeforeDeploymentError(check_before.message)
logger.debug("Checking prerequisites is successful, starting deployment...")
TaskHelper.set_ready_if_not_finished(check_before)
db().delete(check_before)
db().refresh(supertask)
db().flush()
def execute(self, data, check_admin_untagged=False):
# Make a copy of original 'data' due to being changed by
# 'tasks.CheckNetworksTask'
data_copy = copy.deepcopy(data)
locked_tasks = objects.TaskCollection.filter_by(
None,
cluster_id=self.cluster.id,
name=consts.TASK_NAMES.check_networks)
locked_tasks = objects.TaskCollection.order_by(locked_tasks, 'id')
check_networks = objects.TaskCollection.lock_for_update(
locked_tasks).first()
if check_networks:
TaskHelper.set_ready_if_not_finished(check_networks)
db().delete(check_networks)
db().flush()
task = Task(name=consts.TASK_NAMES.check_networks,
cluster=self.cluster)
db().add(task)
db().commit()
self._call_silently(task, tasks.CheckNetworksTask, data_copy,
check_admin_untagged)
task = objects.Task.get_by_uid(task.id,
fail_if_not_found=True,
lock_for_update=True)
if task.status == consts.TASK_STATUSES.running:
# update task status with given data
objects.Task.update(task, {
'status': consts.TASK_STATUSES.ready,
'progress': 100
})
db().commit()
return task
def check_before_deployment(self, supertask):
"""Performs checks before deployment
:param supertask: task SqlAlchemy object
"""
# checking admin intersection with untagged
network_info = self.serialize_network_cfg(self.cluster)
network_info["networks"] = [
n for n in network_info["networks"] if n["name"] != "fuelweb_admin"
]
check_repo_connect = supertask.create_subtask(
consts.TASK_NAMES.check_networks)
self._call_silently(
check_repo_connect,
tasks.CheckRepositoryConnectionFromMasterNodeTask,
)
if check_repo_connect.status == consts.TASK_STATUSES.error:
logger.warning("Checking connectivity to repositories failed: %s",
check_repo_connect.message)
raise errors.CheckBeforeDeploymentError(check_repo_connect.message)
check_networks = supertask.create_subtask(
consts.TASK_NAMES.check_networks)
self._call_silently(check_networks,
tasks.CheckNetworksTask,
data=network_info,
check_admin_untagged=True)
if check_networks.status == consts.TASK_STATUSES.error:
logger.warning("Checking networks failed: %s",
check_networks.message)
raise errors.CheckBeforeDeploymentError(check_networks.message)
TaskHelper.set_ready_if_not_finished(check_networks)
db().delete(check_networks)
db().refresh(supertask)
db().flush()
# checking prerequisites
check_before = supertask.create_subtask(
consts.TASK_NAMES.check_before_deployment)
logger.debug("Checking prerequisites task: %s", check_before.uuid)
self._call_silently(check_before, tasks.CheckBeforeDeploymentTask)
# if failed to check prerequisites
# then task is already set to error
if check_before.status == consts.TASK_STATUSES.error:
logger.warning("Checking prerequisites failed: %s",
check_before.message)
raise errors.CheckBeforeDeploymentError(check_before.message)
logger.debug(
"Checking prerequisites is successful, starting deployment...")
TaskHelper.set_ready_if_not_finished(check_before)
db().delete(check_before)
db().refresh(supertask)
db().flush()
def execute(self):
# locking required tasks
locked_tasks = objects.TaskCollection.lock_cluster_tasks(
self.cluster.id)
# locking cluster
objects.Cluster.get_by_uid(self.cluster.id,
fail_if_not_found=True,
lock_for_update=True)
# locking nodes
nodes = objects.NodeCollection.filter_by(None,
cluster_id=self.cluster.id)
nodes = objects.NodeCollection.order_by(nodes, 'id')
objects.NodeCollection.lock_for_update(nodes).all()
current_cluster_tasks = objects.TaskCollection.filter_by_list(
locked_tasks, 'name', (consts.TASK_NAMES.cluster_deletion, ))
deploy_running = objects.TaskCollection.filter_by(
None,
cluster_id=self.cluster.id,
name=consts.TASK_NAMES.deploy,
status=consts.TASK_STATUSES.running)
deploy_running = objects.TaskCollection.order_by(deploy_running,
'id').first()
if deploy_running:
logger.error(u"Deleting cluster '{0}' "
"while deployment is still running".format(
self.cluster.name))
# Updating action logs for deploy task
TaskHelper.set_ready_if_not_finished(deploy_running)
logger.debug("Removing cluster tasks")
for task in current_cluster_tasks:
if task.status == consts.TASK_STATUSES.running:
db().rollback()
raise errors.DeletionAlreadyStarted()
elif task.status in (consts.TASK_STATUSES.ready,
consts.TASK_STATUSES.error):
for subtask in task.subtasks:
db().delete(subtask)
db().delete(task)
db().flush()
logger.debug("Labeling cluster nodes to delete")
for node in self.cluster.nodes:
node.pending_deletion = True
db().add(node)
db().flush()
self.cluster.status = consts.CLUSTER_STATUSES.remove
db().add(self.cluster)
logger.debug("Creating cluster deletion task")
task = Task(name=consts.TASK_NAMES.cluster_deletion,
cluster=self.cluster)
db().add(task)
db().commit()
self._call_silently(task, tasks.ClusterDeletionTask)
return task
def check_before_deployment(self, supertask):
"""Performs checks before deployment
:param supertask: task SqlAlchemy object
"""
try:
# if there are VIPs with same names in the network configuration
# the error will be raised. Such situation may occur when, for
# example, enabled plugins contain conflicting network
# configuration
network_info = self.serialize_network_cfg(self.cluster)
except (errors.DuplicatedVIPNames, errors.NetworkRoleConflict) as e:
raise errors.CheckBeforeDeploymentError(e.message)
logger.info(u"Network info:\n{0}".format(
jsonutils.dumps(network_info, indent=4)))
# checking admin intersection with untagged
network_info["networks"] = [
n for n in network_info["networks"] if n["name"] != "fuelweb_admin"
]
check_networks = supertask.create_subtask(
consts.TASK_NAMES.check_networks)
self._call_silently(check_networks,
tasks.CheckNetworksTask,
data=network_info,
check_all_parameters=True)
if check_networks.status == consts.TASK_STATUSES.error:
logger.warning("Checking networks failed: %s",
check_networks.message)
raise errors.CheckBeforeDeploymentError(check_networks.message)
TaskHelper.set_ready_if_not_finished(check_networks)
db().delete(check_networks)
db().refresh(supertask)
db().flush()
# checking prerequisites
check_before = supertask.create_subtask(
consts.TASK_NAMES.check_before_deployment)
logger.debug("Checking prerequisites task: %s", check_before.uuid)
self._call_silently(check_before, tasks.CheckBeforeDeploymentTask)
# if failed to check prerequisites
# then task is already set to error
if check_before.status == consts.TASK_STATUSES.error:
logger.warning("Checking prerequisites failed: %s",
check_before.message)
raise errors.CheckBeforeDeploymentError(check_before.message)
logger.debug(
"Checking prerequisites is successful, starting deployment...")
TaskHelper.set_ready_if_not_finished(check_before)
db().delete(check_before)
db().refresh(supertask)
db().flush()
def execute(self, data, check_all_parameters=False, **kwargs):
# Make a copy of original 'data' due to being changed by
# 'tasks.CheckNetworksTask'
data_copy = copy.deepcopy(data)
locked_tasks = objects.TaskCollection.filter_by(
None,
cluster_id=self.cluster.id,
name=consts.TASK_NAMES.check_networks
)
locked_tasks = objects.TaskCollection.order_by(locked_tasks, 'id')
check_networks = objects.TaskCollection.lock_for_update(
locked_tasks
).first()
if check_networks:
TaskHelper.set_ready_if_not_finished(check_networks)
db().delete(check_networks)
db().flush()
task = Task(
name=consts.TASK_NAMES.check_networks,
cluster=self.cluster
)
db().add(task)
db().commit()
self._call_silently(
task,
tasks.CheckNetworksTask,
data_copy,
check_all_parameters
)
task = objects.Task.get_by_uid(
task.id,
fail_if_not_found=True,
lock_for_update=True
)
if task.status == consts.TASK_STATUSES.running:
# update task status with given data
objects.Task.update(
task,
{'status': consts.TASK_STATUSES.ready, 'progress': 100})
db().commit()
return task
def clear_tasks_history(self, force=False):
try:
self.check_running_task(delete_obsolete=False)
except errors.TaskAlreadyRunning:
if not force:
raise
logger.error(
u"Force stop running tasks for cluster %s", self.cluster.name
)
running_tasks = objects.TaskCollection.all_in_progress(
self.cluster.id
)
for task in running_tasks:
# Force set task to finished state and update action log
TaskHelper.set_ready_if_not_finished(task)
# clear tasks history
cluster_tasks = objects.TransactionCollection.get_transactions(
self.cluster.id
)
cluster_tasks.delete(synchronize_session='fetch')
def execute(self, data, check_admin_untagged=False):
locked_tasks = objects.TaskCollection.filter_by(
None,
cluster_id=self.cluster.id,
name=consts.TASK_NAMES.check_networks
)
locked_tasks = objects.TaskCollection.order_by(locked_tasks, 'id')
check_networks = objects.TaskCollection.lock_for_update(
locked_tasks
).first()
if check_networks:
TaskHelper.set_ready_if_not_finished(check_networks)
db().delete(check_networks)
db().flush()
task = Task(
name=consts.TASK_NAMES.check_networks,
cluster=self.cluster
)
db().add(task)
db().commit()
self._call_silently(
task,
tasks.CheckNetworksTask,
data,
check_admin_untagged
)
task = objects.Task.get_by_uid(
task.id,
fail_if_not_found=True,
lock_for_update=True
)
if task.status == consts.TASK_STATUSES.running:
# update task status with given data
data = {'status': consts.TASK_STATUSES.ready, 'progress': 100}
objects.Task.update(task, data)
db().commit()
return task
def clear_tasks_history(self, force=False):
try:
self.check_running_task(delete_obsolete=False)
except errors.TaskAlreadyRunning:
if not force:
raise
logger.error(
u"Force stop running tasks for cluster %s", self.cluster.name
)
running_tasks = objects.TaskCollection.all_in_progress(
self.cluster.id
)
for task in running_tasks:
# Force set task to finished state and update action log
TaskHelper.set_ready_if_not_finished(task)
# clear tasks history
cluster_tasks = objects.TaskCollection.get_cluster_tasks(
self.cluster.id
)
cluster_tasks.delete(synchronize_session='fetch')
def execute(self, **kwargs):
stop_running = objects.TaskCollection.filter_by(
None,
cluster_id=self.cluster.id,
name=consts.TASK_NAMES.stop_deployment
)
stop_running = objects.TaskCollection.order_by(
stop_running, 'id'
).first()
if stop_running:
if stop_running.status in (
consts.TASK_STATUSES.running,
consts.TASK_STATUSES.pending):
raise errors.StopAlreadyRunning(
"Stopping deployment task "
"is already launched"
)
else:
db().delete(stop_running)
db().commit()
deployment_task = objects.TaskCollection.filter_by(
None,
cluster_id=self.cluster.id,
name=consts.TASK_NAMES.deployment,
)
deployment_task = deployment_task.filter(
Task.status != consts.TASK_STATUSES.pending
)
deployment_task = objects.TaskCollection.order_by(
deployment_task, '-id'
).first()
provisioning_task = objects.TaskCollection.filter_by(
None,
cluster_id=self.cluster.id,
name=consts.TASK_NAMES.provision,
)
provisioning_task = provisioning_task.filter(
Task.status != consts.TASK_STATUSES.pending
)
provisioning_task = objects.TaskCollection.order_by(
provisioning_task, '-id'
).first()
if not deployment_task and not provisioning_task:
db().rollback()
raise errors.DeploymentNotRunning(
u"Nothing to stop - deployment is "
u"not running on environment '{0}'".format(
self.cluster.id
)
)
# Updating action logs for deploy task
deploy_task = objects.TaskCollection.filter_by(
None,
cluster_id=self.cluster.id,
name=consts.TASK_NAMES.deploy
)
deploy_task = objects.TaskCollection.order_by(
deploy_task, 'id').first()
if deploy_task:
TaskHelper.set_ready_if_not_finished(deploy_task)
db().commit()
task = Task(
name=consts.TASK_NAMES.stop_deployment,
cluster=self.cluster
)
db().add(task)
db().commit()
self._call_silently(
task,
tasks.StopDeploymentTask,
deploy_task=deployment_task,
provision_task=provisioning_task
)
return task
def check_before_deployment(self, supertask):
"""Performs checks before deployment
:param supertask: task SqlAlchemy object
"""
try:
# if there are VIPs with same names in the network configuration
# the error will be raised. Such situation may occur when, for
# example, enabled plugins contain conflicting network
# configuration
network_info = self.serialize_network_cfg(self.cluster)
except (errors.DuplicatedVIPNames, errors.NetworkRoleConflict) as e:
raise errors.CheckBeforeDeploymentError(e.message)
logger.info(
u"Network info:\n{0}".format(
jsonutils.dumps(network_info, indent=4)
)
)
# checking admin intersection with untagged
network_info["networks"] = [
n for n in network_info["networks"] if n["name"] != "fuelweb_admin"
]
check_networks = supertask.create_subtask(
consts.TASK_NAMES.check_networks)
self._call_silently(
check_networks,
tasks.CheckNetworksTask,
data=network_info,
check_all_parameters=True
)
if check_networks.status == consts.TASK_STATUSES.error:
logger.warning(
"Checking networks failed: %s", check_networks.message
)
raise errors.CheckBeforeDeploymentError(check_networks.message)
TaskHelper.set_ready_if_not_finished(check_networks)
db().delete(check_networks)
db().refresh(supertask)
db().flush()
# checking prerequisites
check_before = supertask.create_subtask(
consts.TASK_NAMES.check_before_deployment
)
logger.debug("Checking prerequisites task: %s", check_before.uuid)
self._call_silently(
check_before,
tasks.CheckBeforeDeploymentTask
)
# if failed to check prerequisites
# then task is already set to error
if check_before.status == consts.TASK_STATUSES.error:
logger.warning(
"Checking prerequisites failed: %s", check_before.message
)
raise errors.CheckBeforeDeploymentError(check_before.message)
logger.debug(
"Checking prerequisites is successful, starting deployment..."
)
TaskHelper.set_ready_if_not_finished(check_before)
db().delete(check_before)
db().refresh(supertask)
db().flush()
def execute(self, **kwargs):
current_tasks = objects.TaskCollection.get_cluster_tasks(
self.cluster.id, names=(consts.TASK_NAMES.cluster_deletion,)
)
# locking cluster
objects.Cluster.get_by_uid(
self.cluster.id,
fail_if_not_found=True,
lock_for_update=True
)
# locking nodes
nodes = objects.NodeCollection.filter_by(
None,
cluster_id=self.cluster.id
)
nodes = objects.NodeCollection.order_by(nodes, 'id')
objects.NodeCollection.lock_for_update(nodes).all()
deploy_running = objects.TaskCollection.filter_by(
None,
cluster_id=self.cluster.id,
name=consts.TASK_NAMES.deploy,
status=consts.TASK_STATUSES.running
)
deploy_running = objects.TaskCollection.order_by(
deploy_running,
'id'
).first()
if deploy_running:
logger.error(
u"Deleting cluster '{0}' "
"while deployment is still running".format(
self.cluster.name
)
)
# Updating action logs for deploy task
TaskHelper.set_ready_if_not_finished(deploy_running)
logger.debug("Removing cluster tasks")
for task in current_tasks:
if task.status == consts.TASK_STATUSES.running:
db().rollback()
raise errors.DeletionAlreadyStarted()
elif task.status in (consts.TASK_STATUSES.ready,
consts.TASK_STATUSES.error):
for subtask in task.subtasks:
db().delete(subtask)
db().delete(task)
db().flush()
logger.debug("Labeling cluster nodes to delete")
for node in self.cluster.nodes:
node.pending_deletion = True
db().add(node)
db().flush()
self.cluster.status = consts.CLUSTER_STATUSES.remove
db().add(self.cluster)
logger.debug("Creating cluster deletion task")
task = Task(name=consts.TASK_NAMES.cluster_deletion,
cluster=self.cluster)
db().add(task)
db().commit()
self._call_silently(
task,
tasks.ClusterDeletionTask
)
return task
def execute(self, **kwargs):
try:
self.check_running_task([
consts.TASK_NAMES.stop_deployment,
consts.TASK_NAMES.reset_environment,
consts.TASK_NAMES.cluster_deletion,
])
except errors.TaskAlreadyRunning:
raise errors.TaskAlreadyRunning(
"Stopping deployment task is already launched"
)
deployment_task = objects.TaskCollection.filter_by(
None,
cluster_id=self.cluster.id,
name=consts.TASK_NAMES.deployment,
)
deployment_task = deployment_task.filter(
Task.status != consts.TASK_STATUSES.pending
)
deployment_task = objects.TaskCollection.order_by(
deployment_task, '-id'
).first()
provisioning_task = objects.TaskCollection.filter_by(
None,
cluster_id=self.cluster.id,
name=consts.TASK_NAMES.provision,
)
provisioning_task = provisioning_task.filter(
Task.status != consts.TASK_STATUSES.pending
)
provisioning_task = objects.TaskCollection.order_by(
provisioning_task, '-id'
).first()
if not deployment_task and not provisioning_task:
db().rollback()
raise errors.DeploymentNotRunning(
u"Nothing to stop - deployment is "
u"not running on environment '{0}'".format(
self.cluster.id
)
)
# Updating action logs for deploy task
deploy_task = objects.TaskCollection.filter_by(
None,
cluster_id=self.cluster.id,
name=consts.TASK_NAMES.deploy
)
deploy_task = objects.TaskCollection.order_by(
deploy_task, 'id').first()
if deploy_task:
TaskHelper.set_ready_if_not_finished(deploy_task)
db().commit()
task = Task(
name=consts.TASK_NAMES.stop_deployment,
cluster=self.cluster
)
db().add(task)
db().commit()
self._call_silently(
task,
tasks.StopDeploymentTask,
deploy_task=deployment_task,
provision_task=provisioning_task
)
return task
def execute(self):
# locking tasks for processing
names = (consts.TASK_NAMES.deploy, consts.TASK_NAMES.stop_deployment,
consts.TASK_NAMES.deployment, consts.TASK_NAMES.provision)
objects.TaskCollection.lock_cluster_tasks(self.cluster.id, names=names)
stop_running = objects.TaskCollection.filter_by(
None,
cluster_id=self.cluster.id,
name=consts.TASK_NAMES.stop_deployment,
)
stop_running = objects.TaskCollection.order_by(stop_running,
'id').first()
if stop_running:
if stop_running.status == consts.TASK_STATUSES.running:
raise errors.StopAlreadyRunning("Stopping deployment task "
"is already launched")
else:
db().delete(stop_running)
db().flush()
deployment_task = objects.TaskCollection.filter_by(
None,
cluster_id=self.cluster.id,
name=consts.TASK_NAMES.deployment,
)
deployment_task = objects.TaskCollection.order_by(
deployment_task, 'id').first()
provisioning_task = objects.TaskCollection.filter_by(
None,
cluster_id=self.cluster.id,
name=consts.TASK_NAMES.provision,
)
provisioning_task = objects.TaskCollection.order_by(
provisioning_task, 'id').first()
if not deployment_task and not provisioning_task:
db().rollback()
raise errors.DeploymentNotRunning(
u"Nothing to stop - deployment is "
u"not running on environment '{0}'".format(self.cluster.id))
# Updating action logs for deploy task
deploy_task = objects.TaskCollection.filter_by(
None, cluster_id=self.cluster.id, name=consts.TASK_NAMES.deploy)
deploy_task = objects.TaskCollection.order_by(deploy_task,
'id').first()
if deploy_task:
TaskHelper.set_ready_if_not_finished(deploy_task)
task = Task(name=consts.TASK_NAMES.stop_deployment,
cluster=self.cluster)
db().add(task)
db().commit()
self._call_silently(task,
tasks.StopDeploymentTask,
deploy_task=deployment_task,
provision_task=provisioning_task)
return task
