def remove_cluster_resp(cls, **kwargs):
logger.info("RPC method remove_cluster_resp received: %s" %
json.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
cls.remove_nodes_resp(**kwargs)
task = TaskHelper.get_task_by_uuid(task_uuid)
cluster = task.cluster
if task.status in ('ready', ):
logger.debug("Removing environment itself")
cluster_name = cluster.name
ips = db().query(IPAddr).filter(
IPAddr.network.in_([n.id for n in cluster.network_groups]))
map(db().delete, ips)
db().flush()
db().delete(cluster)
db().flush()
notifier.notify(
"done", u"Environment '%s' and all its nodes are deleted" %
(cluster_name))
elif task.status in ('error', ):
cluster.status = 'error'
db().add(cluster)
db().flush()
if not task.message:
task.message = "Failed to delete nodes:\n{0}".format(
cls._generate_error_message(task,
error_types=('deletion', )))
notifier.notify("error", task.message, cluster.id)
def check_redhat_credentials_resp(cls, **kwargs):
logger.info("RPC method check_redhat_credentials_resp received: %s" %
json.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
error_msg = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = TaskHelper.get_task_by_uuid(task_uuid)
release_info = task.cache['args']['release_info']
release_id = release_info['release_id']
release = db().query(Release).get(release_id)
if not release:
logger.error(
"download_release_resp: Release"
" with ID %s not found", release_id)
return
if error_msg:
status = 'error'
cls._update_release_state(release_id, 'error')
# TODO(NAME): remove this ugly checks
if 'Unknown error' in error_msg:
error_msg = 'Failed to check Red Hat ' \
'credentials'
if error_msg != 'Task aborted':
notifier.notify('error', error_msg)
result = {"release_info": {"release_id": release_id}}
TaskHelper.update_task_status(task_uuid, status, progress, error_msg,
result)
def download_release_resp(cls, **kwargs):
logger.info("RPC method download_release_resp received: %s" %
json.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
error_msg = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = TaskHelper.get_task_by_uuid(task_uuid)
release_info = task.cache['args']['release_info']
release_id = release_info['release_id']
release = db().query(Release).get(release_id)
if not release:
logger.error(
"download_release_resp: Release"
" with ID %s not found", release_id)
return
if error_msg:
status = 'error'
error_msg = "{0} download and preparation " \
"has failed.".format(release.name)
cls._download_release_error(release_id, error_msg)
elif progress == 100 and status == 'ready':
cls._download_release_completed(release_id)
result = {"release_info": {"release_id": release_id}}
TaskHelper.update_task_status(task_uuid, status, progress, error_msg,
result)
def stop_deployment_resp(cls, **kwargs):
logger.info("RPC method stop_deployment_resp received: %s" %
json.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes', [])
ia_nodes = kwargs.get('inaccessible_nodes', [])
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = TaskHelper.get_task_by_uuid(task_uuid)
stop_tasks = db().query(Task).filter_by(
cluster_id=task.cluster_id, ).filter(
Task.name.in_(["deploy", "deployment", "provision"])).all()
if not stop_tasks:
logger.warning(
"stop_deployment_resp: deployment tasks \
not found for environment '%s'!", task.cluster_id)
if status == "ready":
task.cluster.status = "stopped"
if stop_tasks:
map(db().delete, stop_tasks)
db().commit()
update_nodes = db().query(Node).filter(
Node.id.in_(
[n["uid"] for n in itertools.chain(nodes, ia_nodes)]),
Node.cluster_id == task.cluster_id).yield_per(100)
update_nodes.update(
{
"online": False,
"status": "discover",
"pending_addition": True
},
synchronize_session='fetch')
for n in update_nodes:
n.roles, n.pending_roles = n.pending_roles, n.roles
db().commit()
if ia_nodes:
cls._notify_inaccessible(task.cluster_id,
[n["uid"] for n in ia_nodes],
u"deployment stopping")
message = (u"Deployment of environment '{0}' "
u"was successfully stopped".format(task.cluster.name
or task.cluster_id))
notifier.notify("done", message, task.cluster_id)
TaskHelper.update_task_status(task_uuid, status, progress, message)
def reset_environment_resp(cls, **kwargs):
logger.info("RPC method reset_environment_resp received: %s",
json.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes', [])
ia_nodes = kwargs.get('inaccessible_nodes', [])
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = TaskHelper.get_task_by_uuid(task_uuid)
if status == "ready":
# restoring pending changes
task.cluster.status = "new"
objects.Cluster.add_pending_changes(task.cluster, "attributes")
objects.Cluster.add_pending_changes(task.cluster, "networks")
for node in task.cluster.nodes:
objects.Cluster.add_pending_changes(task.cluster,
"disks",
node_id=node.id)
update_nodes = db().query(Node).filter(
Node.id.in_(
[n["uid"] for n in itertools.chain(nodes, ia_nodes)]),
Node.cluster_id == task.cluster_id).yield_per(100)
update_nodes.update(
{
"online": False,
"status": "discover",
"pending_addition": True,
"pending_deletion": False,
},
synchronize_session='fetch')
for n in update_nodes:
n.roles, n.pending_roles = n.pending_roles, n.roles
db().commit()
if ia_nodes:
cls._notify_inaccessible(task.cluster_id,
[n["uid"] for n in ia_nodes],
u"environment resetting")
message = (u"Environment '{0}' "
u"was successfully reset".format(task.cluster.name
or task.cluster_id))
notifier.notify("done", message, task.cluster_id)
TaskHelper.update_task_status(task.uuid, status, progress, message)
def provision_resp(cls, **kwargs):
logger.info(
"RPC method provision_resp received: %s" %
json.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
nodes = kwargs.get('nodes', [])
task = TaskHelper.get_task_by_uuid(task_uuid)
for node in nodes:
uid = node.get('uid')
node_db = db().query(Node).get(uid)
if not node_db:
logger.warn('Node with uid "{0}" not found'.format(uid))
continue
if node.get('status') == 'error':
node_db.status = 'error'
node_db.progress = 100
node_db.error_type = 'provision'
node_db.error_msg = node.get('error_msg', 'Unknown error')
else:
node_db.status = node.get('status')
node_db.progress = node.get('progress')
db().commit()
task = TaskHelper.get_task_by_uuid(task_uuid)
if nodes and not progress:
progress = TaskHelper.recalculate_provisioning_task_progress(task)
TaskHelper.update_task_status(task.uuid, status, progress, message)
def provision_resp(cls, **kwargs):
logger.info("RPC method provision_resp received: %s" %
json.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
nodes = kwargs.get('nodes', [])
task = TaskHelper.get_task_by_uuid(task_uuid)
for node in nodes:
uid = node.get('uid')
node_db = db().query(Node).get(uid)
if not node_db:
logger.warn('Node with uid "{0}" not found'.format(uid))
continue
if node.get('status') == 'error':
node_db.status = 'error'
node_db.progress = 100
node_db.error_type = 'provision'
node_db.error_msg = node.get('error_msg', 'Unknown error')
else:
node_db.status = node.get('status')
node_db.progress = node.get('progress')
db().commit()
task = TaskHelper.get_task_by_uuid(task_uuid)
if nodes and not progress:
progress = TaskHelper.recalculate_provisioning_task_progress(task)
TaskHelper.update_task_status(task.uuid, status, progress, message)
def remove_cluster_resp(cls, **kwargs):
logger.info(
"RPC method remove_cluster_resp received: %s" %
json.dumps(kwargs)
)
task_uuid = kwargs.get('task_uuid')
cls.remove_nodes_resp(**kwargs)
task = TaskHelper.get_task_by_uuid(task_uuid)
cluster = task.cluster
if task.status in ('ready',):
logger.debug("Removing environment itself")
cluster_name = cluster.name
ips = db().query(IPAddr).filter(
IPAddr.network.in_([n.id for n in cluster.network_groups])
)
map(db().delete, ips)
db().flush()
db().delete(cluster)
db().flush()
notifier.notify(
"done",
u"Environment '%s' and all its nodes are deleted" % (
cluster_name
)
)
elif task.status in ('error',):
cluster.status = 'error'
db().add(cluster)
db().flush()
if not task.message:
task.message = "Failed to delete nodes:\n{0}".format(
cls._generate_error_message(
task,
error_types=('deletion',)
)
)
notifier.notify(
"error",
task.message,
cluster.id
)
def redhat_check_licenses_resp(cls, **kwargs):
logger.info(
"RPC method redhat_check_licenses_resp received: %s" %
json.dumps(kwargs)
)
task_uuid = kwargs.get('task_uuid')
error_msg = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
notify = kwargs.get('msg')
task = TaskHelper.get_task_by_uuid(task_uuid)
release_info = task.cache['args']['release_info']
release_id = release_info['release_id']
release = db().query(Release).get(release_id)
if not release:
logger.error("download_release_resp: Release"
" with ID %s not found", release_id)
return
if error_msg:
status = 'error'
cls._update_release_state(release_id, 'error')
# TODO(NAME): remove this ugly checks
if 'Unknown error' in error_msg:
error_msg = 'Failed to check Red Hat licenses '
if error_msg != 'Task aborted':
notifier.notify('error', error_msg)
if notify:
notifier.notify('error', notify)
result = {
"release_info": {
"release_id": release_id
}
}
TaskHelper.update_task_status(
task_uuid,
status,
progress,
error_msg,
result
)
def download_release_resp(cls, **kwargs):
logger.info(
"RPC method download_release_resp received: %s" %
json.dumps(kwargs)
)
task_uuid = kwargs.get('task_uuid')
error_msg = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = TaskHelper.get_task_by_uuid(task_uuid)
release_info = task.cache['args']['release_info']
release_id = release_info['release_id']
release = db().query(Release).get(release_id)
if not release:
logger.error("download_release_resp: Release"
" with ID %s not found", release_id)
return
if error_msg:
status = 'error'
error_msg = "{0} download and preparation " \
"has failed.".format(release.name)
cls._download_release_error(
release_id,
error_msg
)
elif progress == 100 and status == 'ready':
cls._download_release_completed(release_id)
result = {
"release_info": {
"release_id": release_id
}
}
TaskHelper.update_task_status(
task_uuid,
status,
progress,
error_msg,
result
)
def verify_networks_resp(cls, **kwargs):
logger.info(
"RPC method verify_networks_resp received: %s" %
json.dumps(kwargs)
)
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes')
error_msg = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
# We simply check that each node received all vlans for cluster
task = TaskHelper.get_task_by_uuid(task_uuid)
result = []
# We expect that 'nodes' contains all nodes which we test.
# Situation when some nodes not answered must be processed
# in orchestrator early.
if nodes is None:
# If no nodes in kwargs then we update progress or status only.
pass
elif isinstance(nodes, list):
cached_nodes = task.cache['args']['nodes']
node_uids = [str(n['uid']) for n in nodes]
cached_node_uids = [str(n['uid']) for n in cached_nodes]
forgotten_uids = set(cached_node_uids) - set(node_uids)
if forgotten_uids:
absent_nodes = db().query(Node).filter(
Node.id.in_(forgotten_uids)
).all()
absent_node_names = []
for n in absent_nodes:
if n.name:
absent_node_names.append(n.name)
else:
absent_node_names.append('id: %s' % n.id)
if not error_msg:
error_msg = 'Node(s) {0} didn\'t return data.'.format(
', '.join(absent_node_names)
)
status = 'error'
else:
error_nodes = []
for node in nodes:
cached_nodes_filtered = filter(
lambda n: str(n['uid']) == str(node['uid']),
cached_nodes
)
if not cached_nodes_filtered:
logger.warning(
"verify_networks_resp: arguments contain node "
"data which is not in the task cache: %r",
node
)
continue
cached_node = cached_nodes_filtered[0]
for cached_network in cached_node['networks']:
received_networks_filtered = filter(
lambda n: n['iface'] == cached_network['iface'],
node.get('networks', [])
)
if received_networks_filtered:
received_network = received_networks_filtered[0]
absent_vlans = list(
set(cached_network['vlans']) -
set(received_network['vlans'])
)
else:
logger.warning(
"verify_networks_resp: arguments don't contain"
" data for interface: uid=%s iface=%s",
node['uid'], cached_network['iface']
)
absent_vlans = cached_network['vlans']
if absent_vlans:
data = {'uid': node['uid'],
'interface': cached_network['iface'],
'absent_vlans': absent_vlans}
node_db = db().query(Node).get(node['uid'])
if node_db:
data['name'] = node_db.name
db_nics = filter(
lambda i:
i.name == cached_network['iface'],
node_db.interfaces
)
if db_nics:
data['mac'] = db_nics[0].mac
else:
logger.warning(
"verify_networks_resp: can't find "
"interface %r for node %r in DB",
cached_network['iface'], node_db.id
)
data['mac'] = 'unknown'
else:
logger.warning(
"verify_networks_resp: can't find node "
"%r in DB",
node['uid']
)
error_nodes.append(data)
if error_nodes:
result = error_nodes
status = 'error'
else:
error_msg = (error_msg or
'verify_networks_resp: argument "nodes"'
' have incorrect type')
status = 'error'
logger.error(error_msg)
if status not in ('ready', 'error'):
TaskHelper.update_task_status(task_uuid, status, progress,
error_msg, result)
else:
TaskHelper.update_verify_networks(task_uuid, status, progress,
error_msg, result)
def reset_environment_resp(cls, **kwargs):
logger.info(
"RPC method reset_environment_resp received: %s",
json.dumps(kwargs)
)
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes', [])
ia_nodes = kwargs.get('inaccessible_nodes', [])
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = TaskHelper.get_task_by_uuid(task_uuid)
if status == "ready":
# restoring pending changes
task.cluster.status = "new"
objects.Cluster.add_pending_changes(task.cluster, "attributes")
objects.Cluster.add_pending_changes(task.cluster, "networks")
for node in task.cluster.nodes:
objects.Cluster.add_pending_changes(
task.cluster,
"disks",
node_id=node.id
)
update_nodes = db().query(Node).filter(
Node.id.in_([
n["uid"] for n in itertools.chain(
nodes,
ia_nodes
)
]),
Node.cluster_id == task.cluster_id
).yield_per(100)
update_nodes.update(
{
"online": False,
"status": "discover",
"pending_addition": True,
"pending_deletion": False,
},
synchronize_session='fetch'
)
for n in update_nodes:
n.roles, n.pending_roles = n.pending_roles, n.roles
db().commit()
if ia_nodes:
cls._notify_inaccessible(
task.cluster_id,
[n["uid"] for n in ia_nodes],
u"environment resetting"
)
message = (
u"Environment '{0}' "
u"was successfully reset".format(
task.cluster.name or task.cluster_id
)
)
notifier.notify(
"done",
message,
task.cluster_id
)
TaskHelper.update_task_status(
task.uuid,
status,
progress,
message
)
def stop_deployment_resp(cls, **kwargs):
logger.info(
"RPC method stop_deployment_resp received: %s" %
json.dumps(kwargs)
)
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes', [])
ia_nodes = kwargs.get('inaccessible_nodes', [])
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = TaskHelper.get_task_by_uuid(task_uuid)
stop_tasks = db().query(Task).filter_by(
cluster_id=task.cluster_id,
).filter(
Task.name.in_(["deploy", "deployment", "provision"])
).all()
if not stop_tasks:
logger.warning("stop_deployment_resp: deployment tasks \
not found for environment '%s'!", task.cluster_id)
if status == "ready":
task.cluster.status = "stopped"
if stop_tasks:
map(db().delete, stop_tasks)
db().commit()
update_nodes = db().query(Node).filter(
Node.id.in_([
n["uid"] for n in itertools.chain(
nodes,
ia_nodes
)
]),
Node.cluster_id == task.cluster_id
).yield_per(100)
update_nodes.update(
{
"online": False,
"status": "discover",
"pending_addition": True
},
synchronize_session='fetch'
)
for n in update_nodes:
n.roles, n.pending_roles = n.pending_roles, n.roles
db().commit()
if ia_nodes:
cls._notify_inaccessible(
task.cluster_id,
[n["uid"] for n in ia_nodes],
u"deployment stopping"
)
message = (
u"Deployment of environment '{0}' "
u"was successfully stopped".format(
task.cluster.name or task.cluster_id
)
)
notifier.notify(
"done",
message,
task.cluster_id
)
TaskHelper.update_task_status(
task_uuid,
status,
progress,
message
)
def test_get_task_by_uuid_returns_task(self):
task = Task(name='deploy')
self.db.add(task)
self.db.commit()
task_by_uuid = TaskHelper.get_task_by_uuid(task.uuid)
self.assertEquals(task.uuid, task_by_uuid.uuid)
def verify_networks_resp(cls, **kwargs):
logger.info("RPC method verify_networks_resp received: %s" %
json.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes')
error_msg = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
# We simply check that each node received all vlans for cluster
task = TaskHelper.get_task_by_uuid(task_uuid)
result = []
# We expect that 'nodes' contains all nodes which we test.
# Situation when some nodes not answered must be processed
# in orchestrator early.
if nodes is None:
# If no nodes in kwargs then we update progress or status only.
pass
elif isinstance(nodes, list):
cached_nodes = task.cache['args']['nodes']
node_uids = [str(n['uid']) for n in nodes]
cached_node_uids = [str(n['uid']) for n in cached_nodes]
forgotten_uids = set(cached_node_uids) - set(node_uids)
if forgotten_uids:
absent_nodes = db().query(Node).filter(
Node.id.in_(forgotten_uids)).all()
absent_node_names = []
for n in absent_nodes:
if n.name:
absent_node_names.append(n.name)
else:
absent_node_names.append('id: %s' % n.id)
if not error_msg:
error_msg = 'Node(s) {0} didn\'t return data.'.format(
', '.join(absent_node_names))
status = 'error'
else:
error_nodes = []
for node in nodes:
cached_nodes_filtered = filter(
lambda n: str(n['uid']) == str(node['uid']),
cached_nodes)
if not cached_nodes_filtered:
logger.warning(
"verify_networks_resp: arguments contain node "
"data which is not in the task cache: %r", node)
continue
cached_node = cached_nodes_filtered[0]
for cached_network in cached_node['networks']:
received_networks_filtered = filter(
lambda n: n['iface'] == cached_network['iface'],
node.get('networks', []))
if received_networks_filtered:
received_network = received_networks_filtered[0]
absent_vlans = list(
set(cached_network['vlans']) -
set(received_network['vlans']))
else:
logger.warning(
"verify_networks_resp: arguments don't contain"
" data for interface: uid=%s iface=%s",
node['uid'], cached_network['iface'])
absent_vlans = cached_network['vlans']
if absent_vlans:
data = {
'uid': node['uid'],
'interface': cached_network['iface'],
'absent_vlans': absent_vlans
}
node_db = db().query(Node).get(node['uid'])
if node_db:
data['name'] = node_db.name
db_nics = filter(
lambda i: i.name == cached_network[
'iface'], node_db.interfaces)
if db_nics:
data['mac'] = db_nics[0].mac
else:
logger.warning(
"verify_networks_resp: can't find "
"interface %r for node %r in DB",
cached_network['iface'], node_db.id)
data['mac'] = 'unknown'
else:
logger.warning(
"verify_networks_resp: can't find node "
"%r in DB", node['uid'])
error_nodes.append(data)
if error_nodes:
result = error_nodes
status = 'error'
else:
error_msg = (error_msg or 'verify_networks_resp: argument "nodes"'
' have incorrect type')
status = 'error'
logger.error(error_msg)
if status not in ('ready', 'error'):
TaskHelper.update_task_status(task_uuid, status, progress,
error_msg, result)
else:
TaskHelper.update_verify_networks(task_uuid, status, progress,
error_msg, result)
def test_get_task_by_uuid_returns_task(self):
task = Task(name='deploy')
self.db.add(task)
self.db.commit()
task_by_uuid = TaskHelper.get_task_by_uuid(task.uuid)
self.assertEquals(task.uuid, task_by_uuid.uuid)
def deploy_resp(cls, **kwargs):
logger.info(
"RPC method deploy_resp received: %s" %
json.dumps(kwargs)
)
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes') or []
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = TaskHelper.get_task_by_uuid(task_uuid)
if not task:
# No task found - nothing to do here, returning
logger.warning(
u"No task with uuid '{0}'' found - nothing changed".format(
task_uuid
)
)
return
if not status:
status = task.status
# First of all, let's update nodes in database
for node in nodes:
node_db = db().query(Node).get(node['uid'])
if not node_db:
logger.warning(
u"No node found with uid '{0}' - nothing changed".format(
node['uid']
)
)
continue
update_fields = (
'error_msg',
'error_type',
'status',
'progress',
'online'
)
for param in update_fields:
if param in node:
logger.debug(
u"Updating node {0} - set {1} to {2}".format(
node['uid'],
param,
node[param]
)
)
setattr(node_db, param, node[param])
if param == 'progress' and node.get('status') == 'error' \
or node.get('online') is False:
# If failure occurred with node
# it's progress should be 100
node_db.progress = 100
# Setting node error_msg for offline nodes
if node.get('online') is False \
and not node_db.error_msg:
node_db.error_msg = u"Node is offline"
# Notification on particular node failure
notifier.notify(
"error",
u"Failed to deploy node '{0}': {1}".format(
node_db.name,
node_db.error_msg or "Unknown error"
),
cluster_id=task.cluster_id,
node_id=node['uid'],
task_uuid=task_uuid
)
db().add(node_db)
db().commit()
# We should calculate task progress by nodes info
task = TaskHelper.get_task_by_uuid(task_uuid)
if nodes and not progress:
progress = TaskHelper.recalculate_deployment_task_progress(task)
# Let's check the whole task status
if status in ('error',):
cls._error_action(task, status, progress, message)
elif status in ('ready',):
cls._success_action(task, status, progress)
else:
TaskHelper.update_task_status(task.uuid, status, progress, message)
