def test_error_zabbix_request(self, mock_urlopen):
urlopen_value = Mock()
urlopen_value.read.return_value = json.dumps({
'jsonrpc': '2.0',
'id': '1',
'error': {
'message': "Error connecting to database]",
'code': 1}})
mock_urlopen.return_value = urlopen_value
with self.assertRaises(errors.ZabbixRequestError):
ZabbixManager._make_zabbix_request(
'fake_url',
'user.authenticate',
{'password': '******', 'user': '******'},
auth=None),
def get_zabbix_url(cls, cluster):
zabbix_node = ZabbixManager.get_zabbix_node(cluster)
if zabbix_node is None:
return None
ip_cidr = cls.get_node_network_by_netname(zabbix_node, 'public')['ip']
ip = ip_cidr.split('/')[0]
return 'http://{0}/zabbix'.format(ip)
def get_zabbix_url(cls, cluster):
zabbix_node = ZabbixManager.get_zabbix_node(cluster)
if zabbix_node is None:
return None
ip_cidr = cls.get_node_network_by_netname(
zabbix_node, 'public'
)['ip']
ip = ip_cidr.split('/')[0]
return 'http://{0}/zabbix'.format(ip)
def test_error_zabbix_request(self, mock_urlopen):
urlopen_value = Mock()
urlopen_value.read.return_value = json.dumps({
'jsonrpc': '2.0',
'id': '1',
'error': {
'message': "Error connecting to database]",
'code': 1
}
})
mock_urlopen.return_value = urlopen_value
with self.assertRaises(errors.ZabbixRequestError):
ZabbixManager._make_zabbix_request('fake_url',
'user.authenticate', {
'password': '******',
'user': '******'
},
auth=None),
def execute(cls, task, respond_to='remove_nodes_resp'):
logger.debug("DeletionTask.execute(task=%s)" % task.uuid)
task_uuid = task.uuid
logger.debug("Nodes deletion task is running")
nodes_to_delete = []
nodes_to_restore = []
USE_FAKE = settings.FAKE_TASKS or settings.FAKE_TASKS_AMQP
# no need to call astute if there are no nodes in cluster
if respond_to == 'remove_cluster_resp' and \
not list(task.cluster.nodes):
rcvr = rpc.receiver.NailgunReceiver()
rcvr.remove_cluster_resp(
task_uuid=task_uuid,
status='ready',
progress=100
)
return
for node in task.cluster.nodes:
if node.pending_deletion:
nodes_to_delete.append({
'id': node.id,
'uid': node.id,
'roles': node.roles,
'slave_name': objects.Node.make_slave_name(node)
})
if USE_FAKE:
# only fake tasks
new_node = {}
keep_attrs = (
'id',
'cluster_id',
'roles',
'pending_deletion',
'pending_addition'
)
for prop in object_mapper(node).iterate_properties:
if isinstance(
prop, ColumnProperty
) and prop.key not in keep_attrs:
new_node[prop.key] = getattr(node, prop.key)
nodes_to_restore.append(new_node)
# /only fake tasks
# check if there's a zabbix server in an environment
# and if there is, remove hosts
if ZabbixManager.get_zabbix_node(task.cluster):
zabbix_credentials = ZabbixManager.get_zabbix_credentials(
task.cluster
)
logger.debug("Removing nodes %s from zabbix" % (nodes_to_delete))
try:
ZabbixManager.remove_from_zabbix(
zabbix_credentials, nodes_to_delete
)
except (errors.CannotMakeZabbixRequest,
errors.ZabbixRequestError) as e:
logger.warning("%s, skipping removing nodes from Zabbix", e)
# this variable is used to iterate over it
# and be able to delete node from nodes_to_delete safely
nodes_to_delete_constant = list(nodes_to_delete)
# locking nodes
nodes_ids = [node['id'] for node in nodes_to_delete_constant]
nodes_db = objects.NodeCollection.filter_by_list(
None,
'id',
nodes_ids,
order_by='id'
)
objects.NodeCollection.lock_for_update(nodes_db).all()
for node in nodes_to_delete_constant:
node_db = objects.Node.get_by_uid(node['id'], lock_for_update=True)
slave_name = objects.Node.make_slave_name(node_db)
logger.debug("Removing node from database and pending it "
"to clean its MBR: %s", slave_name)
if node_db.status == 'discover':
logger.info(
"Node is not deployed yet,"
" can't clean MBR: %s", slave_name)
db().delete(node_db)
db().flush()
nodes_to_delete.remove(node)
db().commit()
msg_delete = make_astute_message(
task,
'remove_nodes',
respond_to,
{
'nodes': nodes_to_delete,
'engine': {
'url': settings.COBBLER_URL,
'username': settings.COBBLER_USER,
'password': settings.COBBLER_PASSWORD,
'master_ip': settings.MASTER_IP,
}
}
)
db().flush()
# only fake tasks
if USE_FAKE and nodes_to_restore:
msg_delete['args']['nodes_to_restore'] = nodes_to_restore
# /only fake tasks
logger.debug("Calling rpc remove_nodes method")
rpc.cast('naily', msg_delete)
def execute(cls, task, nodes=None, respond_to='remove_nodes_resp'):
logger.debug("DeletionTask.execute(task=%s, nodes=%s)",
task.uuid, nodes)
task_uuid = task.uuid
logger.debug("Nodes deletion task is running")
# TODO(ikalnitsky): remove this, let the flow always go through Astute
# No need to call Astute if no nodes are specified
if task.name == consts.TASK_NAMES.cluster_deletion and \
not (nodes and nodes['nodes_to_delete']):
logger.debug("No nodes specified, exiting")
rcvr = rpc.receiver.NailgunReceiver()
rcvr.remove_cluster_resp(
task_uuid=task_uuid,
status=consts.TASK_STATUSES.ready,
progress=100
)
return
nodes_to_delete = nodes['nodes_to_delete']
nodes_to_restore = nodes['nodes_to_restore']
# check if there's a Zabbix server in an environment
# and if there is, remove hosts
if (task.name != consts.TASK_NAMES.cluster_deletion
and ZabbixManager.get_zabbix_node(task.cluster)):
zabbix_credentials = ZabbixManager.get_zabbix_credentials(
task.cluster
)
logger.debug("Removing nodes %s from zabbix", nodes_to_delete)
try:
ZabbixManager.remove_from_zabbix(
zabbix_credentials, nodes_to_delete
)
except (errors.CannotMakeZabbixRequest,
errors.ZabbixRequestError) as e:
logger.warning("%s, skipping removing nodes from Zabbix", e)
nodes_to_delete = cls.remove_undeployed_nodes_from_db(nodes_to_delete)
logger.debug(
"Removing nodes from database and pending them to clean their "
"MBR: %s",
', '.join(node['slave_name'] for node in nodes_to_delete)
)
msg_delete = make_astute_message(
task,
'remove_nodes',
respond_to,
{
'nodes': nodes_to_delete,
'engine': {
'url': settings.COBBLER_URL,
'username': settings.COBBLER_USER,
'password': settings.COBBLER_PASSWORD,
'master_ip': settings.MASTER_IP,
}
}
)
db().flush()
# only fake tasks
if cls.use_fake() and nodes_to_restore:
msg_delete['args']['nodes_to_restore'] = nodes_to_restore
# /only fake tasks
logger.debug("Calling rpc remove_nodes method with nodes %s",
nodes_to_delete)
rpc.cast('naily', msg_delete)
def execute(cls,
task,
nodes=None,
respond_to='remove_nodes_resp',
check_ceph=False):
"""Call remote Astute method to remove nodes from a cluster
:param task: Task object
:param nodes: List of nodes to delete
:param respond_to: RPC method which receives data from remote method
:param check_ceph: Boolean flag to tell Astute to run (or not run)
checks to prevent deletion of OSD nodes. If True this task will
fail if a node to be deleted has Ceph data on it. This flag must
be False if deleting all nodes.
"""
logger.debug("DeletionTask.execute(task=%s, nodes=%s)", task.uuid,
nodes)
task_uuid = task.uuid
logger.debug("Nodes deletion task is running")
# TODO(ikalnitsky): remove this, let the flow always go through Astute
# No need to call Astute if no nodes are specified
if task.name == consts.TASK_NAMES.cluster_deletion and \
not (nodes and nodes['nodes_to_delete']):
logger.debug("No nodes specified, exiting")
rcvr = rpc.receiver.NailgunReceiver()
rcvr.remove_cluster_resp(task_uuid=task_uuid,
status=consts.TASK_STATUSES.ready,
progress=100)
return
nodes_to_delete = nodes['nodes_to_delete']
nodes_to_restore = nodes['nodes_to_restore']
# check if there's a Zabbix server in an environment
# and if there is, remove hosts
if (task.name != consts.TASK_NAMES.cluster_deletion
and ZabbixManager.get_zabbix_node(task.cluster)):
zabbix_credentials = ZabbixManager.get_zabbix_credentials(
task.cluster)
logger.debug("Removing nodes %s from zabbix", nodes_to_delete)
try:
ZabbixManager.remove_from_zabbix(zabbix_credentials,
nodes_to_delete)
except (errors.CannotMakeZabbixRequest,
errors.ZabbixRequestError) as e:
logger.warning("%s, skipping removing nodes from Zabbix", e)
nodes_to_delete = cls.remove_undeployed_nodes_from_db(nodes_to_delete)
logger.debug(
"Removing nodes from database and pending them to clean their "
"MBR: %s",
', '.join(node['slave_name'] for node in nodes_to_delete))
msg_delete = make_astute_message(
task, 'remove_nodes', respond_to, {
'nodes': nodes_to_delete,
'check_ceph': check_ceph,
'engine': {
'url': settings.COBBLER_URL,
'username': settings.COBBLER_USER,
'password': settings.COBBLER_PASSWORD,
'master_ip': settings.MASTER_IP,
}
})
db().flush()
# only fake tasks
if cls.use_fake() and nodes_to_restore:
msg_delete['args']['nodes_to_restore'] = nodes_to_restore
# /only fake tasks
logger.debug("Calling rpc remove_nodes method with nodes %s",
nodes_to_delete)
rpc.cast('naily', msg_delete)
def execute(cls, task, respond_to='remove_nodes_resp'):
logger.debug("DeletionTask.execute(task=%s)" % task.uuid)
task_uuid = task.uuid
logger.debug("Nodes deletion task is running")
nodes_to_delete = []
nodes_to_restore = []
USE_FAKE = settings.FAKE_TASKS or settings.FAKE_TASKS_AMQP
# no need to call astute if there are no nodes in cluster
if respond_to == 'remove_cluster_resp' and \
not list(task.cluster.nodes):
rcvr = rpc.receiver.NailgunReceiver()
rcvr.remove_cluster_resp(task_uuid=task_uuid,
status='ready',
progress=100)
return
for node in task.cluster.nodes:
if node.pending_deletion:
nodes_to_delete.append({
'id':
node.id,
'uid':
node.id,
'roles':
node.roles,
'slave_name':
objects.Node.make_slave_name(node)
})
if USE_FAKE:
# only fake tasks
new_node = {}
reset_attrs = (
'id',
'cluster_id',
'roles',
'pending_deletion',
'pending_addition',
'group_id',
)
for prop in object_mapper(node).iterate_properties:
if isinstance(prop, ColumnProperty
) and prop.key not in reset_attrs:
new_node[prop.key] = getattr(node, prop.key)
nodes_to_restore.append(new_node)
# /only fake tasks
# check if there's a zabbix server in an environment
# and if there is, remove hosts
if ZabbixManager.get_zabbix_node(task.cluster):
zabbix_credentials = ZabbixManager.get_zabbix_credentials(
task.cluster)
logger.debug("Removing nodes %s from zabbix" % (nodes_to_delete))
try:
ZabbixManager.remove_from_zabbix(zabbix_credentials,
nodes_to_delete)
except (errors.CannotMakeZabbixRequest,
errors.ZabbixRequestError) as e:
logger.warning("%s, skipping removing nodes from Zabbix", e)
# this variable is used to iterate over it
# and be able to delete node from nodes_to_delete safely
nodes_to_delete_constant = list(nodes_to_delete)
# locking nodes
nodes_ids = [node['id'] for node in nodes_to_delete_constant]
nodes_db = objects.NodeCollection.filter_by_list(None,
'id',
nodes_ids,
order_by='id')
objects.NodeCollection.lock_for_update(nodes_db).all()
for node in nodes_to_delete_constant:
node_db = objects.Node.get_by_uid(node['id'], lock_for_update=True)
slave_name = objects.Node.make_slave_name(node_db)
logger.debug(
"Removing node from database and pending it "
"to clean its MBR: %s", slave_name)
if node_db.status == 'discover':
logger.info("Node is not deployed yet,"
" can't clean MBR: %s", slave_name)
db().delete(node_db)
db().flush()
nodes_to_delete.remove(node)
db().commit()
msg_delete = make_astute_message(
task, 'remove_nodes', respond_to, {
'nodes': nodes_to_delete,
'engine': {
'url': settings.COBBLER_URL,
'username': settings.COBBLER_USER,
'password': settings.COBBLER_PASSWORD,
'master_ip': settings.MASTER_IP,
}
})
db().flush()
# only fake tasks
if USE_FAKE and nodes_to_restore:
msg_delete['args']['nodes_to_restore'] = nodes_to_restore
# /only fake tasks
logger.debug("Calling rpc remove_nodes method")
rpc.cast('naily', msg_delete)