def execute(self):
logger.info(u"Trying to start deployment at cluster '{0}'".format(
self.cluster.name or self.cluster.id, ))
current_tasks = orm().query(Task).filter_by(cluster_id=self.cluster.id,
name="deploy")
for task in current_tasks:
if task.status == "running":
raise errors.DeploymentAlreadyStarted()
elif task.status in ("ready", "error"):
for subtask in task.subtasks:
orm().delete(subtask)
orm().delete(task)
orm().commit()
nodes_to_delete = TaskHelper.nodes_to_delete(self.cluster)
nodes_to_deploy = TaskHelper.nodes_to_deploy(self.cluster)
if not any([nodes_to_deploy, nodes_to_delete]):
raise errors.WrongNodeStatus("No changes to deploy")
self.cluster.status = 'deployment'
orm().add(self.cluster)
orm().commit()
supertask = Task(name="deploy", cluster=self.cluster)
orm().add(supertask)
orm().commit()
task_deletion, task_provision, task_deployment = None, None, None
if nodes_to_delete:
task_deletion = supertask.create_subtask("node_deletion")
self._call_silently(task_deletion, tasks.DeletionTask)
if nodes_to_deploy:
TaskHelper.update_slave_nodes_fqdn(nodes_to_deploy)
task_provision = supertask.create_subtask("provision")
# we assume here that task_provision just adds system to
# cobbler and reboots systems, so it has extreamly small weight
task_provision.weight = 0.05
provision_message = self._call_silently(task_provision,
tasks.ProvisionTask,
method_name='message')
task_provision.cache = provision_message
orm().add(task_provision)
orm().commit()
task_deployment = supertask.create_subtask("deployment")
deployment_message = self._call_silently(task_deployment,
tasks.DeploymentTask,
method_name='message')
task_deployment.cache = deployment_message
orm().add(task_deployment)
orm().commit()
rpc.cast('naily', [provision_message, deployment_message])
logger.debug(u"Deployment: task to deploy cluster '{0}' is {1}".format(
self.cluster.name or self.cluster.id, supertask.uuid))
return supertask
def test_verify_networks_resp_empty_nodes_default_error(self):
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False},
{"api": False}
]
)
cluster_db = self.env.clusters[0]
node1, node2 = self.env.nodes
task = Task(
name="super",
cluster_id=cluster_db.id
)
task.cache = {
"args": {
'nodes': []
}
}
self.db.add(task)
self.db.commit()
kwargs = {'task_uuid': task.uuid,
'status': 'ready',
'nodes': []}
self.receiver.verify_networks_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.status, "error")
error_msg = 'At least two nodes are required to be in ' \
'the environment for network verification.'
self.assertEqual(task.message, error_msg)
def test_verify_networks_resp_empty_nodes_default_error(self):
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False},
{"api": False}
]
)
cluster_db = self.env.clusters[0]
node1, node2 = self.env.nodes
nets_sent = [{'iface': 'eth0', 'vlans': range(100, 105)}]
task = Task(
name="super",
cluster_id=cluster_db.id
)
task.cache = {
"args": {
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': nets_sent}]
}
}
self.db.add(task)
self.db.commit()
kwargs = {'task_uuid': task.uuid,
'status': 'ready',
'nodes': []}
self.receiver.verify_networks_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.status, "error")
error_msg = 'Please add more nodes to the environment ' \
'before performing network verification.'
self.assertEqual(task.message, error_msg)
def test_verify_networks_resp_empty_nodes_default_error(self):
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False},
{"api": False}
]
)
cluster_db = self.env.clusters[0]
node1, node2 = self.env.nodes
task = Task(
name="super",
cluster_id=cluster_db.id
)
task.cache = {
"args": {
'nodes': []
}
}
self.db.add(task)
self.db.commit()
kwargs = {'task_uuid': task.uuid,
'status': 'ready',
'nodes': []}
self.receiver.verify_networks_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.status, "error")
error_msg = 'At least two nodes are required to be in ' \
'the environment for network verification.'
self.assertEqual(task.message, error_msg)
def execute(self, nets, vlan_ids):
task = Task(
name="check_networks",
cluster=self.cluster
)
orm().add(task)
orm().commit()
self._call_silently(
task,
tasks.CheckNetworksTask,
nets
)
orm().refresh(task)
if task.status != 'error':
# this one is connected with UI issues - we need to
# separate if error happened inside nailgun or somewhere
# in the orchestrator, and UI does it by task name.
task.name = "verify_networks"
orm().add(task)
orm().commit()
self._call_silently(
task,
tasks.VerifyNetworksTask,
vlan_ids
)
return task
def test_verify_networks_resp_empty_nodes_custom_error(self):
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False},
{"api": False}
]
)
cluster_db = self.env.clusters[0]
node1, node2 = self.env.nodes
nets_sent = [{'iface': 'eth0', 'vlans': range(100, 105)}]
task = Task(
name="super",
cluster_id=cluster_db.id
)
task.cache = {
"args": {
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': nets_sent}]
}
}
self.db.add(task)
self.db.commit()
error_msg = 'Custom error message.'
kwargs = {'task_uuid': task.uuid,
'status': 'ready',
'nodes': [],
'error': error_msg}
self.receiver.verify_networks_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.status, "error")
self.assertEqual(task.message, error_msg)
def test_verify_networks_resp_partially_without_vlans(self):
"""Verify that network verification partially without vlans passes
"""
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False},
{"api": False}
]
)
cluster_db = self.env.clusters[0]
node1, node2 = self.env.nodes
nets_sent = [{'iface': 'eth0', 'vlans': [0]},
{'iface': 'eth1', 'vlans': range(100, 104)}]
task = Task(
name="super",
cluster_id=cluster_db.id
)
task.cache = {
"args": {
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': nets_sent}]
}
}
self.db.add(task)
self.db.commit()
kwargs = {'task_uuid': task.uuid,
'status': 'ready',
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': nets_sent}]}
self.receiver.verify_networks_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.status, "ready")
def test_verify_networks_resp(self):
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False},
{"api": False}
]
)
cluster_db = self.env.clusters[0]
node1, node2 = self.env.nodes
nets = [{'iface': 'eth0', 'vlans': range(100, 105)}]
task = Task(
name="verify_networks",
cluster_id=cluster_db.id
)
task.cache = {
"args": {
"nodes": [{'uid': node1.id, 'networks': nets},
{'uid': node2.id, 'networks': nets}]
}
}
self.db.add(task)
self.db.commit()
kwargs = {'task_uuid': task.uuid,
'status': 'ready',
'nodes': [{'uid': node1.id, 'networks': nets},
{'uid': node2.id, 'networks': nets}]}
self.receiver.verify_networks_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.status, "ready")
self.assertEqual(task.message, None)
def execute(self, nodes_to_deployment):
TaskHelper.update_slave_nodes_fqdn(nodes_to_deployment)
logger.debug('Nodes to deploy: {0}'.format(
' '.join([n.fqdn for n in nodes_to_deployment])))
task_deployment = Task(name='deployment', cluster=self.cluster)
db().add(task_deployment)
db().commit()
deployment_message = self._call_silently(
task_deployment,
tasks.DeploymentTask,
nodes_to_deployment,
method_name='message')
db().refresh(task_deployment)
task_deployment.cache = deployment_message
for node in nodes_to_deployment:
node.status = 'deploying'
node.progress = 0
db().commit()
rpc.cast('naily', deployment_message)
return task_deployment
def test_verify_networks_resp_partially_without_vlans(self):
"""Verify that network verification partially without vlans passes
"""
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False},
{"api": False}
]
)
cluster_db = self.env.clusters[0]
node1, node2 = self.env.nodes
nets_sent = [{'iface': 'eth0', 'vlans': [0]},
{'iface': 'eth1', 'vlans': range(100, 104)}]
task = Task(
name="super",
cluster_id=cluster_db.id
)
task.cache = {
"args": {
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': nets_sent}]
}
}
self.db.add(task)
self.db.commit()
kwargs = {'task_uuid': task.uuid,
'status': 'ready',
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': nets_sent}]}
self.receiver.verify_networks_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.status, "ready")
def test_verify_networks_resp_without_vlans_only_erred(self):
"""Verify that network verification without vlans fails
when not all sended info received
"""
self.env.create(cluster_kwargs={},
nodes_kwargs=[{
"api": False
}, {
"api": False
}])
cluster_db = self.env.clusters[0]
node1, node2 = self.env.nodes
nets_sent = [{'iface': 'eth0', 'vlans': [0]}]
nets_resp = [{'iface': 'eth0', 'vlans': []}]
task = Task(name="super", cluster_id=cluster_db.id)
task.cache = {
"args": {
'nodes': [{
'uid': node1.id,
'networks': nets_sent
}, {
'uid': node2.id,
'networks': nets_sent
}]
}
}
self.db.add(task)
self.db.commit()
kwargs = {
'task_uuid':
task.uuid,
'status':
'ready',
'nodes': [{
'uid': node1.id,
'networks': nets_resp
}, {
'uid': node2.id,
'networks': nets_resp
}]
}
self.receiver.verify_networks_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.status, "error")
error_nodes = [{
'uid': node1.id,
'interface': 'eth0',
'name': node1.name,
'mac': node1.interfaces[0].mac,
'absent_vlans': nets_sent[0]['vlans']
}, {
'uid': node2.id,
'interface': 'eth0',
'name': node2.name,
'mac': node2.interfaces[0].mac,
'absent_vlans': nets_sent[0]['vlans']
}]
self.assertEqual(task.result, error_nodes)
def test_verify_networks_resp_incomplete_network_data_error(self):
# One node has single interface
meta = self.env.default_metadata()
mac = '02:07:43:78:4F:58'
self.env.set_interfaces_in_meta(
meta, [{'name': 'eth0', 'mac': mac}])
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False, 'name': 'node1'},
{"api": False, 'name': 'node2', 'meta': meta},
{"api": False, 'name': 'node3'}
]
)
cluster_db = self.env.clusters[0]
node1, node2, node3 = self.env.nodes
nets_sent = [{'iface': 'eth0', 'vlans': range(100, 105)},
{'iface': 'eth1', 'vlans': [106]},
{'iface': 'eth2', 'vlans': [107]}]
task = Task(
name="super",
cluster_id=cluster_db.id
)
task.cache = {
"args": {
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': nets_sent},
{'uid': node3.id, 'networks': nets_sent}]
}
}
self.db.add(task)
self.db.commit()
kwargs = {'task_uuid': task.uuid,
'status': 'ready',
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': []},
{'uid': node3.id, 'networks': nets_sent}]}
self.receiver.verify_networks_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.status, "error")
self.assertEqual(task.message, '')
error_nodes = [{'uid': node2.id, 'interface': 'eth0',
'name': node2.name, 'mac': node2.interfaces[0].mac,
'absent_vlans': nets_sent[0]['vlans']},
{'uid': node2.id, 'interface': 'eth1',
'name': node2.name, 'mac': 'unknown',
'absent_vlans': nets_sent[1]['vlans']},
{'uid': node2.id, 'interface': 'eth2',
'name': node2.name, 'mac': 'unknown',
'absent_vlans': nets_sent[2]['vlans']}
]
self.assertEqual(task.result, error_nodes)
def execute(self):
logger.debug("Creating redhat_setup task")
current_tasks = db().query(Task).filter_by(name="redhat_setup")
for task in current_tasks:
for subtask in task.subtasks:
db().delete(subtask)
db().delete(task)
db().commit()
supertask = Task(name="redhat_setup")
supertask.result = {
"release_info": {
"release_id": self.data["release_id"]
}
}
db().add(supertask)
db().commit()
subtasks_to_create = [
('redhat_check_credentials', tasks.RedHatCheckCredentialsTask,
0.01),
('redhat_check_licenses', tasks.RedHatCheckLicensesTask, 0.01),
('redhat_download_release', tasks.RedHatDownloadReleaseTask, 1)
]
messages = []
for task_name, task_class, weight in subtasks_to_create:
task = supertask.create_subtask(task_name)
task.weight = weight
db().add(task)
db().commit()
msg = self._call_silently(task,
task_class,
self.data,
method_name='message')
db().refresh(task)
if task.status == 'error':
TaskHelper.update_task_status(supertask.uuid,
status="error",
progress=100,
msg=task.message)
return supertask
task.cache = msg
db().add(task)
db().commit()
messages.append(msg)
db().refresh(supertask)
if supertask.status == 'error':
return supertask
rpc.cast('naily', messages)
return supertask
def test_verify_networks_resp_error(self):
self.env.create(cluster_kwargs={},
nodes_kwargs=[{
"api": False
}, {
"api": False
}])
cluster_db = self.env.clusters[0]
node1, node2 = self.env.nodes
nets_sent = [{'iface': 'eth0', 'vlans': range(100, 105)}]
nets_resp = [{'iface': 'eth0', 'vlans': range(100, 104)}]
task = Task(name="super", cluster_id=cluster_db.id)
task.cache = {
"args": {
'nodes': [{
'uid': node1.id,
'networks': nets_sent
}, {
'uid': node2.id,
'networks': nets_sent
}]
}
}
self.db.add(task)
self.db.commit()
kwargs = {
'task_uuid':
task.uuid,
'status':
'ready',
'nodes': [{
'uid': node1.id,
'networks': nets_resp
}, {
'uid': node2.id,
'networks': nets_resp
}]
}
self.receiver.verify_networks_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.status, "error")
error_nodes = []
for node in self.env.nodes:
error_nodes.append({
'uid': node.id,
'interface': 'eth0',
'name': node.name,
'absent_vlans': [104],
'mac': node.interfaces[0].mac
})
self.assertEqual(task.message, None)
self.assertEqual(task.result, error_nodes)
def test_verify_networks_resp_forgotten_node_error(self):
self.env.create(cluster_kwargs={},
nodes_kwargs=[{
"api": False,
'name': 'node1'
}, {
"api": False,
'name': 'node2'
}, {
"api": False,
'name': 'node3'
}])
cluster_db = self.env.clusters[0]
node1, node2, node3 = self.env.nodes
nets_sent = [{'iface': 'eth0', 'vlans': range(100, 105)}]
task = Task(name="super", cluster_id=cluster_db.id)
task.cache = {
"args": {
'nodes': [{
'uid': node1.id,
'networks': nets_sent
}, {
'uid': node2.id,
'networks': nets_sent
}, {
'uid': node3.id,
'networks': nets_sent
}]
}
}
self.db.add(task)
self.db.commit()
kwargs = {
'task_uuid':
task.uuid,
'status':
'ready',
'nodes': [{
'uid': node1.id,
'networks': nets_sent
}, {
'uid': node2.id,
'networks': nets_sent
}]
}
self.receiver.verify_networks_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.status, "error")
self.assertRegexpMatches(task.message, node3.name)
self.assertEqual(task.result, [])
def execute(self, nets, vlan_ids):
task = Task(name="check_networks", cluster=self.cluster)
db().add(task)
db().commit()
self._call_silently(task, tasks.CheckNetworksTask, nets)
db().refresh(task)
if task.status != 'error':
# this one is connected with UI issues - we need to
# separate if error happened inside nailgun or somewhere
# in the orchestrator, and UI does it by task name.
task.name = "verify_networks"
db().add(task)
db().commit()
self._call_silently(task, tasks.VerifyNetworksTask, vlan_ids)
return task
def test_redhat_account_validation_success(self):
with patch('nailgun.api.handlers.redhat.db', Mock()):
manager = 'nailgun.api.handlers.redhat.RedHatSetupTaskManager'
with patch(manager, Mock()) as mng:
task = Task()
task.id = 0
mng.return_value.execute.return_value = task
resp = self.app.post(
reverse('RedHatSetupHandler'),
json.dumps({'license_type': 'rhsm',
'username': '******',
'password': '******',
'release_id': self.release.id}),
headers=self.default_headers)
self.assertEquals(resp.status, 202)
def test_verify_networks_resp_error_with_removed_node(self):
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False},
{"api": False}
]
)
cluster_db = self.env.clusters[0]
node1, node2 = self.env.nodes
nets_sent = [{'iface': 'eth0', 'vlans': range(100, 105)}]
nets_resp = [{'iface': 'eth0', 'vlans': range(100, 104)}]
task = Task(
name="super",
cluster_id=cluster_db.id
)
task.cache = {
"args": {
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': nets_sent}]
}
}
self.db.add(task)
self.db.commit()
kwargs = {'task_uuid': task.uuid,
'status': 'ready',
'nodes': [{'uid': node1.id, 'networks': nets_resp},
{'uid': node2.id, 'networks': nets_resp}]}
self.db.delete(node2)
self.db.commit()
self.receiver.verify_networks_resp(**kwargs)
resp = self.app.get(
reverse('TaskHandler', kwargs={'task_id': task.id}),
headers=self.default_headers
)
self.assertEquals(resp.status, 200)
task = json.loads(resp.body)
self.assertEqual(task['status'], "error")
error_nodes = [{'uid': node1.id, 'interface': 'eth0',
'name': node1.name, 'absent_vlans': [104],
'mac': node1.mac},
{'uid': node2.id, 'interface': 'eth0',
'absent_vlans': [104]}]
self.assertEqual(task.get('message'), None)
self.assertEqual(task['result'], error_nodes)
def test_error_node_progress(self):
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False}
]
)
task = Task(
uuid=str(uuid.uuid4()),
name="super",
status="running"
)
self.db.add(task)
self.db.commit()
kwargs = {
'task_uuid': task.uuid,
'progress': 20,
'nodes': [
{
'uid': self.env.nodes[0].id,
'status': 'error',
'progress': 50
}
]
}
self.receiver.deploy_resp(**kwargs)
self.db.refresh(self.env.nodes[0])
self.assertEqual(self.env.nodes[0].progress, 100)
def test_remove_nodes_resp(self):
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False},
{"api": False}
]
)
cluster_db = self.env.clusters[0]
node1, node2 = self.env.nodes
task = Task(
uuid=str(uuid.uuid4()),
name="super",
cluster_id=cluster_db.id
)
self.db.add(task)
self.db.commit()
kwargs = {'task_uuid': task.uuid,
'progress': 100,
'status': 'ready',
'nodes': [{'uid': node1.id},
{'uid': str(node2.id)}]}
self.receiver.remove_nodes_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.status, "ready")
nodes_db = self.db.query(Node).all()
self.assertEquals(len(nodes_db), 0)
def test_notification_delete_cluster_done(self):
cluster = self.env.create_cluster(api=False)
cluster_name = cluster.name
receiver = rcvr.NailgunReceiver()
task = Task(uuid=str(uuid.uuid4()),
name="cluster_deletion",
cluster_id=cluster.id)
self.db.add(task)
self.db.commit()
kwargs = {
'task_uuid': task.uuid,
'status': 'ready',
}
receiver.remove_cluster_resp(**kwargs)
notifications = self.db.query(Notification).all()
self.assertEqual(len(notifications), 1)
self.assertEqual(notifications[0].status, "unread")
self.assertEqual(notifications[0].topic, "done")
self.assertEqual(
notifications[0].message, "Environment '%s' and all its nodes "
"are deleted" % cluster_name)
def test_notification_delete_cluster_failed(self):
cluster = self.env.create_cluster(api=False)
receiver = rcvr.NailgunReceiver()
task = Task(uuid=str(uuid.uuid4()),
name="cluster_deletion",
cluster_id=cluster.id)
self.db.add(task)
self.db.commit()
kwargs = {
'task_uuid': task.uuid,
'status': 'error',
'error': 'Cluster deletion fake error'
}
receiver.remove_cluster_resp(**kwargs)
notifications = self.db.query(Notification).filter_by(
cluster_id=cluster.id).all()
self.assertEqual(len(notifications), 1)
self.assertEqual(notifications[0].status, "unread")
self.assertEqual(notifications[0].topic, "error")
self.assertEqual(notifications[0].cluster_id, cluster.id)
self.assertEqual(notifications[0].message,
"Cluster deletion fake error")
def PUT(self, cluster_id):
""":IMPORTANT: this method should be rewritten to be more RESTful
:returns: JSONized Task object.
:http: * 202 (network checking task failed)
* 200 (network verification task started)
* 404 (cluster not found in db)
"""
cluster = self.get_object_or_404(Cluster, cluster_id)
try:
data = self.validator.validate_networks_update(web.data())
except web.webapi.badrequest as exc:
task = Task(name='check_networks', cluster=cluster)
db().add(task)
db().commit()
TaskHelper.set_error(task.uuid, exc.data)
logger.error(traceback.format_exc())
json_task = build_json_response(TaskHandler.render(task))
raise web.accepted(data=json_task)
vlan_ids = [{
'name': n['name'],
'vlans': NetworkGroup.generate_vlan_ids_list(n)
} for n in data['networks']]
task_manager = VerifyNetworksTaskManager(cluster_id=cluster.id)
task = task_manager.execute(data, vlan_ids)
return TaskHandler.render(task)
def test_node_deploy_resp(self):
node = self.env.create_node(api=False)
node2 = self.env.create_node(api=False)
task = Task(uuid=str(uuid.uuid4()), name="deploy")
self.db.add(task)
self.db.commit()
kwargs = {
'task_uuid':
task.uuid,
'nodes': [{
'uid': node.id,
'status': 'deploying'
}, {
'uid': node2.id,
'status': 'error'
}]
}
self.receiver.deploy_resp(**kwargs)
self.db.refresh(node)
self.db.refresh(node2)
self.db.refresh(task)
self.assertEqual((node.status, node2.status), ("deploying", "error"))
# it is running because we don't stop deployment
# if there are error nodes
self.assertEqual(task.status, "running")
def test_remove_nodes_resp_failure(self):
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False},
{"api": False}
]
)
cluster_db = self.env.clusters[0]
node1, node2 = self.env.nodes
task = Task(
uuid=str(uuid.uuid4()),
name="super",
cluster_id=cluster_db.id
)
self.db.add(task)
self.db.commit()
kwargs = {'task_uuid': task.uuid,
'progress': 100,
'status': 'error',
'nodes': [],
'error_nodes': [{'uid': node1.id,
'error': "RPC method failed"}]}
self.receiver.remove_nodes_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.status, "error")
nodes_db = self.db.query(Node).all()
error_node = self.db.query(Node).get(node1.id)
self.db.refresh(error_node)
self.assertEquals(len(nodes_db), 2)
self.assertEquals(error_node.status, "error")
def execute(self):
logger.debug("Creating release dowload task")
task = Task(name="download_release")
db().add(task)
db().commit()
self._call_silently(task, tasks.DownloadReleaseTask, self.release_data)
return task
def test_verify_networks_with_dhcp_subtask_erred(self):
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False},
{"api": False}
]
)
cluster_db = self.env.clusters[0]
node1, node2 = self.env.nodes
nets_sent = [{'iface': 'eth0', 'vlans': range(100, 105)}]
task = Task(
name="verify_networks",
cluster_id=cluster_db.id
)
task.cache = {
"args": {
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': nets_sent}]
}
}
self.db.add(task)
self.db.commit()
dhcp_subtask = Task(
name='check_dhcp',
cluster_id=cluster_db.id,
parent_id=task.id,
status='error',
message='DHCP ERROR'
)
self.db.add(dhcp_subtask)
self.db.commit()
kwargs = {'task_uuid': task.uuid,
'status': 'ready',
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': []}]}
self.receiver.verify_networks_resp(**kwargs)
self.assertEqual(task.status, "error")
self.assertEqual(task.message, u'DHCP ERROR')
self.assertEqual(task.result, [{
u'absent_vlans': [100, 101, 102, 103, 104],
u'interface': 'eth0',
u'mac': node2.interfaces[0].mac,
u'name': None,
u'uid': node2.id}])
def execute(self):
logger.debug("Creating redhat_setup task")
current_tasks = db().query(Task).filter_by(name="redhat_setup")
for task in current_tasks:
for subtask in task.subtasks:
db().delete(subtask)
db().delete(task)
db().commit()
supertask = Task(name="redhat_setup")
supertask.result = {"release_info": {"release_id": self.data["release_id"]}}
db().add(supertask)
db().commit()
subtasks_to_create = [
("redhat_check_credentials", tasks.RedHatCheckCredentialsTask, 0.01),
("redhat_check_licenses", tasks.RedHatCheckLicensesTask, 0.01),
("redhat_download_release", tasks.RedHatDownloadReleaseTask, 1),
]
messages = []
for task_name, task_class, weight in subtasks_to_create:
task = supertask.create_subtask(task_name)
task.weight = weight
db().add(task)
db().commit()
msg = self._call_silently(task, task_class, self.data, method_name="message")
db().refresh(task)
if task.status == "error":
TaskHelper.update_task_status(supertask.uuid, status="error", progress=100, msg=task.message)
return supertask
task.cache = msg
db().add(task)
db().commit()
messages.append(msg)
db().refresh(supertask)
if supertask.status == "error":
return supertask
rpc.cast("naily", messages)
return supertask
def execute(self, nets, vlan_ids):
self.remove_previous_task()
task = Task(
name="check_networks",
cluster=self.cluster
)
if not task.cluster.nodes:
task.status = 'error'
task.message = ('There should be at least 1 node for dhcp check.'
'And 2 nodes for connectivity check')
db().add(task)
db().commit()
self._call_silently(
task,
tasks.CheckNetworksTask,
data=nets,
check_admin_untagged=True
)
db().refresh(task)
if task.status != 'error':
# this one is connected with UI issues - we need to
# separate if error happened inside nailgun or somewhere
# in the orchestrator, and UI does it by task name.
dhcp_subtask = Task(
name='check_dhcp',
cluster=self.cluster,
parent_id=task.id)
db().add(dhcp_subtask)
db().commit()
db().refresh(task)
task.name = 'verify_networks'
self._call_silently(
task,
tasks.VerifyNetworksTask,
vlan_ids
)
return task
def execute(self):
logger.info("Trying to start capacity_log task")
self.check_running_task('capacity_log')
task = Task(name='capacity_log')
db().add(task)
db().commit()
self._call_silently(task, tasks.GenerateCapacityLogTask)
return task
def test_task_progress(self):
task = Task(uuid=str(uuid.uuid4()), name="super", status="running")
self.db.add(task)
self.db.commit()
kwargs = {'task_uuid': task.uuid, 'progress': 20}
self.receiver.deploy_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.progress, 20)
self.assertEqual(task.status, "running")
def test_verify_networks_resp_without_vlans_only_erred(self):
"""Verify that network verification without vlans fails
when not all sended info received
"""
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False},
{"api": False}
]
)
cluster_db = self.env.clusters[0]
node1, node2 = self.env.nodes
nets_sent = [{'iface': 'eth0', 'vlans': [0]}]
nets_resp = [{'iface': 'eth0', 'vlans': []}]
task = Task(
name="super",
cluster_id=cluster_db.id
)
task.cache = {
"args": {
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': nets_sent}]
}
}
self.db.add(task)
self.db.commit()
kwargs = {'task_uuid': task.uuid,
'status': 'ready',
'nodes': [{'uid': node1.id, 'networks': nets_resp},
{'uid': node2.id, 'networks': nets_resp}]}
self.receiver.verify_networks_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.status, "error")
error_nodes = [{'uid': node1.id, 'interface': 'eth0',
'name': node1.name, 'mac': node1.interfaces[0].mac,
'absent_vlans': nets_sent[0]['vlans']},
{'uid': node2.id, 'interface': 'eth0',
'name': node2.name, 'mac': node2.interfaces[0].mac,
'absent_vlans': nets_sent[0]['vlans']}]
self.assertEqual(task.result, error_nodes)
def execute(self, data):
task = Task(name="check_networks", cluster=self.cluster)
orm().add(task)
orm().commit()
self._call_silently(task, tasks.CheckNetworksTask, data)
orm().refresh(task)
if task.status == 'running':
TaskHelper.update_task_status(task.uuid,
status="ready",
progress=100)
return task
def test_verify_networks_with_dhcp_subtask(self):
"""Test verifies that when dhcp subtask is ready and
verify_networks errored - verify_networks will be in error
"""
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False},
{"api": False}
]
)
cluster_db = self.env.clusters[0]
node1, node2 = self.env.nodes
nets_sent = [{'iface': 'eth0', 'vlans': range(100, 105)}]
task = Task(
name="verify_networks",
cluster_id=cluster_db.id
)
task.cache = {
"args": {
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': nets_sent}]
}
}
self.db.add(task)
self.db.commit()
dhcp_subtask = Task(
name='check_dhcp',
cluster_id=cluster_db.id,
parent_id=task.id,
status='ready'
)
self.db.add(dhcp_subtask)
self.db.commit()
kwargs = {'task_uuid': task.uuid,
'status': 'ready',
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': []}]}
self.receiver.verify_networks_resp(**kwargs)
self.assertEqual(task.status, "error")
def test_verify_networks_resp_incomplete_network_data_on_first_node(self):
"""Test verifies that when network data is incomplete on first node
task would not fail and be erred as expected
"""
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False, 'name': 'node1'},
{"api": False, 'name': 'node2'},
]
)
cluster_db = self.env.clusters[0]
node1, node2 = self.env.nodes
nets_sent = [{'iface': 'eth0', 'vlans': range(100, 105)}]
task = Task(
name="super",
cluster_id=cluster_db.id
)
task.cache = {
"args": {
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': nets_sent}]
}
}
self.db.add(task)
self.db.commit()
kwargs = {'task_uuid': task.uuid,
'status': 'ready',
'nodes': [{'uid': node1.id, 'networks': []},
{'uid': node2.id, 'networks': nets_sent}]}
self.receiver.verify_networks_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.status, "error")
self.assertEqual(task.message, None)
error_nodes = [{'uid': node1.id, 'interface': 'eth0',
'name': node1.name, 'mac': node1.interfaces[0].mac,
'absent_vlans': nets_sent[0]['vlans']}]
self.assertEqual(task.result, error_nodes)
def test_verify_networks_resp_incomplete_network_data_on_first_node(self):
"""Test verifies that when network data is incomplete on first node
task would not fail and be erred as expected
"""
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False, 'name': 'node1'},
{"api": False, 'name': 'node2'},
]
)
cluster_db = self.env.clusters[0]
node1, node2 = self.env.nodes
nets_sent = [{'iface': 'eth0', 'vlans': range(100, 105)}]
task = Task(
name="super",
cluster_id=cluster_db.id
)
task.cache = {
"args": {
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': nets_sent}]
}
}
self.db.add(task)
self.db.commit()
kwargs = {'task_uuid': task.uuid,
'status': 'ready',
'nodes': [{'uid': node1.id, 'networks': []},
{'uid': node2.id, 'networks': nets_sent}]}
self.receiver.verify_networks_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.status, "error")
self.assertEqual(task.message, '')
error_nodes = [{'uid': node1.id, 'interface': 'eth0',
'name': node1.name, 'mac': node1.interfaces[0].mac,
'absent_vlans': nets_sent[0]['vlans']}]
self.assertEqual(task.result, error_nodes)
def execute(self):
logger.info("Trying to start dump_environment task")
self.check_running_task('dump')
task = Task(name="dump")
db().add(task)
db().commit()
self._call_silently(
task,
tasks.DumpTask,
)
return task
def test_verify_networks_resp_error(self):
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False},
{"api": False}
]
)
cluster_db = self.env.clusters[0]
node1, node2 = self.env.nodes
nets_sent = [{'iface': 'eth0', 'vlans': range(100, 105)}]
nets_resp = [{'iface': 'eth0', 'vlans': range(100, 104)}]
task = Task(
name="super",
cluster_id=cluster_db.id
)
task.cache = {
"args": {
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': nets_sent}]
}
}
self.db.add(task)
self.db.commit()
kwargs = {'task_uuid': task.uuid,
'status': 'ready',
'nodes': [{'uid': node1.id, 'networks': nets_resp},
{'uid': node2.id, 'networks': nets_resp}]}
self.receiver.verify_networks_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.status, "error")
error_nodes = []
for node in self.env.nodes:
error_nodes.append({'uid': node.id, 'interface': 'eth0',
'name': node.name, 'absent_vlans': [104],
'mac': node.mac})
self.assertEqual(task.message, None)
self.assertEqual(task.result, error_nodes)
def execute(self, nets, vlan_ids):
task = Task(name="check_networks", cluster=self.cluster)
if not task.cluster.nodes:
task.status = 'error'
task.message = ('There should be at least 1 node for dhcp check.'
'And 2 nodes for connectivity check')
db().add(task)
db().commit()
self._call_silently(task, tasks.CheckNetworksTask, nets)
db().refresh(task)
if task.status != 'error':
# this one is connected with UI issues - we need to
# separate if error happened inside nailgun or somewhere
# in the orchestrator, and UI does it by task name.
dhcp_subtask = Task(name='check_dhcp',
cluster=self.cluster,
parent_id=task.id)
db().add(dhcp_subtask)
db().commit()
db().refresh(task)
task.name = 'verify_networks'
self._call_silently(task, tasks.VerifyNetworksTask, vlan_ids)
return task
def execute(self, nodes_to_provision):
"""Run provisioning task on specified nodes
Constraints: currently this task cannot deploy RedHat.
For redhat here should be added additional
tasks e.i. check credentials, check licenses,
redhat downloading.
Status of this task you can track here:
https://blueprints.launchpad.net/fuel/+spec
/nailgun-separate-provisioning-for-redhat
"""
TaskHelper.update_slave_nodes_fqdn(nodes_to_provision)
logger.debug('Nodes to provision: {0}'.format(
' '.join([n.fqdn for n in nodes_to_provision])))
task_provision = Task(name='provision', cluster=self.cluster)
db().add(task_provision)
db().commit()
provision_message = self._call_silently(
task_provision,
tasks.ProvisionTask,
nodes_to_provision,
method_name='message'
)
db().refresh(task_provision)
task_provision.cache = provision_message
for node in nodes_to_provision:
node.pending_addition = False
node.status = 'provisioning'
node.progress = 0
db().commit()
rpc.cast('naily', provision_message)
return task_provision
def test_verify_networks_resp_forgotten_node_error(self):
self.env.create(
cluster_kwargs={},
nodes_kwargs=[
{"api": False, 'name': 'node1'},
{"api": False, 'name': 'node2'},
{"api": False, 'name': 'node3'}
]
)
cluster_db = self.env.clusters[0]
node1, node2, node3 = self.env.nodes
nets_sent = [{'iface': 'eth0', 'vlans': range(100, 105)}]
task = Task(
name="super",
cluster_id=cluster_db.id
)
task.cache = {
"args": {
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': nets_sent},
{'uid': node3.id, 'networks': nets_sent}]
}
}
self.db.add(task)
self.db.commit()
kwargs = {'task_uuid': task.uuid,
'status': 'ready',
'nodes': [{'uid': node1.id, 'networks': nets_sent},
{'uid': node2.id, 'networks': nets_sent}]}
self.receiver.verify_networks_resp(**kwargs)
self.db.refresh(task)
self.assertEqual(task.status, "error")
self.assertRegexpMatches(task.message, node3.name)
self.assertEqual(task.result, [])
def execute(self):
logger.info(
u"Trying to start deployment at cluster '{0}'".format(
self.cluster.name or self.cluster.id,
)
)
current_tasks = db().query(Task).filter_by(
cluster_id=self.cluster.id,
name="deploy"
)
for task in current_tasks:
if task.status == "running":
raise errors.DeploymentAlreadyStarted()
elif task.status in ("ready", "error"):
for subtask in task.subtasks:
db().delete(subtask)
db().delete(task)
db().commit()
task_messages = []
nodes_to_delete = TaskHelper.nodes_to_delete(self.cluster)
nodes_to_deploy = TaskHelper.nodes_to_deploy(self.cluster)
nodes_to_provision = TaskHelper.nodes_to_provision(self.cluster)
if not any([nodes_to_provision, nodes_to_deploy, nodes_to_delete]):
raise errors.WrongNodeStatus("No changes to deploy")
self.cluster.status = 'deployment'
db().add(self.cluster)
db().commit()
supertask = Task(
name="deploy",
cluster=self.cluster
)
db().add(supertask)
db().commit()
if not self.cluster.replaced_provisioning_info \
and not self.cluster.replaced_deployment_info:
try:
self.check_before_deployment(supertask)
except errors.CheckBeforeDeploymentError:
return supertask
# in case of Red Hat
if self.cluster.release.operating_system == "RHEL":
try:
redhat_messages = self._redhat_messages(
supertask,
# provision only?
[
{"uid": n.id, "platform_name": n.platform_name}
for n in nodes_to_provision
]
)
except Exception as exc:
TaskHelper.update_task_status(
supertask.uuid,
status='error',
progress=100,
msg=str(exc)
)
return supertask
task_messages.extend(redhat_messages)
# /in case of Red Hat
task_deletion, task_provision, task_deployment = None, None, None
if nodes_to_delete:
task_deletion = supertask.create_subtask("node_deletion")
logger.debug("Launching deletion task: %s", task_deletion.uuid)
self._call_silently(
task_deletion,
tasks.DeletionTask
)
if nodes_to_provision:
TaskHelper.update_slave_nodes_fqdn(nodes_to_provision)
logger.debug("There are nodes to provision: %s",
" ".join([n.fqdn for n in nodes_to_provision]))
task_provision = supertask.create_subtask("provision")
# we assume here that task_provision just adds system to
# cobbler and reboots it, so it has extremely small weight
task_provision.weight = 0.05
provision_message = self._call_silently(
task_provision,
tasks.ProvisionTask,
method_name='message'
)
db().refresh(task_provision)
# if failed to generate task message for orchestrator
# then task is already set to error
if task_provision.status == 'error':
return supertask
task_provision.cache = provision_message
db().add(task_provision)
db().commit()
task_messages.append(provision_message)
if nodes_to_deploy:
TaskHelper.update_slave_nodes_fqdn(nodes_to_deploy)
logger.debug("There are nodes to deploy: %s",
" ".join([n.fqdn for n in nodes_to_deploy]))
task_deployment = supertask.create_subtask("deployment")
deployment_message = self._call_silently(
task_deployment,
tasks.DeploymentTask,
method_name='message'
)
# if failed to generate task message for orchestrator
# then task is already set to error
if task_deployment.status == 'error':
return supertask
task_deployment.cache = deployment_message
db().add(task_deployment)
db().commit()
task_messages.append(deployment_message)
if nodes_to_provision:
for node in nodes_to_provision:
node.status = 'provisioning'
db().commit()
if task_messages:
rpc.cast('naily', task_messages)
logger.debug(
u"Deployment: task to deploy cluster '{0}' is {1}".format(
self.cluster.name or self.cluster.id,
supertask.uuid
)
)
return supertask
def execute(self):
logger.info(u"Trying to start deployment at cluster '{0}'".format(self.cluster.name or self.cluster.id))
current_tasks = db().query(Task).filter_by(cluster_id=self.cluster.id, name="deploy")
for task in current_tasks:
if task.status == "running":
raise errors.DeploymentAlreadyStarted()
elif task.status in ("ready", "error"):
for subtask in task.subtasks:
db().delete(subtask)
db().delete(task)
db().commit()
task_messages = []
nodes_to_delete = TaskHelper.nodes_to_delete(self.cluster)
nodes_to_deploy = TaskHelper.nodes_to_deploy(self.cluster)
nodes_to_provision = TaskHelper.nodes_to_provision(self.cluster)
if not any([nodes_to_provision, nodes_to_deploy, nodes_to_delete]):
raise errors.WrongNodeStatus("No changes to deploy")
self.cluster.status = "deployment"
db().add(self.cluster)
db().commit()
supertask = Task(name="deploy", cluster=self.cluster)
db().add(supertask)
db().commit()
# checking admin intersection with untagged
network_info = NetworkConfigurationSerializer.serialize_for_cluster(self.cluster)
check_networks = supertask.create_subtask("check_networks")
self._call_silently(check_networks, tasks.CheckNetworksTask, data=network_info, check_admin_untagged=True)
db().refresh(check_networks)
if check_networks.status == "error":
return supertask
db().delete(check_networks)
db().commit()
# checking prerequisites
check_before = supertask.create_subtask("check_before_deployment")
logger.debug("Checking prerequisites task: %s", check_before.uuid)
self._call_silently(check_before, tasks.CheckBeforeDeploymentTask)
db().refresh(check_before)
# if failed to check prerequisites
# then task is already set to error
if check_before.status == "error":
logger.debug("Checking prerequisites failed: %s", check_before.message)
return supertask
logger.debug("Checking prerequisites is successful, starting deployment...")
db().delete(check_before)
db().commit()
# in case of Red Hat
if self.cluster.release.operating_system == "RHEL":
try:
redhat_messages = self._redhat_messages(
supertask,
# provision only?
[{"uid": n.id, "platform_name": n.platform_name} for n in nodes_to_provision],
)
except Exception as exc:
TaskHelper.update_task_status(supertask.uuid, status="error", progress=100, msg=str(exc))
return supertask
task_messages.extend(redhat_messages)
# /in case of Red Hat
task_deletion, task_provision, task_deployment = None, None, None
if nodes_to_delete:
task_deletion = supertask.create_subtask("node_deletion")
logger.debug("Launching deletion task: %s", task_deletion.uuid)
self._call_silently(task_deletion, tasks.DeletionTask)
if nodes_to_provision:
TaskHelper.update_slave_nodes_fqdn(nodes_to_provision)
logger.debug("There are nodes to provision: %s", " ".join([n.fqdn for n in nodes_to_provision]))
task_provision = supertask.create_subtask("provision")
# we assume here that task_provision just adds system to
# cobbler and reboots it, so it has extremely small weight
task_provision.weight = 0.05
provision_message = self._call_silently(task_provision, tasks.ProvisionTask, method_name="message")
db().refresh(task_provision)
# if failed to generate task message for orchestrator
# then task is already set to error
if task_provision.status == "error":
return supertask
task_provision.cache = provision_message
db().add(task_provision)
db().commit()
task_messages.append(provision_message)
if nodes_to_deploy:
TaskHelper.update_slave_nodes_fqdn(nodes_to_deploy)
logger.debug("There are nodes to deploy: %s", " ".join([n.fqdn for n in nodes_to_deploy]))
task_deployment = supertask.create_subtask("deployment")
deployment_message = self._call_silently(task_deployment, tasks.DeploymentTask, method_name="message")
# if failed to generate task message for orchestrator
# then task is already set to error
if task_deployment.status == "error":
return supertask
task_deployment.cache = deployment_message
db().add(task_deployment)
db().commit()
task_messages.append(deployment_message)
if task_messages:
rpc.cast("naily", task_messages)
logger.debug(
u"Deployment: task to deploy cluster '{0}' is {1}".format(
self.cluster.name or self.cluster.id, supertask.uuid
)
)
return supertask
def execute(self):
logger.info(
u"Trying to start deployment at cluster '{0}'".format(
self.cluster.name or self.cluster.id,
)
)
current_tasks = orm().query(Task).filter_by(
cluster_id=self.cluster.id,
name="deploy"
)
for task in current_tasks:
if task.status == "running":
raise errors.DeploymentAlreadyStarted()
elif task.status in ("ready", "error"):
for subtask in task.subtasks:
orm().delete(subtask)
orm().delete(task)
orm().commit()
nodes_to_delete = TaskHelper.nodes_to_delete(self.cluster)
nodes_to_deploy = TaskHelper.nodes_to_deploy(self.cluster)
if not any([nodes_to_deploy, nodes_to_delete]):
raise errors.WrongNodeStatus("No changes to deploy")
self.cluster.status = 'deployment'
orm().add(self.cluster)
orm().commit()
supertask = Task(
name="deploy",
cluster=self.cluster
)
orm().add(supertask)
orm().commit()
task_deletion, task_provision, task_deployment = None, None, None
if nodes_to_delete:
task_deletion = supertask.create_subtask("node_deletion")
self._call_silently(
task_deletion,
tasks.DeletionTask
)
if nodes_to_deploy:
TaskHelper.update_slave_nodes_fqdn(nodes_to_deploy)
task_provision = supertask.create_subtask("provision")
# we assume here that task_provision just adds system to
# cobbler and reboots systems, so it has extreamly small weight
task_provision.weight = 0.05
provision_message = self._call_silently(
task_provision,
tasks.ProvisionTask,
method_name='message'
)
task_provision.cache = provision_message
orm().add(task_provision)
orm().commit()
task_deployment = supertask.create_subtask("deployment")
deployment_message = self._call_silently(
task_deployment,
tasks.DeploymentTask,
method_name='message'
)
task_deployment.cache = deployment_message
orm().add(task_deployment)
orm().commit()
rpc.cast('naily', [provision_message, deployment_message])
logger.debug(
u"Deployment: task to deploy cluster '{0}' is {1}".format(
self.cluster.name or self.cluster.id,
supertask.uuid
)
)
return supertask
