def test_crud_node(self):
""" It tests the basic CRUD operations of an Node class """
# We verify the object is not in the db after creating it
node = Node()
node.name = "node1"
node.information_retrieved = True
self.assertIsNone(node.id)
# We store the object in the db
db.session.add(node)
# We recover the node from the db
node = db.session.query(Node).filter_by(name='node1').first()
self.assertIsNotNone(node.id)
self.assertEquals("node1", node.name)
self.assertTrue(node.information_retrieved)
# We update the node
node.information_retrieved = False
db.session.commit()
node = db.session.query(Node).filter_by(name='node1').first()
self.assertFalse(node.information_retrieved)
# We check the deletion
db.session.delete(node)
count = db.session.query(Node).filter_by(name='node1').count()
self.assertEquals(0, count)
def test_many_to_many_relations_with_nodes(self):
"""
It tests the many to many relations with Nodes
"""
node_1 = Node()
node_1.name = "node1"
node_1.information_retrieved = False
node_2 = Node()
node_2.name = "node2"
node_2.information_retrieved = False
db.session.add(node_1)
db.session.add(node_2)
execution_1 = Execution()
execution_1.status = "x1"
execution_2 = Execution()
execution_2.status = "x2"
db.session.add(execution_1)
db.session.add(execution_2)
db.session.commit()
execution_1.nodes = [node_1, node_2]
execution_2.nodes = [node_2, node_1]
db.session.commit()
execution = db.session.query(Execution).filter_by(status="x1").first()
self.assertEquals(node_1, execution.nodes[0])
self.assertEquals(node_2, execution.nodes[1])
execution = db.session.query(Execution).filter_by(status="x2").first()
self.assertEquals(node_2, execution.nodes[0])
self.assertEquals(node_1, execution.nodes[1])
def setUp(self):
"""
It creates the memory db
"""
db.create_all()
# We store some Applications in the db for the tests
application_1 = Application()
application_1.name = 'AppName_1'
application_2 = Application()
application_2.name = 'AppName_2'
# Adding executing scripts
execution_script_1 = ExecutionConfiguration()
execution_script_1.execution_type = "slurm:sbatch"
execution_script_2 = ExecutionConfiguration()
execution_script_2.execution_type = "slurm:sbatch2"
application_2.execution_configurations = [
execution_script_1, execution_script_2
]
db.session.add(application_1)
db.session.add(application_2)
# We store some testbeds in the db for the tests
testbed_1 = Testbed("name_1", True, "slurm", "ssh", "user@server",
['slurm'])
testbed_2 = Testbed("name_2", False, "slurm", "ssh", "user@server",
['slurm'])
testbed_3 = Testbed("name_3", True, "slurm", "ssh", "user@server",
['slurm', 'slurm:singularity'])
db.session.add(testbed_1)
db.session.add(testbed_2)
db.session.add(testbed_3)
db.session.commit()
deployment = Deployment()
deployment.executable_id = execution_script_1.id
deployment.testbed_id = testbed_1.id
db.session.add(deployment)
# We store some nodes in the db for the tests
node_1 = Node()
node_1.name = "node_1"
node_1.information_retrieved = True
node_2 = Node()
node_2.name = "node_2"
node_2.information_retrieved = False
db.session.add(node_1)
db.session.add(node_2)
execution = Execution()
execution.execution_type = "execution_type"
execution.status = "status"
db.session.add(execution)
db.session.commit()
def test_node_gpu_relation(self):
"""
Unit test that verifies that the correct relation between
the GPU class and the node class is built
"""
# We create a node
node = Node()
node.name = "node1"
node.information_retrieved = True
# We add several CPUs to it
node.gpus = [
GPU("Nvidia", "GeForce"),
GPU("AMD", "Raedon")
]
# We save everything to the db
db.session.add(node)
db.session.commit()
# We retrived and verify that the gpus are there
node = db.session.query(Node).filter_by(name='node1').first()
self.assertEquals(2, len(node.gpus))
self.assertEquals("Nvidia", node.gpus[0].vendor_id)
self.assertEquals("AMD", node.gpus[1].vendor_id)
# Lets delete a gpu directly from the session
db.session.delete(node.gpus[1])
db.session.commit()
node = db.session.query(Node).filter_by(name='node1').first()
self.assertEquals(1, len(node.gpus))
self.assertEquals("Nvidia", node.gpus[0].vendor_id)
def test_node_cpu_relation(self):
"""
Unit test that verifies that the correct relation between
the testbed class and the node class is built
"""
# We create a node
node = Node()
node.name = "node1"
node.information_retrieved = True
# We add several CPUs to it
node.cpus = [
CPU("Intel", "Xeon", "x86_64", "e6333", "2600Mhz", True, 2, "cache", "111"),
CPU("AMD", "Zen", "x86_64", "xxxxx", "2600Mhz", True, 2, "cache", "111")
]
# We save everything to the db
db.session.add(node)
db.session.commit()
# We retrived and verify that the cpus are there
node = db.session.query(Node).filter_by(name='node1').first()
self.assertEquals(2, len(node.cpus))
self.assertEquals("Intel", node.cpus[0].vendor_id)
self.assertEquals("AMD", node.cpus[1].vendor_id)
# Lets delete a cpu directly from the session
db.session.delete(node.cpus[1])
db.session.commit()
node = db.session.query(Node).filter_by(name='node1').first()
self.assertEquals(1, len(node.cpus))
self.assertEquals("Intel", node.cpus[0].vendor_id)
def test_node_memory_relation(self):
"""
Unit test that verifies that the correct relation between
the Memory class and the node class is built
"""
# We create a node
node = Node()
node.name = "node1"
node.information_retrieved = True
# We add several CPUs to it
node.memories = [
Memory(11111, "bytes"),
Memory(11211, "bytes")
]
# We save everything to the db
db.session.add(node)
db.session.commit()
# We retrived and verify that the gpus are there
node = db.session.query(Node).filter_by(name='node1').first()
self.assertEquals(2, len(node.memories))
self.assertEquals(11111, node.memories[0].size)
self.assertEquals(11211, node.memories[1].size)
# Lets delete a gpu directly from the session
db.session.delete(node.memories[1])
db.session.commit()
node = db.session.query(Node).filter_by(name='node1').first()
self.assertEquals(1, len(node.memories))
self.assertEquals(11111, node.memories[0].size)
def _create_initial_db_data(self):
"""
This method creates some initial data in the db that is employed by
some of the tests here.
"""
# We add two testbeds to the db
testbed = Testbed("name1", True, Testbed.slurm_category, "ssh",
"user@server", ['slurm'])
# We add some nodes to Testbed_1
node_1 = Node()
node_1.name = "node_1"
node_1.information_retrieved = True
node_2 = Node()
node_2.name = "node_2"
node_2.information_retrieved = True
testbed.nodes = [node_1, node_2]
node_1.disabled = True
db.session.add(testbed)
db.session.commit()
return testbed, node_1, node_2
def test_testbed_node_relation(self):
"""
Unit test that verifies that the correct relation between
the testbed class and the node class is built
"""
# We create a testbed
testbed = Testbed("name", True, "slurm", "ssh", "user@server", ['slurm'])
# We create several nodes
node_1 = Node()
node_1.name = "node1"
node_1.information_retrieved = True
node_2 = Node()
node_2.name = "node2"
node_2.information_retrieved = False
node_3 = Node()
node_3.name = "node3"
node_3.information_retrieved = True
testbed.nodes = [ node_1, node_2, node_3 ]
# We save everything to the db
db.session.add(testbed)
db.session.commit()
# We retrieve and verify that the nodes are there
testbed = db.session.query(Testbed).filter_by(name='name').first()
self.assertEquals(3, len(testbed.nodes))
self.assertEquals("node1", testbed.nodes[0].name)
self.assertTrue(testbed.nodes[0].information_retrieved)
self.assertEquals("node2", testbed.nodes[1].name)
self.assertFalse(testbed.nodes[1].information_retrieved)
self.assertEquals("node3", testbed.nodes[2].name)
self.assertTrue(testbed.nodes[2].information_retrieved)
# lets remove a node from testbed
testbed.remove_node(testbed.nodes[2])
# We commit the state and it should be updated
db.session.commit()
testbed = db.session.query(Testbed).filter_by(name='name').first()
self.assertEquals(2, len(testbed.nodes))
self.assertEquals("node1", testbed.nodes[0].name)
self.assertTrue(testbed.nodes[0].information_retrieved)
self.assertEquals("node2", testbed.nodes[1].name)
self.assertFalse(testbed.nodes[1].information_retrieved)
# Lets delete the node directly from the session
db.session.delete(testbed.nodes[1])
db.session.commit()
testbed = db.session.query(Testbed).filter_by(name='name').first()
self.assertEquals(1, len(testbed.nodes))
self.assertEquals("node1", testbed.nodes[0].name)
self.assertTrue(testbed.nodes[0].information_retrieved)
# It should be only one node in the db
nodes = db.session.query(Node).all()
self.assertEquals(2, len(nodes))
self.assertEquals("node1", nodes[0].name)
self.assertTrue(nodes[0].information_retrieved)
self.assertIsNotNone(nodes[0].testbed)
self.assertEquals("node3", nodes[1].name)
self.assertTrue(nodes[1].information_retrieved)
self.assertIsNone(nodes[1].testbed)
def test_can_create_the_node(self):
"""
Checks the right behaviour that checks if it is possible to add
a node
"""
# Node not associated to any db
node = Node()
node.name = "xxx"
node.information_retrieved = True
is_possible = alde.can_create_the_node(node)
self.assertEquals(True, is_possible['create'])
self.assertEquals('', is_possible['reason'])
# Node has an ivalid testbed id
node.testbed_id = 100000
is_possible = alde.can_create_the_node(node)
self.assertEquals(False, is_possible['create'])
self.assertEquals('Testbed does not exist', is_possible['reason'])
# Testbed exists but information is retrieved automatically
node.testbed_id = 1
is_possible = alde.can_create_the_node(node)
self.assertEquals(False, is_possible['create'])
self.assertEquals('Testbed is configured to automatically retrieve information of nodes',
is_possible['reason'])
# Testeb exists but information is not retrieved automatically
node.testbed_id = 2
is_possible = alde.can_create_the_node(node)
self.assertEquals(True, is_possible['create'])
self.assertEquals('', is_possible['reason'])
# Testbed info is passed by the url:
node = Node()
node.name = "xxx"
node.information_retrieved = True
## Testbed does not allow it
is_possible = alde.can_create_the_node(node, testbed_id=1)
self.assertEquals(False, is_possible['create'])
self.assertEquals('Testbed is configured to automatically retrieve information of nodes',
is_possible['reason'])
## Testbed does allow it
is_possible = alde.can_create_the_node(node, testbed_id=2)
self.assertEquals(True, is_possible['create'])
self.assertEquals('', is_possible['reason'])
# Testbed and nodes are created at the same time
testbed_1 = {
'name' : 'name_3',
'on_line' : True
}
testbed_2 = {
'name' : 'name_3',
'on_line' : False
}
## Testbed does not allow it
is_possible = alde.can_create_the_node(node, testbed=testbed_1)
self.assertEquals(False, is_possible['create'])
self.assertEquals('Testbed is configured to automatically retrieve information of nodes',
is_possible['reason'])
## Testbed does allow it
is_possible = alde.can_create_the_node(node, testbed=testbed_2)
self.assertEquals(True, is_possible['create'])
self.assertEquals('', is_possible['reason'])
def test_update_node_information(self, mock_shell):
"""
Test that the correct work of this function
"""
l = LogCapture() # we capture the logger
command = "pbsnodes"
params = ["-x"]
# We store some data in the db for the test.
# We add a testbed to the db
testbed = Testbed("name1", True, Testbed.torque_category,
Testbed.protocol_ssh, "user@server", ['torque'])
# We add some nodes to Testbed_1
node_1 = Node()
node_1.name = "node-1.novalocal"
node_1.information_retrieved = True
node_2 = Node()
node_2.name = "node-2.novalocal"
node_2.information_retrieved = True
testbed.nodes = [node_1, node_2]
node_1.disabled = True
db.session.add(testbed)
db.session.commit()
# We mock the command call
mock_shell.return_value = self.command_pbsnodes_output
testbeds.facade.update_testbed_node_information(testbed)
# We verify the results
node_1 = db.session.query(Node).filter_by(
name='node-1.novalocal').first()
node_2 = db.session.query(Node).filter_by(
name='node-2.novalocal').first()
self.assertEqual('free', node_1.state)
self.assertEqual(1, len(node_1.memories))
self.assertEqual(Memory.KILOBYTE, node_1.memories[0].units)
self.assertEqual(131583196, node_1.memories[0].size)
self.assertEqual(1, len(node_1.gpus))
self.assertEqual('free', node_2.state)
self.assertEqual(1, len(node_2.memories))
self.assertEqual(Memory.KILOBYTE, node_2.memories[0].units)
self.assertEqual(131583197, node_2.memories[0].size)
self.assertEqual(0, len(node_2.gpus))
mock_shell.assert_called_with(command=command,
server="user@server",
params=params)
self.assertEqual(1, mock_shell.call_count)
# Checking that we are logging the correct messages
l.check(
('root', 'INFO',
'Updating information for node: node-1.novalocal if necessary'),
('root', 'INFO',
'Updating memory information for node: node-1.novalocal'),
('root', 'INFO',
'Updating gpu information for node: node-1.novalocal'),
('root', 'INFO',
'Updating information for node: node-2.novalocal if necessary'),
('root', 'INFO',
'Updating memory information for node: node-2.novalocal'),
)
l.uninstall() # We uninstall the capture of the logger
def test_update_node_information(self, mock_shell):
"""
Test that the correct work of this function
"""
l = LogCapture() # we cature the logger
command = "scontrol"
params = ["-o", "--all", "show", "node"]
# We store some data in the db for the test.
# We add two testbeds to the db
testbed = Testbed("name1", True, Testbed.slurm_category,
Testbed.protocol_ssh, "user@server", ['slurm'])
# We add some nodes to Testbed_1
node_1 = Node()
node_1.name = "nd80"
node_1.information_retrieved = True
node_2 = Node()
node_2.name = "nd23"
node_2.information_retrieved = True
testbed.nodes = [node_1, node_2]
node_1.disabled = True
db.session.add(testbed)
db.session.commit()
# We mock the command call
mock_shell.return_value = self.command_scontrol_output
slurm.update_node_information()
# We verify the results
node_80 = db.session.query(Node).filter_by(name='nd80').first()
node_23 = db.session.query(Node).filter_by(name='nd23').first()
self.assertEquals('ALLOCATED', node_80.state)
self.assertEquals(1, len(node_80.memories))
self.assertEquals(Memory.MEGABYTE, node_80.memories[0].units)
self.assertEquals(6850663, node_80.memories[0].size)
self.assertEquals(0, len(node_80.gpus))
self.assertEquals('MAINT', node_23.state)
self.assertEquals(1, len(node_23.memories))
self.assertEquals(Memory.MEGABYTE, node_23.memories[0].units)
self.assertEquals(24018, node_23.memories[0].size)
self.assertEquals(2, len(node_23.gpus))
self.assertEquals('Nvidia', node_23.gpus[0].vendor_id)
self.assertEquals('Nvidia TESLA C2075', node_23.gpus[0].model_name)
self.assertEquals('Nvidia', node_23.gpus[1].vendor_id)
self.assertEquals('Nvidia TESLA C2075', node_23.gpus[1].model_name)
mock_shell.assert_called_with(command=command,
server="user@server",
params=params)
self.assertEquals(1, mock_shell.call_count)
# Checking that we are logging the correct messages
l.check(('root', 'INFO',
'Updating information for node: nd80 if necessary'),
('root', 'INFO', 'Updating memory information for node: nd80'),
('root', 'INFO',
'Updating information for node: nd23 if necessary'),
('root', 'INFO', 'Updating memory information for node: nd23'),
('root', 'INFO', 'Updating gpu information for node: nd23'))
l.uninstall() # We uninstall the capture of the logger
def test_update_cpu_node_information(self, mock_parser):
"""
Test that the correct work of this function
"""
l = LogCapture() # we cature the logger
# We store some data in the db for the test.
testbed, node_1, node_2 = self._create_initial_db_data()
node_3 = Node()
node_3.name = "node_3"
node_3.information_retrieved = True
testbed.nodes.append(node_3)
node_3.cpus = [
CPU("Intel", "Xeon", "x86_64", "e6333", "2600Mhz", True, 2,
"cache", "111")
]
# So, testbed has 3 nodes, one disabled and the other ones enabled
db.session.commit()
cpus_result = [
CPU("Intel2", "Xeon2", "x86_64", "e6333", "2600Mhz", True, 2,
"cache", "111"),
CPU("Intel3", "Xeon3", "x86_64", "e6333", "2600Mhz", True, 2,
"cache", "111")
]
mock_parser.return_value = cpus_result
slurm.update_cpu_node_information()
# We verify the results
self.assertEquals(
0, len(db.session.query(CPU).filter_by(vendor_id="Intel").all()))
self.assertEquals(
1, len(db.session.query(CPU).filter_by(vendor_id="Intel2").all()))
self.assertEquals(
1, len(db.session.query(CPU).filter_by(vendor_id="Intel3").all()))
calls = [mock.call(testbed, node_2), mock.call(testbed, node_3)]
mock_parser.assert_has_calls(calls)
self.assertEquals(2, mock_parser.call_count)
# In case an error occours retrieving the information
mock_parser.return_value = []
slurm.update_cpu_node_information()
calls = [mock.call(testbed, node_2), mock.call(testbed, node_3)]
mock_parser.assert_has_calls(calls)
self.assertEquals(4, mock_parser.call_count)
# Checking that we are logging the correct messages
l.check(('root', 'INFO', 'Updating CPU info for node: node_2'),
('root', 'INFO', 'Updating CPU info for node: node_3'),
('root', 'ERROR',
'Impossible to update CPU info for node: node_2'),
('root', 'ERROR',
'Impossible to update CPU info for node: node_3'))
l.uninstall() # We uninstall the capture of the logger
def test_get_cpuinfo_node(self, mock_shell):
"""
It verfies that given a testbed it is possible to get the cpuinfo
information of the node.
"""
l = LogCapture() # we cature the logger
# When the testbed is local
testbed = Testbed("name1",
True,
Testbed.slurm_category,
Testbed.protocol_local,
"user@server",
['slurm'])
node_1 = Node() # We add some nodes to Testbed_1
node_1.name = "node_1"
node_1.information_retrieved = True
node_2 = Node()
node_2.name = "node_2"
node_2.information_retrieved = True
testbed.nodes = [ node_1, node_2]
node_1.disabled = True
node_3 = Node()
node_3.name = "node_3"
node_3.information_retrieved = True
# When the node does not belong to the testbed it should return empty list
cpus = parser.get_cpuinfo_node(testbed, node_3)
self.assertEquals(0, len(cpus))
# When the node is there, we have to get double CPU info
mock_shell.return_value = self.command_output
cpus = parser.get_cpuinfo_node(testbed, node_2)
self.assertEquals(8, len(cpus))
mock_shell.assert_called_with(command="ssh",
params=["node_2", "'cat", "/proc/cpuinfo'"])
self.assertEqual(mock_shell.call_count, 1)
# When the node is dissabled it should return an empty list
cpus = parser.get_cpuinfo_node(testbed, node_1)
self.assertEquals(0, len(cpus))
self.assertEqual(mock_shell.call_count, 1)
# When the testbed is using ssh protocol
testbed = Testbed("name1",
True,
Testbed.slurm_category,
Testbed.protocol_ssh,
"user@server",
['slurm'])
testbed.nodes = [node_2]
mock_shell.return_value = self.command_output
cpus = parser.get_cpuinfo_node(testbed, node_2)
self.assertEquals(8, len(cpus))
mock_shell.assert_called_with(command="ssh",
server=testbed.endpoint,
params=["node_2", "'cat", "/proc/cpuinfo'"])
self.assertEqual(mock_shell.call_count, 2)
# We simulate what happens if we get an exception executing the command
error = subprocess.CalledProcessError(returncode=255, cmd="ls")
mock_shell.side_effect = error
cpus = parser.get_cpuinfo_node(testbed, node_2)
self.assertEquals(0, len(cpus))
self.assertEqual(mock_shell.call_count, 3)
# When the testbed has an unknown protocol
testbed = Testbed("name1",
True,
Testbed.slurm_category,
"xxx",
"user@server",
['slurm'])
testbed.nodes = [node_2]
cpus = parser.get_cpuinfo_node(testbed, node_2)
self.assertEquals(0, len(cpus))
self.assertEqual(mock_shell.call_count, 3)
# We verify that we raised the right errors
# Checking that we are logging the correct messages
l.check(
('root', 'ERROR', 'Exception trying to get the node cpu info'),
('root', 'INFO', 'Tesbed protocol: xxx not supported to get node information')
)
l.uninstall() # We uninstall the capture of the logger
