def setUp(self):
django.setup()
Scheduler.objects.initialize_scheduler()
self._driver = MagicMock()
self._job_exe_manager = RunningJobExecutionManager()
self._job_type_manager = JobTypeManager()
self._node_manager = NodeManager()
self._offer_manager = OfferManager()
self._scheduler_manager = SchedulerManager()
self._workspace_manager = WorkspaceManager()
self._scheduler_manager.sync_with_database()
self.node_agent_1 = 'agent_1'
self.node_agent_2 = 'agent_2'
self.node_1 = node_test_utils.create_node(hostname='host_1', slave_id=self.node_agent_1)
self.node_2 = node_test_utils.create_node(hostname='host_2', slave_id=self.node_agent_2)
self.slave_infos = [SlaveInfo('host_1', slave_id=self.node_agent_1),
SlaveInfo('host_2', slave_id=self.node_agent_2)]
self._node_manager.add_agent_ids([self.node_agent_1, self.node_agent_2])
with patch('scheduler.sync.node_manager.api.get_slaves') as mock_get_slaves:
mock_get_slaves.return_value = self.slave_infos
self._node_manager.sync_with_database('master_host', 5050)
self.queue_1 = queue_test_utils.create_queue(cpus_required=4.0, mem_required=1024.0, disk_in_required=100.0,
disk_out_required=200.0, disk_total_required=300.0)
self.queue_2 = queue_test_utils.create_queue(cpus_required=8.0, mem_required=512.0, disk_in_required=400.0,
disk_out_required=45.0, disk_total_required=445.0)
self._job_type_manager.sync_with_database()
self._scheduling_thread = SchedulingThread(self._driver, '123', self._job_exe_manager, self._job_type_manager,
self._node_manager, self._offer_manager, self._scheduler_manager,
self._workspace_manager)
def test_nodes_stats(self, mock_get_slaves):
"""This method tests retrieving all the nodes statistics
for the three hour duration requested"""
mock_get_slaves.return_value = [
SlaveInfo(self.node1.hostname, self.node1.port,
HardwareResources(1, 2, 3)),
SlaveInfo(self.node3.hostname, self.node3.port,
HardwareResources(4, 5, 6)),
]
response = self.client.generic('GET',
'/v4/nodes/status/?started=PT3H00M0S')
self.assertEqual(response.status_code, status.HTTP_200_OK,
response.content)
result = json.loads(response.content)
self.assertTrue(isinstance(result, dict),
'result must be a dictionary')
assert_message = '({0} != 3) Expected to get the 3 nodes'.format(
len(result['results']))
self.assertEqual(len(result['results']), 3, assert_message)
for entry in result['results']:
hostname = entry['node']['hostname']
self.assertIn(hostname, [
self.node1.hostname, self.node2.hostname, self.node3.hostname
])
if hostname == self.node1.hostname:
self.assertTrue(entry['is_online'])
self.assertEqual(len(entry['job_exe_counts']), 2)
for status_count in entry['job_exe_counts']:
if status_count['status'] == 'COMPLETED':
self.assertEqual(status_count['count'], 1)
elif status_count['status'] == 'FAILED':
self.assertEqual(status_count['count'], 1)
else:
self.fail('Unexpected job execution status found: %s' %
status_count['status'])
elif hostname == self.node2.hostname:
self.assertFalse(entry['is_online'])
for status_count in entry['job_exe_counts']:
if status_count['status'] == 'FAILED' and status_count[
'category'] == 'DATA':
self.assertEqual(status_count['count'], 1)
elif status_count['status'] == 'FAILED' and status_count[
'category'] == 'SYSTEM':
self.assertEqual(status_count['count'], 1)
else:
self.fail('Unexpected job execution status found: %s' %
status_count['status'])
elif hostname == self.node3.hostname:
self.assertTrue(entry['is_online'])
self.assertEqual(len(entry['job_exes_running']), 1)
for status_count in entry['job_exe_counts']:
if status_count['status'] == 'RUNNING':
self.assertEqual(status_count['count'], 1)
def setUp(self):
django.setup()
self.node_agent_1 = 'agent_1'
self.node_agent_2 = 'agent_2'
self.node_1 = node_test_utils.create_node(hostname='host_1', slave_id=self.node_agent_1)
self.slave_infos = [SlaveInfo('host_1', slave_id=self.node_agent_1),
SlaveInfo('host_2', slave_id=self.node_agent_2)]
def test_nodes_system_stats(self, mock_get_slaves):
'''This method tests for when a node has not processed any jobs for the duration of time requested.'''
mock_get_slaves.return_value = [
SlaveInfo(self.node1.hostname, self.node1.port, HardwareResources(1, 2, 3)),
SlaveInfo(self.node3.hostname, self.node3.port, HardwareResources(4, 5, 6)),
]
url = u'/nodes/status/?started=PT1H30M0S'
response = self.client.generic('GET', url)
self.assertEqual(response.status_code, status.HTTP_200_OK)
result = json.loads(response.content)
self.assertTrue(isinstance(result, dict), u'result must be a dictionary')
assert_message = u'({0} != 3) Expected to get the 3 nodes'.format(len(result[u'results']))
self.assertEqual(len(result[u'results']), 3, assert_message)
for entry in result[u'results']:
hostname = entry[u'node'][u'hostname']
self.assertIn(hostname, [self.node1.hostname, self.node2.hostname, self.node3.hostname])
if hostname == self.node1.hostname:
self.assertTrue(entry[u'is_online'])
self.assertEqual(len(entry[u'job_exe_counts']), 2)
for status_count in entry[u'job_exe_counts']:
if status_count[u'status'] == u'COMPLETED':
self.assertEqual(status_count[u'count'], 1)
elif status_count[u'status'] == u'FAILED':
self.assertEqual(status_count[u'count'], 1)
else:
self.fail(u'Unexpected job execution status found: %s' % status_count[u'status'])
elif hostname == self.node2.hostname:
self.assertFalse(entry[u'is_online'])
for status_count in entry[u'job_exe_counts']:
if status_count[u'status'] == u'FAILED' and status_count[u'category'] == u'DATA':
self.assertEqual(status_count[u'count'], 1)
elif status_count[u'status'] == u'FAILED' and status_count[u'category'] == u'SYSTEM':
self.assertEqual(status_count[u'count'], 1)
else:
self.fail(u'Unexpected job execution status found: %s' % status_count[u'status'])
elif hostname == self.node3.hostname:
self.assertTrue(entry[u'is_online'])
self.assertEqual(len(entry[u'job_exes_running']), 1)
for status_count in entry[u'job_exe_counts']:
if status_count[u'status'] == u'RUNNING':
self.assertEqual(status_count[u'count'], 1)
def test_get_node_master_disconnected(self, mock_get_slave):
'''Test calling the Get Node method with a disconnected master.'''
mock_get_slave.return_value = SlaveInfo(self.node2.hostname, self.node2.port)
url = '/nodes/%d/' % self.node2.id
response = self.client.get(url)
data = json.loads(response.content)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertNotIn('resources', data)
self.assertEqual(data['disconnected'], True)
def test_get_node_master_disconnected(self, mock_get_slave):
"""Test calling the Get Node method with a disconnected master."""
mock_get_slave.return_value = SlaveInfo(self.node2.hostname,
self.node2.port)
response = self.client.get('/v4/nodes/%d/' % self.node2.id)
self.assertEqual(response.status_code, status.HTTP_200_OK,
response.content)
result = json.loads(response.content)
self.assertNotIn('resources', result)
self.assertEqual(result['disconnected'], True)
def setUp(self):
django.setup()
Scheduler.objects.initialize_scheduler()
self._driver = MagicMock()
scheduler_mgr.sync_with_database()
offer_mgr.clear()
self.node_agent_1 = 'agent_1'
self.node_agent_2 = 'agent_2'
self.slave_infos = [
SlaveInfo('host_1', slave_id=self.node_agent_1),
SlaveInfo('host_2', slave_id=self.node_agent_2)
]
node_mgr.clear()
node_mgr.register_agent_ids([self.node_agent_1, self.node_agent_2])
with patch('scheduler.node.manager.api.get_slaves') as mock_get_slaves:
mock_get_slaves.return_value = self.slave_infos
node_mgr.sync_with_database('master_host', 5050)
# Ignore initial cleanup tasks and health check tasks
for node in node_mgr.get_nodes():
node._last_heath_task = now()
node._initial_cleanup_completed()
node._update_state()
self.queue_1 = queue_test_utils.create_queue(cpus_required=4.0,
mem_required=1024.0,
disk_in_required=100.0,
disk_out_required=200.0,
disk_total_required=300.0)
self.queue_2 = queue_test_utils.create_queue(cpus_required=8.0,
mem_required=512.0,
disk_in_required=400.0,
disk_out_required=45.0,
disk_total_required=445.0)
job_type_mgr.sync_with_database()
self._scheduling_thread = SchedulingThread(self._driver, '123')
def test_get_node_success(self, mock_get_slave):
"""Test successfully calling the Get Node method."""
mock_get_slave.return_value = SlaveInfo(
self.node2.hostname, self.node2.port,
HardwareResources(4., 2048., 40000.))
url = rest_util.get_url('/nodes/%d/' % self.node2.id)
response = self.client.get(url)
self.assertEqual(response.status_code, status.HTTP_200_OK,
response.content)
result = json.loads(response.content)
self.assertIn('hostname', result)
self.assertEqual(result['hostname'], self.node2.hostname)
def test_get_node_success(self, mock_get_slave):
'''Test successfully calling the Get Node method.'''
mock_get_slave.return_value = SlaveInfo(self.node2.hostname, self.node2.port,
HardwareResources(4., 2048., 40000.))
url = '/nodes/%d/' % self.node2.id
response = self.client.get(url)
data = json.loads(response.content)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertIn('hostname', data)
self.assertEqual(data['hostname'], self.node2.hostname)
self.assertEqual(data['resources']['total']['cpus'], 4.)
self.assertEqual(data['resources']['total']['mem'], 2048.)
self.assertEqual(data['resources']['total']['disk'], 40000.)
self.assertEqual(data['job_exes_running'], [])
self.assertNotIn('disconnected', data)
def test_update_node_unpause(self, mock_get_slave):
"""Tests unpausing the node and specifying a reason."""
mock_get_slave.return_value = SlaveInfo(
self.node2.hostname, self.node2.port,
HardwareResources(4., 2048., 40000.))
json_data = {'is_paused': False, 'pause_reason': 'Test reason'}
url = rest_util.get_url('/nodes/%d/' % self.node2.id)
response = self.client.patch(url, json.dumps(json_data),
'application/json')
self.assertEqual(response.status_code, status.HTTP_200_OK,
response.content)
result = json.loads(response.content)
self.assertEqual(result['is_paused'], False)
self.assertIsNone(result['pause_reason'])
self.assertIn('hostname', result)
self.assertEqual(result['hostname'], self.node2.hostname)
def test_update_node_success(self, mock_get_slave):
"""Test successfully calling the Update Node method."""
mock_get_slave.return_value = SlaveInfo(
self.node2.hostname, self.node2.port,
HardwareResources(4., 2048., 40000.))
json_data = {
'is_paused': True,
'pause_reason': 'Test reason',
}
url = rest_util.get_url('/nodes/%d/' % self.node2.id)
response = self.client.patch(url, json.dumps(json_data),
'application/json')
self.assertEqual(response.status_code, status.HTTP_200_OK,
response.content)
result = json.loads(response.content)
self.assertEqual(result['is_paused'], True)
self.assertEqual(result['pause_reason'], json_data['pause_reason'])
self.assertIn('hostname', result)
self.assertEqual(result['hostname'], self.node2.hostname)