def test_failed_has_link(self):
self.link_patcher.stop()
mock.patch(
'chroma_agent.lib.corosync.CorosyncRingInterface.__getattr__',
return_value=False).start()
import errno
def boom(*args):
# EMULTIHOP is what gets raised with IB interfaces
raise IOError(errno.EMULTIHOP)
mock.patch('fcntl.ioctl', side_effect=boom).start()
from chroma_agent.lib.corosync import get_ring0
iface = get_ring0()
# add shell commands to be expected
self.add_commands(
CommandCaptureCommand(
('/sbin/ip', 'link', 'set', 'dev', iface.name, 'up')),
CommandCaptureCommand(
('/sbin/ip', 'link', 'set', 'dev', iface.name, 'down')))
self.assertFalse(iface.has_link)
self.assertRanAllCommandsInOrder()
def test_device_node_versions(self):
for scsi_id in ["/sbin/scsi_id", "/lib/udev/scsi_id"]:
# Check runs with correct scsi_id
self.existing_files = [scsi_id]
self.reset_command_capture()
self.add_commands(
CommandCaptureCommand(
((scsi_id, '-g', '-p', '0x80', '/dev/blop'))),
CommandCaptureCommand(
((scsi_id, '-g', '-p', '0x83', '/dev/blop'))))
result = self.block_devices._device_node(1, "/dev/blop", 1, None,
'1234')
self.assertRanAllCommandsInOrder()
self.assertEqual(
result, {
'parent': None,
'major_minor': 1,
'serial_83': '',
'serial_80': '',
'path': '/dev/blop',
'filesystem_type': None,
'partition_number': '1234',
'size': 1
})
def test_close_port(self):
# test closing a port, commands should be issued in order and the rule should be
# removed from 'rules' member of FirewallControl class instance
# add rule object so we can test it is removed when we issue the remove() command
self.test_firewall.rules.append(self.example_port_rule)
self.assertEqual(len(self.test_firewall.rules), 1)
self.add_commands(
CommandCaptureCommand(
('/usr/bin/firewall-cmd',
'--remove-port=%s/%s' % (self.port, self.proto))),
CommandCaptureCommand(
('/usr/bin/firewall-cmd',
'--remove-port=%s/%s' % (self.port, self.proto),
'--permanent')))
response = self.test_firewall.remove_rule(self.port,
self.proto,
self.desc,
persist=True)
# None return value indicates success
self.assertEqual(response, None)
# rule should have been removed from list
self.assertEqual(len(self.test_firewall.rules), 0)
self.assertRanAllCommandsInOrder()
def test_service_register_listener_and_receive_success(self):
self.add_commands(
CommandCaptureCommand(('/sbin/service', 'test_service', 'start'),
executions_remaining=1),
CommandCaptureCommand(('/sbin/service', 'test_service', 'status'),
executions_remaining=1),
CommandCaptureCommand(('/sbin/service', 'test_service', 'stop'),
executions_remaining=1),
CommandCaptureCommand(('/sbin/service', 'test_service', 'status'),
rc=1,
executions_remaining=1))
self.received_codes = []
ServiceControl.register_listener('test_service', self.example_func)
response = self.test_service.start(retry_time=0.1, validate_time=0.1)
self.assertEqual(response, None)
self.assertListEqual(self.received_codes,
[('test_service', 'SERVICESTARTING'),
('test_service', 'SERVICESTARTED')])
self.received_codes = []
response = self.test_service.stop(retry_time=0.1, validate_time=0.1)
self.assertEqual(response, None)
self.assertListEqual(self.received_codes,
[('test_service', 'SERVICESTOPPING'),
('test_service', 'SERVICESTOPPED')])
def test_open_address(self):
# test opening all ports on a specific address, commands should be issued in order and the
# rule should be recorded in 'rules' member of FirewallControl class instance
self.assertEqual(len(self.test_firewall.rules), 0)
self.add_commands(
CommandCaptureCommand(('service', 'iptables', 'status'),
stdout=self.chain_output),
CommandCaptureCommand(
('/sbin/iptables', '-I', 'INPUT', '4', '-m', 'state',
'--state', 'NEW', '-m', self.proto, '-p', self.proto, '-d',
self.address, '-j', 'ACCEPT')))
response = self.test_firewall.add_rule(0,
self.proto,
self.desc,
persist=False,
address=self.address)
# class instance should have record of added rule
self.assertEqual(len(self.test_firewall.rules), 1)
self.assertEqual(self.test_firewall.rules[0],
self.example_address_rule)
# None return value indicates success
self.assertEqual(response, None)
self.assertRanAllCommandsInOrder()
def test_mkfs_no_mkfsoptions(self):
"""
Check --mkfsoptions are added to list of parameters provided to mkfs.lustre command when some parameters have
been provided.
"""
self.add_commands(
CommandCaptureCommand(
('zpool', 'set', 'failmode=panic', 'zpool1')),
CommandCaptureCommand(
('mkfs.lustre', '--mgs', '--backfstype=zfs',
'--mkfsoptions="mountpoint=none"', 'zpool1/MGS')))
mkfs_options = ['--mgs', '--backfstype=zfs']
self.assertEqual(
self.filesystem.mkfs('MGS', mkfs_options), {
'uuid': '123456789123',
'filesystem_type': 'zfs',
'inode_size': None,
'inode_count': None
})
self.assertRanAllCommandsInOrder()
self.uuid_prop_mock.assert_called_once()
self.type_prop_mock.assert_called_once()
def test_mkfs_no_mountpoint_option(self):
"""
Check mountpoint is added to --mkfsoptions when parameter has been provided without mountpoint property
assignment.
"""
self.add_commands(
CommandCaptureCommand(
('zpool', 'set', 'failmode=panic', 'zpool1')),
CommandCaptureCommand(
('mkfs.lustre', '--mgs', '--backfstype=zfs',
'--mkfsoptions="utf8only=on -o mountpoint=none"',
'zpool1/MGS')))
mkfs_options = [
'--mgs', '--backfstype=zfs', '--mkfsoptions="utf8only=on"'
]
self.assertEqual(
self.filesystem.mkfs('MGS', mkfs_options), {
'uuid': '123456789123',
'filesystem_type': 'zfs',
'inode_size': None,
'inode_count': None
})
self.assertRanAllCommandsInOrder()
self.uuid_prop_mock.assert_called_once()
self.type_prop_mock.assert_called_once()
def test_register_before_instance(self):
ServiceControl.register_listener('test_service_2', self.example_func)
test_service_2 = ServiceControl.create('test_service_2')
self.add_commands(
CommandCaptureCommand(('/sbin/service', 'test_service_2', 'start'),
executions_remaining=1),
CommandCaptureCommand(
('/sbin/service', 'test_service_2', 'status'),
executions_remaining=1),
CommandCaptureCommand(('/sbin/service', 'test_service_2', 'stop'),
executions_remaining=1),
CommandCaptureCommand(
('/sbin/service', 'test_service_2', 'status'),
rc=1,
executions_remaining=1))
self.received_codes = []
response = test_service_2.start(retry_time=0.1, validate_time=0.1)
self.assertEqual(response, None)
self.assertListEqual(self.received_codes,
[('test_service_2', 'SERVICESTARTING'),
('test_service_2', 'SERVICESTARTED')])
self.received_codes = []
ServiceControl.unregister_listener('test_service_2', self.example_func)
response = test_service_2.stop(retry_time=0.1, validate_time=0.1)
self.assertEqual(response, None)
self.assertListEqual(self.received_codes, [])
def test_service_register_listener_and_receive_fail_validation(self):
self.add_commands(
CommandCaptureCommand(('/sbin/service', 'test_service', 'start')),
CommandCaptureCommand(('/sbin/service', 'test_service', 'status'),
rc=1))
self.received_codes = []
ServiceControl.register_listener('test_service', self.example_func)
response = self.test_service.start(retry_time=0.1, validate_time=0.1)
self.assertEqual(
response,
'Service test_service is not running after being started')
self.assertListEqual(self.received_codes,
[('test_service', 'SERVICESTARTING'),
('test_service', 'SERVICESTARTERROR')])
# reset so we can test both stop and start fail validation within same test
self.reset_command_capture()
self.add_commands(
CommandCaptureCommand(('/sbin/service', 'test_service', 'stop')),
CommandCaptureCommand(('/sbin/service', 'test_service', 'status'),
rc=0))
self.received_codes = []
response = self.test_service.stop(retry_time=0.1, validate_time=0.1)
self.assertEqual(
response,
'Service test_service is still running after being stopped')
self.assertListEqual(self.received_codes,
[('test_service', 'SERVICESTOPPING'),
('test_service', 'SERVICESTOPERROR')])
def test_start_should_be_idempotent(self):
self.single_commands(
CommandCaptureCommand(
('/sbin/service', 'chroma-copytool-monitor-%s' % self.ct_id,
'status')),
CommandCaptureCommand(
('/sbin/service', 'chroma-copytool-monitor-%s' % self.ct_id,
'stop')),
CommandCaptureCommand(
('/sbin/service', 'chroma-copytool-monitor-%s' % self.ct_id,
'status'),
rc=1),
CommandCaptureCommand(
('/sbin/service', 'chroma-copytool-monitor-%s' % self.ct_id,
'start')),
CommandCaptureCommand(
('/sbin/service', 'chroma-copytool-monitor-%s' % self.ct_id,
'status')),
CommandCaptureCommand(
('/sbin/service', 'chroma-copytool-%s' % self.ct_id,
'status')),
CommandCaptureCommand(
('/sbin/service', 'chroma-copytool-%s' % self.ct_id, 'stop')),
CommandCaptureCommand(
('/sbin/service', 'chroma-copytool-%s' % self.ct_id, 'status'),
rc=1),
CommandCaptureCommand(
('/sbin/service', 'chroma-copytool-%s' % self.ct_id, 'start')),
CommandCaptureCommand(
('/sbin/service', 'chroma-copytool-%s' % self.ct_id,
'status')))
self.assertAgentOK(start_monitored_copytool(self.ct_id))
self.assertRanAllCommandsInOrder()
def test_failnode(self):
self.add_commands(CommandCaptureCommand(("sync", ), 0, "", ""),
CommandCaptureCommand(("sync", ), 0, "", ""),
CommandCaptureCommand(("init", "0"), 0, "", ""))
fail_node([])
self.assertRanAllCommandsInOrder()
def test_close_address(self):
# test remove_rule with address removes relevant rule from the class instance 'rules' list,
# also test correct shell command is issued
# add rule object so we can test it is removed when we issue the remove() command
self.test_firewall.rules.append(self.example_address_rule)
self.assertEqual(len(self.test_firewall.rules), 1)
self.add_commands(
CommandCaptureCommand(('service', 'iptables', 'status'),
stdout=self.chain_output),
CommandCaptureCommand(
('/sbin/iptables', '-D', 'INPUT', '-m', 'state', '--state',
'NEW', '-m', self.proto, '-p', self.proto, '-d', self.address,
'-j', 'ACCEPT')))
response = self.test_firewall.remove_rule(0,
self.proto,
self.desc,
persist=False,
address=self.address)
# None return value indicates success
self.assertEqual(response, None)
# rule should have been removed from list
self.assertEqual(len(self.test_firewall.rules), 0)
self.assertRanAllCommandsInOrder()
def test_resolve_devices_multiple_matches(self):
"""
Check when zpool contains device basenames which are a substring of another, the listed partitions are
resolved to the correct devices
"""
self.add_commands(
CommandCaptureCommand(
("zpool", "list", "-PHv", "-o", "name", "zfsPool1"),
stdout="""\
zfsPool1
/dev/a/dev1111 9.94G 228K 9.94G - 0% 0%
/dev/b/dev111 9.94G 228K 9.94G - 0% 0%
/dev/c/dev11 9.94G 228K 9.94G - 0% 0%
/dev/d/dev1 9.94G 228K 9.94G - 0% 0%
/dev/disk/by-id2/md0-0011 9.94G 228K 9.94G - 0% 0%
/dev/disk/by-id1/md0-00111 9.94G 228K 9.94G - 0% 0%\n"""
),
CommandCaptureCommand(
("zpool", "list", "-Hv", "-o", "name", "zfsPool1"),
stdout="""\
zfsPool1
dev 9.94G 228K 9.94G - 0% 0%
dev1 9.94G 228K 9.94G - 0% 0%
dev11 9.94G 228K 9.94G - 0% 0%
dev111 9.94G 228K 9.94G - 0% 0%
md0-0011 9.94G 228K 9.94G - 0% 0%
md0-001 9.94G 228K 9.94G - 0% 0%\n"""))
self.assertItemsEqual(self._get_zfs_devices('zfsPool1'), [
'/dev/disk/by-id1/md0-0011', '/dev/disk/by-id2/md0-001',
'/dev/d/dev', '/dev/c/dev1', '/dev/b/dev11', '/dev/a/dev111'
])
self.assertRanAllCommandsInOrder()
def test_lvm2_not_installed(self):
self.add_commands(CommandCaptureCommand(('vgs', '--units', 'b', '--noheadings', '-o', 'vg_name,vg_uuid,vg_size'),
rc=self.CommandNotFound),
CommandCaptureCommand(('dmsetup', 'table'),
stdout="No devices found"))
dm_setup_table = DmsetupTable({})
self.assertEqual(dm_setup_table.vgs, {})
self.assertEqual(dm_setup_table.lvs, {})
self.assertRanAllCommandsInOrder()
self.reset_command_capture()
self.add_commands(CommandCaptureCommand(('vgs', '--units', 'b', '--noheadings', '-o', 'vg_name,vg_uuid,vg_size'),
rc=self.CommandNotFound),
CommandCaptureCommand(('lvs', '--units', 'b', '--noheadings', '-o', 'lv_name,lv_uuid,lv_size,lv_path', 'bob'),
rc=self.CommandNotFound))
for vgs in dm_setup_table._get_vgs():
self.assertFalse(True, '_get_vgs did not deal with missing vgs')
for lvs in dm_setup_table._get_lvs('bob'):
self.assertFalse(True, '_get_lvs did not deal with missing lvg')
self.assertRanAllCommandsInOrder()
def test_resolve_devices_duplicate_device_basenames(self):
"""
Check when zpool contains duplicate device basenames the listed partitions are resolved to the correct devices
"""
self.add_commands(
CommandCaptureCommand(
("zpool", "list", "-PHv", "-o", "name", "zfsPool1"),
stdout="""\
zfsPool1
/dev/sde1 9.94G 228K 9.94G - 0% 0%
/dev/disk/by-id/scsi-SCSI_DISK_1a/scsi-SCSI_DISK_1-part1 9.94G 228K 9.94G - 0% 0%
/dev/disk/by-id/scsi-SCSI_DISK_1b/scsi-SCSI_DISK_1-part1 9.94G 228K 9.94G - 0% 0%\n"""
),
CommandCaptureCommand(
("zpool", "list", "-Hv", "-o", "name", "zfsPool1"),
stdout="""\
zfsPool1
sde 9.94G 228K 9.94G - 0% 0%
scsi-SCSI_DISK_1 9.94G 228K 9.94G - 0% 0%
scsi-SCSI_DISK_1 9.94G 228K 9.94G - 0% 0%\n"""))
self.assertItemsEqual(self._get_zfs_devices('zfsPool1'), [
'/dev/disk/by-id/scsi-SCSI_DISK_1a/scsi-SCSI_DISK_1',
'/dev/disk/by-id/scsi-SCSI_DISK_1b/scsi-SCSI_DISK_1', '/dev/sde'
])
self.assertRanAllCommandsInOrder()
def test_corosync_causes_session_to_reestablish(self):
"""Connecting to crm_mon fails
Could not establish cib_ro connection: Connection refused rc=107
This represents any case that is unknown; which is rc: 0, 10. In this
case, the response should be treated like an SPI boundary. This means
log it and re-raise. But, raising is what causes the trashing, so
instead just return None
see also: https://jira.hpdd.intel.com/browse/HYD-1914
"""
# Simulate crm_mon returning an unexpected error code
self.add_commands(
CommandCaptureCommand(CMD, rc=107),
CommandCaptureCommand(('service', 'corosync', 'status'), rc=1),
CommandCaptureCommand(('service', 'pacemaker', 'status'), rc=1))
plugin = CorosyncPlugin(None)
result_dict = plugin.start_session()
self.assertEqual(result_dict["state"]["corosync"], 'stopped')
self.assertEqual(result_dict["state"]["pacemaker"], 'stopped')
self.assertEqual(result_dict["crm_info"], None)
self.assertRanAllCommandsInOrder()
def test_mkfs_multiple_mkfsoptions_with_mountpoint(self):
"""
Check mountpoint setting is overwritten in --mkfsoptions when parameter has been provided with
mountpoint property assignment and multiple other options.
"""
self.add_commands(
CommandCaptureCommand(
('zpool', 'set', 'failmode=panic', 'zpool1')),
CommandCaptureCommand((
'mkfs.lustre', '--mgs', '--backfstype=zfs',
'--mkfsoptions="listsnapshots=on -o utf8only=on -o mountpoint=none"',
'zpool1/MGS')))
mkfs_options = [
'--mgs', '--backfstype=zfs',
'--mkfsoptions="listsnapshots=on -o mountpoint=zpool1/MGS -o utf8only=on"'
]
self.assertEqual(
self.filesystem.mkfs('MGS', mkfs_options), {
'uuid': '123456789123',
'filesystem_type': 'zfs',
'inode_size': None,
'inode_count': None
})
self.assertRanAllCommandsInOrder()
self.uuid_prop_mock.assert_called_once()
self.type_prop_mock.assert_called_once()
def test_service_start_success(self):
self.add_commands(
CommandCaptureCommand(('/sbin/service', 'test_service', 'start')),
CommandCaptureCommand(('/sbin/service', 'test_service', 'status')))
response = self.test_service.start(retry_time=0.1, validate_time=0.1)
self.assertEqual(response, None)
self.assertEqual(self.commands_ran_count, 2)
def test_initialize_modules(self):
self.patch_init_modules.stop()
self.add_commands(CommandCaptureCommand(('modprobe', 'osd_zfs')),
CommandCaptureCommand(('modprobe', 'zfs')))
self.blockdevice._initialize_modules()
self.assertTrue(self.blockdevice._modules_initialized)
self.assertRanAllCommandsInOrder()
def test_uuid(self):
self.add_commands(
CommandCaptureCommand(('zpool', 'list', '-H', '-o', 'name'),
stdout=self.blockdevice._device_path),
CommandCaptureCommand(('zfs', 'get', '-H', '-o', 'value', 'guid',
self.blockdevice._device_path),
stdout='169883839435093209\n'))
self.assertEqual('169883839435093209', self.blockdevice.uuid)
self.assertRanAllCommandsInOrder()
def _add_import_commands(self, name='Boris'):
self.add_commands(
CommandCaptureCommand(('zpool', 'list', '-H', '-o', 'name'),
stdout=name))
if name != self.zpool_name:
self.add_commands(
CommandCaptureCommand(
('zpool', 'import', '-f', '-N', '-o', 'readonly=on', '-o',
'cachefile=none', self.zpool_name)))
def test_set_profile_fail(self):
# Three times because yum will try three times.
self.add_commands(CommandCaptureCommand(('yum', 'install', '-y', '--enablerepo=iml-agent', 'chroma-agent-management'), rc=1, stdout="Bad command stdout", stderr="Bad command stderr"),
CommandCaptureCommand(('yum', 'install', '-y', '--enablerepo=iml-agent', 'chroma-agent-management'), rc=1, stdout="Bad command stdout", stderr="Bad command stderr"),
CommandCaptureCommand(('yum', 'install', '-y', '--enablerepo=iml-agent', 'chroma-agent-management'), rc=1, stdout="Bad command stdout", stderr="Bad command stderr"))
config.update('settings', 'profile', {'managed': False})
# Go from managed = False to managed = True, but it will fail.
self.assertEqual(agent_updates.update_profile({'managed': True}), agent_error('Unable to set profile because yum returned Bad command stdout'))
self.assertRanAllCommandsInOrder()
def test_initialize_modules(self):
self.patch_init_modules.stop()
self.add_commands(
CommandCaptureCommand(('modprobe', 'osd_ldiskfs'), rc=1),
CommandCaptureCommand(('modprobe', 'ldiskfs')))
self.filesystem._initialize_modules()
self.assertTrue(self.filesystem._modules_initialized)
self.assertRanAllCommandsInOrder()
def test_service_stop_success(self):
self.add_commands(
CommandCaptureCommand(('/sbin/service', 'test_service', 'stop'),
executions_remaining=1),
CommandCaptureCommand(('/sbin/service', 'test_service', 'status'),
rc=1,
executions_remaining=1))
response = self.test_service.stop(retry_time=0.1, validate_time=0.1)
self.assertEqual(response, None)
self.assertEqual(self.commands_ran_count, 2)
def test_service_start_validate_fail(self):
self.add_commands(
CommandCaptureCommand(('/sbin/service', 'test_service', 'start')),
CommandCaptureCommand(('/sbin/service', 'test_service', 'status'),
rc=1))
response = self.test_service.start(retry_time=0.1, validate_time=0.1)
self.assertEqual(
response,
'Service test_service is not running after being started')
self.assertEqual(self.commands_ran_count, 12)
def test_service_register_listener_and_receive_fail_initial(self):
self.add_commands(
CommandCaptureCommand(('/sbin/service', 'test_service', 'start'),
rc=1,
stderr='Service Failed To Start',
executions_remaining=1),
CommandCaptureCommand(('/sbin/service', 'test_service', 'start'),
rc=1,
stderr='Service Failed To Start',
executions_remaining=1))
self.received_codes = []
ServiceControl.register_listener('test_service', self.example_func)
response = self.test_service.start(retry_time=0.1,
validate_time=0.1,
retry_count=1)
self.assertEqual(
response,
"Error (1) running '/sbin/service test_service start': '' 'Service Failed To Start'"
)
self.assertListEqual(self.received_codes,
[('test_service', 'SERVICESTARTING'),
('test_service', 'SERVICESTARTERROR')])
# reset so we can test both stop and start fail validation within same test
self.reset_command_capture()
self.add_commands(
CommandCaptureCommand(('/sbin/service', 'test_service', 'stop'),
rc=1,
stderr='Service Failed To Stop',
executions_remaining=1),
CommandCaptureCommand(('/sbin/service', 'test_service', 'stop'),
rc=1,
stderr='Service Failed To Stop',
executions_remaining=1))
self.received_codes = []
response = self.test_service.stop(retry_time=0.1,
validate_time=0.1,
retry_count=1)
self.assertEqual(
response,
"Error (1) running '/sbin/service test_service stop': '' 'Service Failed To Stop'"
)
self.assertListEqual(self.received_codes,
[('test_service', 'SERVICESTOPPING'),
('test_service', 'SERVICESTOPERROR')])
def test_stop_should_be_idempotent2(self):
self.single_commands(
CommandCaptureCommand(
('/sbin/service', 'chroma-copytool-monitor-%s' % self.ct_id,
'status'),
rc=1),
CommandCaptureCommand(
('/sbin/service', 'chroma-copytool-%s' % self.ct_id, 'status'),
rc=1))
with mock.patch('os.path.exists', return_value=True):
self.assertAgentOK(stop_monitored_copytool(self.ct_id))
self.assertRanAllCommandsInOrder()
def test_service_disable_success(self):
self.add_commands(
CommandCaptureCommand(('/sbin/chkconfig', 'test_service', 'off'),
executions_remaining=1),
CommandCaptureCommand(('/sbin/chkconfig', 'test_service'),
rc=1,
executions_remaining=1))
response = self.test_service.disable()
self.assertEqual(response, None)
response = self.test_service.enabled
self.assertEqual(response, False)
self.assertRanAllCommandsInOrder()
def test_mount_fail_second(self):
""" Test when initial mount fails and the retry fails, exception is raised """
self.add_commands(
CommandCaptureCommand(
('mount', '-t', 'lustre', '/dev/sda1', '/mnt/OST0000'),
rc=5,
executions_remaining=1),
CommandCaptureCommand(
('mount', '-t', 'lustre', '/dev/sda1', '/mnt/OST0000'),
rc=5,
executions_remaining=1))
self.assertRaises(RuntimeError, self.filesystem.mount, '/mnt/OST0000')
self.assertRanAllCommandsInOrder()
def test_mount_fail_initial(self):
""" Test when initial mount fails, retry succeeds and result returned """
self.add_commands(
CommandCaptureCommand(
('mount', '-t', 'lustre', '/dev/sda1', '/mnt/OST0000'),
rc=5,
executions_remaining=1),
CommandCaptureCommand(
('mount', '-t', 'lustre', '/dev/sda1', '/mnt/OST0000'),
rc=0,
executions_remaining=1))
self.filesystem.mount('/mnt/OST0000')
self.assertRanAllCommandsInOrder()
