def main():
"""Check that swift owns its relevant files and directories."""
# Check /etc/swift
config_results = []
not_swift_owned_config(config_results)
empty_files(config_results)
# Check files under /srv/node
data_results = []
not_swift_owned_data(data_results)
# Generate metrics. Use the "reason" field from the *first* failure
# in each category to populate the msg field for Severity.fail. If there
# are several failures, the user will have to resolve them one by one.
metrics = []
if config_results:
metrics.append(MetricData.single(__name__ + '.config', Severity.fail,
message='{message}',
msgkeys=config_results[0]))
else:
metrics.append(MetricData.single(__name__ + '.config', Severity.ok,
message='OK'))
if data_results:
metrics.append(MetricData.single(__name__ + '.data', Severity.fail,
message='{message}',
msgkeys=data_results[0]))
else:
metrics.append(MetricData.single(__name__ + '.data', Severity.ok,
message='OK'))
return metrics
def test_get_logical_drive_info_failures(self):
tests = [(LOGICAL_DRIVE_LUN_FAIL, "lun_status"), (LOGICAL_DRIVE_CACHE_FAIL, "cache_status")]
test_slot = "1"
for test_data, failed_component in tests:
mock_command = mock.Mock()
mock_command.return_value = CommandResult(0, test_data)
with mock.patch("swiftlm.hp_hardware.hpssacli.run_cmd", mock_command):
actual = hpssacli.get_logical_drive_info(test_slot)
expected_lun = MetricData.single(
hpssacli.__name__ + ".logical_drive",
Severity.ok,
"OK",
{
"component": "logical_drive",
"logical_drive": "L",
"sub_component": "lun_status",
"caching": "Enabled",
"status": "OK",
},
)
expected_cache = MetricData.single(
hpssacli.__name__ + ".logical_drive",
Severity.ok,
"OK",
{
"component": "logical_drive",
"logical_drive": "L",
"sub_component": "cache_status",
"caching": "Enabled",
"status": "OK",
},
)
if expected_lun["sub_component"] == failed_component:
expected_lun.value = Severity.fail
expected_lun["status"] = "Fail"
expected_lun._message = hpssacli.BASE_RESULT.messages["l_drive"]
if expected_cache["sub_component"] == failed_component:
expected_lun["caching"] = "Disabled"
actual = self.check_metrics(expected_lun, actual)
if expected_cache["sub_component"] == failed_component:
expected_cache.value = Severity.fail
expected_cache["caching"] = "Disabled"
expected_cache._message = hpssacli.BASE_RESULT.messages["l_cache"]
if expected_lun["sub_component"] == failed_component:
expected_cache["status"] = "Fail"
actual = self.check_metrics(expected_cache, actual)
self.assertFalse(actual, "Got more metrics than expected")
def test_get_logical_drive_info_failures(self):
tests = [
(LOGICAL_DRIVE_LUN_FAIL, 'lun_status'),
(LOGICAL_DRIVE_CACHE_FAIL, 'cache_status')
]
test_slot = "1"
for test_data, failed_component in tests:
mock_command = mock.Mock()
mock_command.return_value = CommandResult(0, test_data)
with mock.patch('swiftlm.hp_hardware.hpssacli.run_cmd',
mock_command):
actual = hpssacli.get_logical_drive_info(test_slot)
expected_lun = MetricData.single(
hpssacli.__name__ + '.logical_drive',
Severity.ok, 'OK',
{'component': 'logical_drive',
'logical_drive': 'L',
'sub_component': 'lun_status',
'caching': 'Enabled',
'status': "OK"})
expected_cache = MetricData.single(
hpssacli.__name__ + '.logical_drive',
Severity.ok, 'OK',
{'component': 'logical_drive',
'logical_drive': 'L',
'sub_component': 'cache_status',
'caching': 'Enabled',
'status': "OK"})
if expected_lun['sub_component'] == failed_component:
expected_lun.value = Severity.fail
expected_lun['status'] = 'Fail'
expected_lun._message = (hpssacli.BASE_RESULT.messages
['l_drive'])
if expected_cache['sub_component'] == failed_component:
expected_lun['caching'] = 'Disabled'
actual = self.check_metrics(expected_lun, actual)
if expected_cache['sub_component'] == failed_component:
expected_cache.value = Severity.fail
expected_cache['caching'] = 'Disabled'
expected_cache._message = (hpssacli.BASE_RESULT.messages
['l_cache'])
if expected_lun['sub_component'] == failed_component:
expected_cache['status'] = 'Fail'
actual = self.check_metrics(expected_cache, actual)
self.assertFalse(actual, 'Got more metrics than expected')
def test_get_logical_drive_info(self):
# Test that normal output and bugged output give exactly
# the same results
mock_command = mock.Mock()
test_slot = "1"
mock_command.return_value = CommandResult(0, LOGICAL_DRIVE_DATA)
with mock.patch("swiftlm.hp_hardware.hpssacli.run_cmd", mock_command):
data_1 = hpssacli.get_logical_drive_info(test_slot)
self.assertIsInstance(data_1, list)
self.assertTrue(len(data_1), 3)
mock_command = mock.Mock()
mock_command.return_value = CommandResult(0, LOGICAL_DRIVE_DATA_BUGGED)
with mock.patch("swiftlm.hp_hardware.hpssacli.run_cmd", mock_command):
data_2 = hpssacli.get_logical_drive_info(test_slot)
self.assertIsInstance(data_2, list)
self.assertTrue(len(data_2), 3)
# Check the data is the same for both
for d in data_1:
data_2 = self.check_metrics(d, data_2)
# Check data is as expected.
expected_lun = MetricData.single(
hpssacli.__name__ + ".logical_drive",
Severity.ok,
"OK",
{
"component": "logical_drive",
"sub_component": "lun_status",
"status": "OK",
"logical_drive": "L",
"caching": "Enabled",
},
)
data_1 = self.check_metrics(expected_lun, data_1)
expected_cache = MetricData.single(
hpssacli.__name__ + ".logical_drive",
Severity.ok,
"OK",
{
"component": "logical_drive",
"sub_component": "cache_status",
"status": "OK",
"logical_drive": "L",
"caching": "Enabled",
},
)
data_1 = self.check_metrics(expected_cache, data_1)
self.assertFalse(data_1, "Got more metrics than expected with" "LOGICAL_DRIVE_DATA")
self.assertFalse(data_2, "Got more metrics than expected with" "LOGICAL_DRIVE_DATA_BUGGED")
def main():
args = parse_args()
metrics = []
for func in args.selected:
try:
r = func()
if isinstance(r, list) and r and isinstance(r[0], MetricData):
metrics.extend([result.metric() for result in r])
elif isinstance(r, MetricData):
metrics.append(r.metric())
except: # noqa
t, v, tb = sys.exc_info()
backtrace = ' '.join(traceback.format_exception(t, v, tb))
r = MetricData.single('check.failure',
Severity.fail,
'{check} failed with: {error}',
{'check': str(func),
'error': backtrace.replace('\n', ' '),
'component': 'swiftlm-scan',
'service': 'object-storage'})
metrics.append(r.metric())
pass
FORMATS[args.format](metrics, args.pretty)
def main():
args = parse_args()
metrics = []
for func in args.selected:
try:
r = func()
if isinstance(r, list) and r and isinstance(r[0], MetricData):
metrics.extend([result.metric() for result in r])
elif isinstance(r, MetricData):
metrics.append(r.metric())
except: # noqa
t, v, tb = sys.exc_info()
backtrace = ' '.join(traceback.format_exception(t, v, tb))
r = MetricData.single(
'check.failure', Severity.fail, '{check} failed with: {error}',
{
'check': str(func),
'error': backtrace.replace('\n', ' '),
'component': 'swiftlm-scan',
'service': 'object-storage'
})
metrics.append(r.metric())
pass
FORMATS[args.format](metrics, args.pretty)
def test_metrics_dirs_not_exist(self):
expected = [
MetricData.single('swiftlm.swift.file_ownership.config',
Severity.fail,
message='dummy'),
MetricData.single('swiftlm.swift.file_ownership.data',
Severity.fail,
message='dummy')
]
with mock.patch('swiftlm.swift.file_ownership.server_type',
lambda x: x == ServerType.object):
with mock.patch('pwd.getpwuid') as mock_pwuid:
mock_pwuid.return_value = mock.Mock(pw_name='swift')
results = FO.main()
self.assertEqual(len(expected), 2)
same = metrics_are_similar(results, expected)
self.assertEqual('', same, msg=same)
def test_get_logical_drive_info(self):
# Test that normal output and bugged output give exactly
# the same results
mock_command = mock.Mock()
test_slot = "1"
mock_command.return_value = CommandResult(0, LOGICAL_DRIVE_DATA)
with mock.patch('swiftlm.hp_hardware.hpssacli.run_cmd',
mock_command):
data_1 = hpssacli.get_logical_drive_info(test_slot)
self.assertIsInstance(data_1, list)
self.assertTrue(len(data_1), 3)
mock_command = mock.Mock()
mock_command.return_value = CommandResult(0, LOGICAL_DRIVE_DATA_BUGGED)
with mock.patch('swiftlm.hp_hardware.hpssacli.run_cmd',
mock_command):
data_2 = hpssacli.get_logical_drive_info(test_slot)
self.assertIsInstance(data_2, list)
self.assertTrue(len(data_2), 3)
# Check the data is the same for both
for d in data_1:
data_2 = self.check_metrics(d, data_2)
# Check data is as expected.
expected_lun = MetricData.single(
hpssacli.__name__ + '.logical_drive',
Severity.ok, 'OK',
{'component': 'logical_drive', 'sub_component': 'lun_status',
'status': "OK", 'logical_drive': 'L', 'caching': 'Enabled'})
data_1 = self.check_metrics(expected_lun, data_1)
expected_cache = MetricData.single(
hpssacli.__name__ + '.logical_drive',
Severity.ok, 'OK',
{'component': 'logical_drive', 'sub_component': 'cache_status',
'status': "OK", 'logical_drive': 'L', 'caching': 'Enabled'})
data_1 = self.check_metrics(expected_cache, data_1)
self.assertFalse(data_1, 'Got more metrics than expected with'
'LOGICAL_DRIVE_DATA')
self.assertFalse(data_2, 'Got more metrics than expected with'
'LOGICAL_DRIVE_DATA_BUGGED')
def test_details_ok(self):
mock_command = Mock()
mock_command.return_value = CommandResult(0, 'stratum=1,offset=2,')
with patch('swiftlm.systems.ntp.run_cmd', mock_command):
with patch('swiftlm.systems.ntp.check_status', lambda: []):
actual = ntp.main()
self.assertIsInstance(actual, list)
self.assertEqual(len(actual), 2)
actual = [a.metric() for a in actual]
expected = [
MetricData.single(ntp.__name__+'.stratum', '1', ''),
MetricData.single(ntp.__name__+'.offset', '2', '')
]
for e in expected:
self.assertIn(e.metric(), actual)
def check_rsync():
metrics = []
rsync_running, ip_port_match = get_rsync_bind_ip()
if not rsync_running:
dimensions = get_base_dimensions()
dimensions["component"] = "rsync"
metrics.append(
MetricData.single('swiftlm.swift.swift_services',
Severity.fail,
message='rsync is not running',
dimensions=dimensions))
return metrics
else:
dimensions = get_base_dimensions()
dimensions["component"] = "rsync"
metrics.append(
MetricData.single('swiftlm.swift.swift_services',
Severity.ok,
message='rsync is running',
dimensions=dimensions))
if not ip_port_match:
dimensions = get_base_dimensions()
dimensions["component"] = "rsync"
metrics.append(
MetricData.single(
'swiftlm.swift.swift_services.check_ip_port',
Severity.fail,
message='rsync is not listening on the correct ip or port',
dimensions=dimensions))
else:
dimensions = get_base_dimensions()
dimensions["component"] = "rsync"
metrics.append(
MetricData.single('swiftlm.swift.swift_services.check_ip_port',
Severity.ok,
message='OK',
dimensions=dimensions))
return metrics
def test_metrics_dirs_are_owned(self):
self._create_etc_file('rsyslog.conf', content='blah')
self._create_etc_file('rsyncd.conf', content='blah')
path1 = os.path.join(self.etc_dir, 'swift', 'object-server', '1')
path2 = os.path.join(self.srv_dir, 'node', '1')
os.makedirs(path1)
os.makedirs(path2)
expected = [
MetricData.single('swiftlm.swift.file_ownership.config',
Severity.ok,
message='OK'),
MetricData.single('swiftlm.swift.file_ownership.data',
Severity.ok,
message='OK')
]
with mock.patch('swiftlm.swift.file_ownership.server_type',
lambda x: x == ServerType.object):
with mock.patch('pwd.getpwuid') as mock_pwuid:
mock_pwuid.return_value = mock.Mock(pw_name='swift')
results = FO.main()
self.assertEqual(len(expected), 2)
same = metrics_are_same(results, expected)
self.assertEqual('', same, msg=same)
def test_load_avg(self):
mock_command = Mock()
mock_command.return_value = '2.15 1.81 1.69 2/1570 29660\n'
with patch('swiftlm.systems.system._get_proc_file', mock_command):
actual = system.get_load_average()
self.assertIsInstance(actual, list)
self.assertEqual(len(actual), 1)
r = actual[0]
self.assertIsInstance(r, MetricData)
expected = MetricData.single('load.host.val.five', value=1.81)
self.assertEqual(r, expected)
def test_status_ok(self):
mock_command = Mock()
mock_command.return_value = CommandResult(0, '')
with patch('swiftlm.systems.ntp.run_cmd', mock_command):
with patch('swiftlm.systems.ntp.check_details', lambda: []):
actual = ntp.main()
self.assertIsInstance(actual, list)
self.assertEqual(len(actual), 1)
r = actual[0]
self.assertIsInstance(r, MetricData)
expected = MetricData.single(ntp.__name__, Severity.ok,
ntp.BASE_RESULT.messages['ok'])
self.assertEqual(r, expected)
def test_diskusage(self, *args):
expected = [MetricData.single('swiftlm.diskusage.host.val.size',
532676608,
dimensions={'mount': 'na',
'service':
'object-storage'}),
MetricData.single('swiftlm.diskusage.host.val.used',
338939904,
dimensions={'mount': 'na',
'service':
'object-storage'}),
MetricData.single('swiftlm.diskusage.host.val.avail',
193736704,
dimensions={'mount': 'na',
'service':
'object-storage'}),
MetricData.single('swiftlm.diskusage.host.val.usage',
64.0,
dimensions={'mount': 'na',
'service':
'object-storage'}),
MetricData.single('swiftlm.diskusage.host.max.usage',
64.0,
dimensions={'service':
'object-storage'}),
MetricData.single('swiftlm.diskusage.host.min.usage',
64.0,
dimensions={'service':
'object-storage'}),
MetricData.single('swiftlm.diskusage.host.avg.usage',
64.0,
dimensions={'service':
'object-storage'})]
results = check_mounts.diskusage()
for result in results:
if result in expected:
expected.remove(result)
else:
self.assertEqual(True, False,
msg='Not expecting %s' % result.__repr__())
self.assertEqual(0, len(expected), msg='Missing: %s' % expected)
def test_get_physical_drive_info(self):
# List of tuples.
# t[0] = Data set that hpssacli should return
# t[1] = Tuple(Severity, Message, Status)
tests = [
(PHYSICAL_DRIVE_DATA, (Severity.ok, "OK", "OK")),
(PHYSICAL_DRIVE_STATUS_FAIL, (Severity.fail, hpssacli.BASE_RESULT.messages["physical_drive"], "FAIL")),
]
test_slot = "1"
for test_data, expected_metrics in tests:
mock_command = mock.Mock()
mock_command.return_value = CommandResult(0, test_data)
with mock.patch("swiftlm.hp_hardware.hpssacli.run_cmd", mock_command):
actual = hpssacli.get_physical_drive_info(test_slot)
self.assertIsInstance(actual, list)
self.assertTrue(len(actual), 1)
r = actual[0]
self.assertIsInstance(r, MetricData)
expected = MetricData.single(
hpssacli.__name__ + ".physical_drive",
expected_metrics[0], # Severity
expected_metrics[1], # Message
{
"status": expected_metrics[2], # Status
"serial": "YFJMHTZD",
"box": "1",
"bay": "1",
"component": "physical_drive",
"controller_slot": "1",
},
)
self.assertEqual(r, expected)
def test_details_fail(self):
mock_command = Mock()
mock_command.return_value = CommandResult(0, 'stratum=1,')
with patch('swiftlm.systems.ntp.run_cmd', mock_command):
with patch('swiftlm.systems.ntp.check_status', lambda: []):
actual = ntp.main()
self.assertIsInstance(actual, list)
self.assertEqual(len(actual), 2)
actual = [a.metric() for a in actual]
failed = CheckFailure.child()
failed.value = Severity.fail
failed['check'] = ntp.__name__ + '.offset'
failed['error'] = 'Output does not contain "offset"'
expected = [
failed,
MetricData.single(ntp.__name__+'.stratum', '1', ''),
]
for e in expected:
self.assertIn(e.metric(), actual)
def test_get_physical_drive_info(self):
# List of tuples.
# t[0] = Data set that hpssacli should return
# t[1] = Tuple(Severity, Message, Status)
tests = [
(PHYSICAL_DRIVE_DATA, (Severity.ok, 'OK', 'OK')),
(PHYSICAL_DRIVE_STATUS_FAIL, (
Severity.fail,
hpssacli.BASE_RESULT.messages['physical_drive'],
'FAIL'))
]
test_slot = "1"
for test_data, expected_metrics in tests:
mock_command = mock.Mock()
mock_command.return_value = CommandResult(0, test_data)
with mock.patch('swiftlm.hp_hardware.hpssacli.run_cmd',
mock_command):
actual = hpssacli.get_physical_drive_info(test_slot)
self.assertIsInstance(actual, list)
self.assertTrue(len(actual), 1)
r = actual[0]
self.assertIsInstance(r, MetricData)
expected = MetricData.single(
hpssacli.__name__ + '.physical_drive',
expected_metrics[0], # Severity
expected_metrics[1], # Message
{'status': expected_metrics[2], # Status
'serial': 'YFJMHTZD', 'box': '1', 'bay': '1',
'component': 'physical_drive', 'controller_slot': '1'})
self.assertEqual(r, expected)
def do_it(func, metric_name, slot_used):
# Test first failure condition.
# could be anything from hpssacli is missing to insufficent
# privileges
mock_command = mock.Mock()
mock_command.return_value = CommandResult(1, 'error')
with mock.patch('swiftlm.hp_hardware.hpssacli.run_cmd',
mock_command):
if slot_used == "N/A":
actual = func()
else:
actual = func(slot_used)
self.assertIsInstance(actual, list)
self.assertTrue(len(actual), 1)
r = actual[0]
if slot_used == "N/A":
expected = MetricData.single(
'check.failure', Severity.fail,
'{check} failed with: {error}',
{'check': hpssacli.__name__ + '.' + metric_name,
'error': 'error', 'component': 'swiftlm-scan'})
else:
expected = MetricData.single(
'check.failure', Severity.fail,
'{check} slot: {slot} failed with: {error}',
{'check': hpssacli.__name__ + '.' + metric_name,
'error': 'error', 'slot': slot_used,
'component': 'swiftlm-scan'})
self.assertEqual(r, expected)
# Test hpssacli providing no output.
mock_command = mock.Mock()
mock_command.return_value = CommandResult(0, '')
with mock.patch('swiftlm.hp_hardware.hpssacli.run_cmd',
mock_command):
if slot_used == "N/A":
actual = func()
else:
actual = func(slot_used)
self.assertIsInstance(actual, list)
self.assertTrue(len(actual), 1)
r = actual[0]
if slot_used == "N/A":
expected = MetricData.single(
'check.failure', Severity.fail,
'{check} failed with: {error}',
{'check': hpssacli.__name__ + '.' + metric_name,
'error': 'No usable output from hpssacli',
'component': 'swiftlm-scan'})
else:
expected = MetricData.single(
'check.failure', Severity.fail,
'{check} slot: {slot} failed with: {error}',
{'check': hpssacli.__name__ + '.' + metric_name,
'error': 'No usable output from hpssacli',
'slot': slot_used,
'component': 'swiftlm-scan'})
self.assertEqual(r, expected)
def do_it(func, metric_name, slot_used):
# Test first failure condition.
# could be anything from hpssacli is missing to insufficent
# privileges
mock_command = mock.Mock()
mock_command.return_value = CommandResult(1, "error")
with mock.patch("swiftlm.hp_hardware.hpssacli.run_cmd", mock_command):
if slot_used == "N/A":
actual = func()
else:
actual = func(slot_used)
self.assertIsInstance(actual, list)
self.assertTrue(len(actual), 1)
r = actual[0]
if slot_used == "N/A":
expected = MetricData.single(
"check.failure",
Severity.fail,
"{check} failed with: {error}",
{"check": hpssacli.__name__ + "." + metric_name, "error": "error", "component": "swiftlm-scan"},
)
else:
expected = MetricData.single(
"check.failure",
Severity.fail,
"{check} slot: {slot} failed with: {error}",
{
"check": hpssacli.__name__ + "." + metric_name,
"error": "error",
"slot": slot_used,
"component": "swiftlm-scan",
},
)
self.assertEqual(r, expected)
# Test hpssacli providing no output.
mock_command = mock.Mock()
mock_command.return_value = CommandResult(0, "")
with mock.patch("swiftlm.hp_hardware.hpssacli.run_cmd", mock_command):
if slot_used == "N/A":
actual = func()
else:
actual = func(slot_used)
self.assertIsInstance(actual, list)
self.assertTrue(len(actual), 1)
r = actual[0]
if slot_used == "N/A":
expected = MetricData.single(
"check.failure",
Severity.fail,
"{check} failed with: {error}",
{
"check": hpssacli.__name__ + "." + metric_name,
"error": "No usable output from hpssacli",
"component": "swiftlm-scan",
},
)
else:
expected = MetricData.single(
"check.failure",
Severity.fail,
"{check} slot: {slot} failed with: {error}",
{
"check": hpssacli.__name__ + "." + metric_name,
"error": "No usable output from hpssacli",
"slot": slot_used,
"component": "swiftlm-scan",
},
)
self.assertEqual(r, expected)
def get_logical_drive_info(slot, cache_check=True):
"""
array L
Logical Drive: 12
Size: 1.8 TB
Fault Tolerance: 0
Heads: 255
Sectors Per Track: 32
Cylinders: 65535
Strip Size: 256 KB
Full Stripe Size: 256 KB
Status: OK
Caching: Enabled
Unique Identifier: 600508B1001CEA938043498011A76404
Disk Name: /dev/sdl
Mount Points: /srv/node/disk11 1.8 TB Partition Number 2
OS Status: LOCKED
Logical Drive Label: AF3C73D8PACCR0M9VZ41S4QEB69
Drive Type: Data
LD Acceleration Method: Controller Cache
BUG:
It appears that the current build of hpssacli has a bug and outputs
Disk Name and Mount Points on the same line. We work around this by
checking for these specifically but that could fail if they change
"""
results = []
drive_result = BASE_RESULT.child()
drive_result.name += '.' + 'logical_drive'
rc = run_cmd(LOCK_FILE_COMMAND +
'hpssacli ctrl slot=%s ld all show detail' % slot)
if rc.exitcode != 0:
r = MetricData.single(
'check.failure', Severity.fail,
'{check} slot: {slot} failed with: {error}', {
'check': drive_result.name,
'slot': slot,
'error': str(rc.output),
'component': 'swiftlm-scan'
})
return [r]
# Remove blank lines and strip trailing/leading spaces for each line
lines = [l.strip() for l in rc.output.split('\n') if l.strip()]
if not lines:
r = MetricData.single(
'check.failure', Severity.fail,
'{check} slot: {slot} failed with: {error}', {
'check': drive_result.name,
'slot': slot,
'error': 'No usable output from hpssacli',
'component': 'swiftlm-scan'
})
return [r]
# First line should be the controller model and slot number.
# We already have this so remove it if it exists
if is_cont_heading(lines[0]):
lines = lines[1:]
drives = []
drive_info = {}
for line in lines:
# If we see two colons we have to assume that it is a bugged version
# of hpssacli and split them accordingly.
cc = line.count(':')
if cc == 2:
_, dn, mp = line.split(':')
drive_info['disk name'] = dn.strip().split()[0]
drive_info['mount points'] = mp.strip()
continue
# The Array # line may be useful in the future but does not follow
# the format of colon seperated infommation.
# It is also the only delimiter between drives. We create a new
# drive_info dict when we see it.
if line.startswith('array '):
if drive_info:
drives.append(drive_info)
drive_info = {}
drive_info['array'] = line.split()[1]
continue
k, v = line.split(':', 1)
k = k.strip().lower()
v = v.strip()
drive_info[k] = v
# Have to add the last drive.
if drive_info:
drives.append(drive_info)
for d in drives:
results.extend(check_logical_drive(d, drive_result, cache_check))
return results
def get_physical_drive_info(slot):
"""
Parses drive data from hpssacli in the form.
There are multiple drives in the output.
array A
physicaldrive 2C:1:1
Port: 2C
Box: 1
Bay: 1
Status: OK
Drive Type: Data Drive
Interface Type: SAS
Size: 2 TB
Native Block Size: 512
Rotational Speed: 7200
Firmware Revision: HPD3
Serial Number: YFJMHTZD
Model: HP MB2000FBUCL
Current Temperature (C): 27
Maximum Temperature (C): 38
PHY Count: 2
PHY Transfer Rate: 6.0Gbps, Unknown
"""
results = []
drive_result = BASE_RESULT.child(dimensions={
'controller_slot': str(slot),
})
drive_result.name += '.physical_drive'
rc = run_cmd(LOCK_FILE_COMMAND +
'hpssacli ctrl slot=%s pd all show detail' % slot)
if rc.exitcode != 0:
r = MetricData.single(
'check.failure', Severity.fail,
'{check} slot: {slot} failed with: {error}', {
'check': drive_result.name,
'slot': slot,
'error': str(rc.output),
'component': 'swiftlm-scan'
})
return [r]
# Remove blank lines and strip trailing/leading spaces for each line
lines = [l.strip() for l in rc.output.split('\n') if l.strip()]
if not lines:
r = MetricData.single(
'check.failure', Severity.fail,
'{check} slot: {slot} failed with: {error}', {
'check': drive_result.name,
'slot': slot,
'error': 'No usable output from hpssacli',
'component': 'swiftlm-scan'
})
return [r]
if is_cont_heading(lines[0]):
lines = lines[1:]
drives = []
drive_info = {}
for line in lines:
# The first two lines for each drive are special.
# The physicaldrive line will contain 2 colons and duplicates
# information so we drop it.
cc = line.count(':')
if cc > 1:
continue
# The Array # line may be useful in the future but does not follow
# the format of colon seperated infommation.
# It is also the only delimiter between drives. We create a new
# drive_info dict when we see it.
if line.startswith('array '):
if drive_info:
drives.append(drive_info)
drive_info = {}
drive_info['array'] = line.split()[1]
continue
k, v = line.split(':', 1)
k = k.strip().lower()
v = v.strip()
drive_info[k] = v
# Have to add the last drive.
if drive_info:
drives.append(drive_info)
for d in drives:
results.extend(check_physical_drive(d, drive_result))
return results
def get_smart_array_info():
"""
parses controller data from hpssacli in the form.
returns a dict.
key's are lowercased versions of the key name on each line,
including special characters.
Values are not changed.
keys 'model' and 'slot' are parsed from the first line
Smart Array P410 in Slot 1
Bus Interface: PCI
Slot: 1
Serial Number: PACCR0M9VZ41S4Q
Cache Serial Number: PACCQID12061TTQ
RAID 6 (ADG) Status: Disabled
Controller Status: OK
Hardware Revision: C
Firmware Version: 6.60
Rebuild Priority: Medium
Expand Priority: Medium
Surface Scan Delay: 15 secs
Surface Scan Mode: Idle
Queue Depth: Automatic
Monitor and Performance Delay: 60 min
Elevator Sort: Enabled
Degraded Performance Optimization: Disabled
Inconsistency Repair Policy: Disabled
Wait for Cache Room: Disabled
Surface Analysis Inconsistency Notification: Disabled
Post Prompt Timeout: 15 secs
Cache Board Present: True
Cache Status: OK
Cache Ratio: 25% Read / 75% Write
Drive Write Cache: Disabled
Total Cache Size: 256 MB
Total Cache Memory Available: 144 MB
No-Battery Write Cache: Disabled
Cache Backup Power Source: Batteries
Battery/Capacitor Count: 1
Battery/Capacitor Status: OK
SATA NCQ Supported: True
Number of Ports: 2 Internal only
Encryption Supported: False
Driver Name: hpsa
Driver Version: 3.4.0
Driver Supports HP SSD Smart Path: False
Smart Array P440ar in Slot 0 (Embedded) (HBA Mode)
Bus Interface: PCI
Slot: 0
Serial Number: PDNLH0BRH7V7GC
Cache Serial Number: PDNLH0BRH7V7GC
Controller Status: OK
Hardware Revision: B
Firmware Version: 2.14
Controller Temperature (C): 50
Number of Ports: 2 Internal only
Driver Name: hpsa
Driver Version: 3.4.4
HBA Mode Enabled: True
PCI Address (Domain:Bus:Device.Function): 0000:03:00.0
Negotiated PCIe Data Rate: PCIe 3.0 x8 (7880 MB/s)
Controller Mode: HBA
Controller Mode Reboot: Not Required
Current Power Mode: MaxPerformance
Host Serial Number: MXQ51906YF
"""
results = []
controller_result = BASE_RESULT.child()
controller_result.name += '.' + 'smart_array'
rc = run_cmd(LOCK_FILE_COMMAND + 'hpssacli ctrl all show detail')
if rc.exitcode != 0:
if 'Error: No controllers detected.' in str(rc.output):
return []
r = MetricData.single('check.failure',
Severity.fail,
'{check} failed with: {error}',
{'check': controller_result.name,
'error': str(rc.output),
'component': 'swiftlm-scan'})
return [r]
if rc.output:
lines = rc.output.split('\n')
else:
r = MetricData.single('check.failure',
Severity.fail,
'{check} failed with: {error}',
{'check': controller_result.name,
'error': 'No usable output from hpssacli',
'component': 'swiftlm-scan'})
return [r]
controllers = []
info = {}
for line in lines:
# Ignore blank lines
if (not line) or (line.isspace()) or (line == "\n"):
continue
if is_cont_heading(line):
if info:
controllers.append(info)
# To get controller model, assume that the line is in the form:
# <model> in Slot <slot>
model = line.strip().split("in Slot")[0].strip()
info = {'model': model}
continue
k, v = line.split(':', 1)
k = k.strip().lower()
v = v.strip()
info[k] = v
if info:
controllers.append(info)
controller_slots = []
for c in controllers:
results.extend(check_controller(c, controller_result))
if c.get('slot'):
controller_slots.append(c.get('slot'))
return results, controller_slots
def get_logical_drive_info(slot, cache_check=True):
"""
array L
Logical Drive: 12
Size: 1.8 TB
Fault Tolerance: 0
Heads: 255
Sectors Per Track: 32
Cylinders: 65535
Strip Size: 256 KB
Full Stripe Size: 256 KB
Status: OK
Caching: Enabled
Unique Identifier: 600508B1001CEA938043498011A76404
Disk Name: /dev/sdl
Mount Points: /srv/node/disk11 1.8 TB Partition Number 2
OS Status: LOCKED
Logical Drive Label: AF3C73D8PACCR0M9VZ41S4QEB69
Drive Type: Data
LD Acceleration Method: Controller Cache
BUG:
It appears that the current build of hpssacli has a bug and outputs
Disk Name and Mount Points on the same line. We work around this by
checking for these specifically but that could fail if they change
"""
results = []
drive_result = BASE_RESULT.child()
drive_result.name += '.' + 'logical_drive'
rc = run_cmd(
LOCK_FILE_COMMAND + 'hpssacli ctrl slot=%s ld all show detail' % slot)
if rc.exitcode != 0:
r = MetricData.single('check.failure',
Severity.fail,
'{check} slot: {slot} failed with: {error}',
{'check': drive_result.name,
'slot': slot,
'error': str(rc.output),
'component': 'swiftlm-scan'})
return [r]
# Remove blank lines and strip trailing/leading spaces for each line
lines = [l.strip() for l in rc.output.split('\n') if l.strip()]
if not lines:
r = MetricData.single('check.failure',
Severity.fail,
'{check} slot: {slot} failed with: {error}',
{'check': drive_result.name,
'slot': slot,
'error': 'No usable output from hpssacli',
'component': 'swiftlm-scan'})
return [r]
# First line should be the controller model and slot number.
# We already have this so remove it if it exists
if is_cont_heading(lines[0]):
lines = lines[1:]
drives = []
drive_info = {}
for line in lines:
# If we see two colons we have to assume that it is a bugged version
# of hpssacli and split them accordingly.
cc = line.count(':')
if cc == 2:
_, dn, mp = line.split(':')
drive_info['disk name'] = dn.strip().split()[0]
drive_info['mount points'] = mp.strip()
continue
# The Array # line may be useful in the future but does not follow
# the format of colon seperated infommation.
# It is also the only delimiter between drives. We create a new
# drive_info dict when we see it.
if line.startswith('array '):
if drive_info:
drives.append(drive_info)
drive_info = {}
drive_info['array'] = line.split()[1]
continue
k, v = line.split(':', 1)
k = k.strip().lower()
v = v.strip()
drive_info[k] = v
# Have to add the last drive.
if drive_info:
drives.append(drive_info)
for d in drives:
results.extend(check_logical_drive(d, drive_result, cache_check))
return results
def get_physical_drive_info(slot):
"""
Parses drive data from hpssacli in the form.
There are multiple drives in the output.
array A
physicaldrive 2C:1:1
Port: 2C
Box: 1
Bay: 1
Status: OK
Drive Type: Data Drive
Interface Type: SAS
Size: 2 TB
Native Block Size: 512
Rotational Speed: 7200
Firmware Revision: HPD3
Serial Number: YFJMHTZD
Model: HP MB2000FBUCL
Current Temperature (C): 27
Maximum Temperature (C): 38
PHY Count: 2
PHY Transfer Rate: 6.0Gbps, Unknown
"""
results = []
drive_result = BASE_RESULT.child(dimensions={
'controller_slot': str(slot),
})
drive_result.name += '.physical_drive'
rc = run_cmd(
LOCK_FILE_COMMAND + 'hpssacli ctrl slot=%s pd all show detail' % slot)
if rc.exitcode != 0:
r = MetricData.single('check.failure',
Severity.fail,
'{check} slot: {slot} failed with: {error}',
{'check': drive_result.name,
'slot': slot,
'error': str(rc.output),
'component': 'swiftlm-scan'})
return [r]
# Remove blank lines and strip trailing/leading spaces for each line
lines = [l.strip() for l in rc.output.split('\n') if l.strip()]
if not lines:
r = MetricData.single('check.failure',
Severity.fail,
'{check} slot: {slot} failed with: {error}',
{'check': drive_result.name,
'slot': slot,
'error': 'No usable output from hpssacli',
'component': 'swiftlm-scan'})
return [r]
if is_cont_heading(lines[0]):
lines = lines[1:]
drives = []
drive_info = {}
for line in lines:
# The first two lines for each drive are special.
# The physicaldrive line will contain 2 colons and duplicates
# information so we drop it.
cc = line.count(':')
if cc > 1:
continue
# The Array # line may be useful in the future but does not follow
# the format of colon seperated infommation.
# It is also the only delimiter between drives. We create a new
# drive_info dict when we see it.
if line.startswith('array '):
if drive_info:
drives.append(drive_info)
drive_info = {}
drive_info['array'] = line.split()[1]
continue
k, v = line.split(':', 1)
k = k.strip().lower()
v = v.strip()
drive_info[k] = v
# Have to add the last drive.
if drive_info:
drives.append(drive_info)
for d in drives:
results.extend(check_physical_drive(d, drive_result))
return results
def get_smart_array_info():
"""
parses controller data from hpssacli in the form.
returns a dict.
key's are lowercased versions of the key name on each line,
including special characters.
Values are not changed.
keys 'model' and 'slot' are parsed from the first line
Smart Array P410 in Slot 1
Bus Interface: PCI
Slot: 1
Serial Number: PACCR0M9VZ41S4Q
Cache Serial Number: PACCQID12061TTQ
RAID 6 (ADG) Status: Disabled
Controller Status: OK
Hardware Revision: C
Firmware Version: 6.60
Rebuild Priority: Medium
Expand Priority: Medium
Surface Scan Delay: 15 secs
Surface Scan Mode: Idle
Queue Depth: Automatic
Monitor and Performance Delay: 60 min
Elevator Sort: Enabled
Degraded Performance Optimization: Disabled
Inconsistency Repair Policy: Disabled
Wait for Cache Room: Disabled
Surface Analysis Inconsistency Notification: Disabled
Post Prompt Timeout: 15 secs
Cache Board Present: True
Cache Status: OK
Cache Ratio: 25% Read / 75% Write
Drive Write Cache: Disabled
Total Cache Size: 256 MB
Total Cache Memory Available: 144 MB
No-Battery Write Cache: Disabled
Cache Backup Power Source: Batteries
Battery/Capacitor Count: 1
Battery/Capacitor Status: OK
SATA NCQ Supported: True
Number of Ports: 2 Internal only
Encryption Supported: False
Driver Name: hpsa
Driver Version: 3.4.0
Driver Supports HP SSD Smart Path: False
Smart Array P440ar in Slot 0 (Embedded) (HBA Mode)
Bus Interface: PCI
Slot: 0
Serial Number: PDNLH0BRH7V7GC
Cache Serial Number: PDNLH0BRH7V7GC
Controller Status: OK
Hardware Revision: B
Firmware Version: 2.14
Controller Temperature (C): 50
Number of Ports: 2 Internal only
Driver Name: hpsa
Driver Version: 3.4.4
HBA Mode Enabled: True
PCI Address (Domain:Bus:Device.Function): 0000:03:00.0
Negotiated PCIe Data Rate: PCIe 3.0 x8 (7880 MB/s)
Controller Mode: HBA
Controller Mode Reboot: Not Required
Current Power Mode: MaxPerformance
Host Serial Number: MXQ51906YF
"""
results = []
controller_result = BASE_RESULT.child()
controller_result.name += '.' + 'smart_array'
rc = run_cmd(LOCK_FILE_COMMAND + 'hpssacli ctrl all show detail')
if rc.exitcode != 0:
if 'Error: No controllers detected.' in str(rc.output):
return []
r = MetricData.single(
'check.failure', Severity.fail, '{check} failed with: {error}', {
'check': controller_result.name,
'error': str(rc.output),
'component': 'swiftlm-scan'
})
return [r]
if rc.output:
lines = rc.output.split('\n')
else:
r = MetricData.single(
'check.failure', Severity.fail, '{check} failed with: {error}', {
'check': controller_result.name,
'error': 'No usable output from hpssacli',
'component': 'swiftlm-scan'
})
return [r]
controllers = []
info = {}
for line in lines:
# Ignore blank lines
if (not line) or (line.isspace()) or (line == "\n"):
continue
if is_cont_heading(line):
if info:
controllers.append(info)
# To get controller model, assume that the line is in the form:
# <model> in Slot <slot>
model = line.strip().split("in Slot")[0].strip()
info = {'model': model}
continue
k, v = line.split(':', 1)
k = k.strip().lower()
v = v.strip()
info[k] = v
if info:
controllers.append(info)
controller_slots = []
for c in controllers:
results.extend(check_controller(c, controller_result))
if c.get('slot'):
controller_slots.append(c.get('slot'))
return results, controller_slots
def main():
args = parse_args()
metrics = []
for func in args.selected:
try:
r = func()
if isinstance(r, list) and r and isinstance(r[0], MetricData):
metrics.extend([result.metric() for result in r])
elif isinstance(r, MetricData):
metrics.append(r.metric())
except SwiftlmCheckFailure as err:
r = MetricData.single('check.failure',
Severity.fail,
'{error} | Failed with: {check}',
dimensions={
'component': 'swiftlm-scan',
'service': 'object-storage'
},
msgkeys={
'check': func.__module__,
'error': str(err)
})
metrics.append(r.metric())
except: # noqa
t, v, tb = sys.exc_info()
backtrace = ' '.join(traceback.format_exception(t, v, tb))
r = MetricData.single('check.failure',
Severity.fail,
'{error} | Failed with: {check}',
dimensions={
'component': 'swiftlm-scan',
'service': 'object-storage'
},
msgkeys={
'check': func.__module__,
'error': backtrace.replace('\n', ' ')
})
metrics.append(r.metric())
# There is no point in reporting multiple measurements of
# swiftlm.check.failure metric in same cycle.
check_failures_found = []
for metric in metrics:
if metric.get('metric') == 'swiftlm.check.failure':
check_failures_found.append(metric)
if check_failures_found:
# Remove all except one instance
for metric in check_failures_found[:-1]:
metrics.remove(metric)
else:
r = MetricData.single('check.failure',
Severity.ok,
'ok',
dimensions={
'component': 'swiftlm-scan',
'service': 'object-storage'
})
metrics.append(r.metric())
dumped_metrics = FORMATS[args.format](metrics, args.pretty)
out_stream = sys.stdout
if args.filename:
try:
with lock_file(args.filename, 2, unlink=False) as cf:
cf.truncate()
cf.write(dumped_metrics)
except (Exception, Timeout) as err:
print('ERROR: %s' % err)
sys.exit(1)
else:
out_stream = sys.stdout
out_stream.write(dumped_metrics)
