def output_metrics(alarm_id, expected_state, metrics):
print('Generating metrics, waiting for state change to %s' %
expected_state)
hostnames = ['AA', 'BB', 'CC']
for x in range(0, 90):
for metric in metrics:
metric_name = metric[0]
dimensions = metric[1]
args = ['metric-create', '--dimensions']
hostname = hostnames[x % len(hostnames)]
args.append(dimensions + ',' + 'hostname=' + hostname)
args.append(metric_name)
args.append('42')
cli_wrapper.run_mon_cli(args, useJson=False)
state = cli_wrapper.get_alarm_state(alarm_id)
if state == expected_state:
break
time.sleep(1)
if state != expected_state:
print('Did not change to state %s instead was %s in %d seconds' %
(expected_state, state, x), file=sys.stderr)
return False
print('Changed to state %s in %d seconds' % (state, x))
return True
def check_alarm_history(alarm_id, states):
transitions = len(states) - 1
print('Checking Alarm History')
# May take some time for Alarm history to flow all the way through
for _ in range(0, 10):
result_json = cli_wrapper.run_mon_cli(['alarm-history', alarm_id])
if len(result_json) >= transitions:
break
time.sleep(4)
result = True
if transitions != len(result_json):
print('Wrong number of history entries, expected %d but was %d' %
(transitions, len(result_json)), file=sys.stderr)
return False
# Alarm history is reverse sorted by date
index = transitions - 1
for i in range(0, transitions):
old_state = states[i]
new_state = states[i+1]
alarm_json = result_json[index]
if not check_expected(old_state, alarm_json['old_state'], 'old_state',
i):
result = False
if not check_expected(new_state, alarm_json['new_state'], 'new_state',
i):
result = False
if not check_expected(alarm_id, alarm_json['alarm_id'], 'alarm_id',
i):
result = False
index = index - 1
if result:
print('Alarm History is OK')
return result
def check_alarm_history(alarm_id, states):
transitions = len(states) - 1
print('Checking Alarm History')
# May take some time for Alarm history to flow all the way through
for _ in range(0, 20):
result_json = cli_wrapper.run_mon_cli(['alarm-history', alarm_id])
if len(result_json) >= transitions:
break
time.sleep(4)
result = True
if transitions != len(result_json):
print('Wrong number of history entries, expected %d but was %d' %
(transitions, len(result_json)),
file=sys.stderr)
return False
# Alarm history is sorted by date
index = 0
for i in range(0, transitions):
old_state = states[i]
new_state = states[i + 1]
alarm_json = result_json[index]
if not check_expected(old_state, alarm_json['old_state'], 'old_state',
i):
result = False
if not check_expected(new_state, alarm_json['new_state'], 'new_state',
i):
result = False
if not check_expected(alarm_id, alarm_json['alarm_id'], 'alarm_id', i):
result = False
index = index + 1
if result:
print('Alarm History is OK')
return result
def get_metrics(name, dimensions, since):
dimensions_arg = ''
for key, value in dimensions.iteritems():
if dimensions_arg != '':
dimensions_arg = dimensions_arg + ','
dimensions_arg = dimensions_arg + key + '=' + value
return cli_wrapper.run_mon_cli(['measurement-list', '--dimensions',
dimensions_arg, name, since])
def get_metrics(name, dimensions, since):
dimensions_arg = ''
for key, value in dimensions.iteritems():
if dimensions_arg != '':
dimensions_arg = dimensions_arg + ','
dimensions_arg = dimensions_arg + key + '=' + value
return cli_wrapper.run_mon_cli(
['measurement-list', '--dimensions', dimensions_arg, name, since])
def get_metrics(name, dimensions):
print('Getting metrics for %s ' % (name + str(dimensions)))
dimensions_arg = ''
for key, value in dimensions.iteritems():
if dimensions_arg != '':
dimensions_arg = dimensions_arg + ','
dimensions_arg = dimensions_arg + key + '=' + value
return cli_wrapper.run_mon_cli(['measurement-list', '--dimensions',
dimensions_arg, name, '00'])
def main():
utils.setup_cli()
alarm_name = 'alarm_crud'
metric_name = 'alarm_crud'
base_dimension = 'service=alarm_test'
expression = 'max(%s{%s}) > 0' % (metric_name, base_dimension)
description = alarm_name + ' Description'
cli_wrapper.delete_alarm_if_exists(alarm_name)
# Add Alarm
alarm_id = cli_wrapper.create_alarm(alarm_name, expression,
description=description)
print('Created Alarm with id %s' % alarm_id)
# Ensure it is created in the right state
initial_state = 'UNDETERMINED'
if not utils.check_alarm_state(alarm_id, initial_state):
return 1
states = []
states.append(initial_state)
# List Alarms, make sure new one shows up
alarm_json = cli_wrapper.find_alarm_by_name(alarm_name)
if alarm_json is None:
print('Did not find alarm named %s using alarm-list' %
alarm_name, file=sys.stderr)
return 1
if alarm_id != alarm_json['id']:
print('Alarm %s has wrong id, expected %s but was %s' %
(alarm_name, alarm_id, alarm_json['id']), file=sys.stderr)
return 1
# Output metrics that will cause it to go ALARM
# Wait for it to change to ALARM
if not output_metrics(alarm_id, 'ALARM', [[metric_name, base_dimension]]):
return 1
states.append('ALARM')
# Modify Alarm by adding new expression that will cause it to go OK
print('Modify Alarm expression so it will go to OK')
new_metric_name = 'other_metric'
new_dimension = 'dim=42'
new_expression = '%s and max(%s{%s}) > 100' % (expression,
new_metric_name,
new_dimension)
alarm_json = cli_wrapper.patch_alarm(alarm_id, '--expression',
new_expression)
if alarm_json['expression'] != new_expression:
print('Did not change expression to %s instead was %s' %
(new_expression, alarm_json['expression']), file=sys.stderr)
return 1
# Output metrics that will cause it to go OK
# Wait for it to change to OK
if not output_metrics(alarm_id, 'OK', [[metric_name, base_dimension],
[new_metric_name, new_dimension]]):
return 1
states.append('OK')
# Modify Alarm by deleting expression that will cause Alarm to go ALARM
print('Delete Alarm sub expression so it will go to ALARM')
cli_wrapper.patch_alarm(alarm_id, '--expression', expression)
# Output metrics that will cause it to go ALARM
# Wait for it to change to ALARM
print('Output extra dimensions to make sure match occurs')
extra_dimension = base_dimension + ',Extra=More'
if not output_metrics(alarm_id, 'ALARM',
[[metric_name, extra_dimension]]):
return 1
states.append('ALARM')
# Modify Alarm by setting alarm state to OK
print('Set Alarm to OK, wait for transition back to ALARM')
cli_wrapper.change_alarm_state(alarm_id, 'OK')
states.append('OK')
# Output metrics that will cause it to go back to ALARM
# Wait for it to change to ALARM
if not output_metrics(alarm_id, 'ALARM',
[[metric_name, base_dimension],
[new_metric_name, new_dimension]]):
return 1
states.append('ALARM')
# Query History
# Delete ALARM
print('Delete alarm')
cli_wrapper.run_mon_cli(['alarm-delete', alarm_id], useJson=False)
# Ensure it can't be queried
if cli_wrapper.find_alarm_by_name(alarm_name) is not None:
print('Still found alarm %s after it was deleted' % alarm_name,
file=sys.stderr)
return 1
# Query History, ensure they still show up
if not utils.check_alarm_history(alarm_id, states):
return 1
# Success
return 0
