def do_kv_store_wait_value(key_name, expected_value, timeout=30):
import itertools
spinners = itertools.cycle(CHARACTERS.spinners)
for i in range(timeout):
expr_64 = base64.b64encode(expr)
try:
r = post_opsbro_json('/agent/evaluator/eval', {'expr': expr_64},
timeout=20)
except get_request_errors() as exp:
logger.error(exp)
return
if r is True:
cprint('\n %s ' % CHARACTERS.arrow_left, color='grey', end='')
cprint('%s ' % CHARACTERS.check, color='green', end='')
cprint('The expression ', end='')
cprint('%s' % expr, color='magenta', end='')
cprint(' is now evaluated to ', end='')
cprint('True', color='green')
sys.exit(0)
# Not detected? increase loop
cprint('\r %s ' % next(spinners), color='blue', end='')
cprint('%s' % expr, color='magenta', end='')
cprint(' is ', end='')
cprint('not True', color='magenta', end='')
cprint(' (%d/%d)' % (i, timeout), end='')
# As we did not \n, we must flush stdout to print it
sys.stdout.flush()
time.sleep(1)
cprint("\nThe expression %s was not evaluated to True after %s seconds" %
(expr, timeout))
sys.exit(2)
def do_agent_parameters_remove(parameter_name, str_value):
if parameter_name not in list_type_parameters:
cprint(
'Error: the parameter %s is not a list. Cannot use the add/remove. Please use set instead'
% parameter_name)
sys.exit(2)
parameters_file_path = DEFAULT_CFG_FILE
yml_parameter_remove(parameters_file_path,
parameter_name,
str_value,
file_display='agent.%s' % parameter_name)
# Groups is a bit special as it can be load directly by the agent
if parameter_name == 'groups':
try:
did_change = get_opsbro_json('/agent/parameters/remove/groups/%s' %
str_value)
except get_request_errors():
cprint(
' | The agent seems to not be started. Skipping hot group removing.',
color='grey')
return
if did_change:
cprint(
" | The agent groups are updated too. You don't need to restart your daemon.",
color='grey')
return
cprint(
'NOTE: only the yml configuration file is modified. You need to restart your agent to use this modification',
color='grey')
return
def do_compliance_state(compliance_name=''):
uri = '/compliance/state'
try:
(code, r) = get_opsbro_local(uri)
except get_request_errors() as exp:
logger.error(exp)
return
try:
compliances = jsoner.loads(r)
except ValueError as exp: # bad json
logger.error('Bad return from the server %s' % exp)
return
print_h1('Compliances')
packs = {}
for (cname, compliance) in compliances.items():
if compliance_name and compliance['name'] != compliance_name:
continue
pack_name = compliance['pack_name']
if pack_name not in packs:
packs[pack_name] = {}
packs[pack_name][cname] = compliance
pnames = list(packs.keys()) # python3
pnames.sort()
for pname in pnames:
pack_entries = packs[pname]
cprint('* Pack %s' % pname, color='blue')
cnames = list(pack_entries.keys()) # python3
cnames.sort()
for cname in cnames:
cprint(' - %s' % pname, color='blue', end='')
compliance = pack_entries[cname]
__print_compliance_entry(compliance)
def do_kv_store_wait_exists(key_name, timeout=30):
import itertools
spinners = itertools.cycle(CHARACTERS.spinners)
for i in range(timeout):
try:
(code, value) = get_opsbro_local('/kv/%s' % key_name)
except get_request_errors() as exp:
logger.error('Cannot join opsbro agent for info: %s' % exp)
sys.exit(1)
if code == 200:
cprint('\n %s ' % CHARACTERS.arrow_left, color='grey', end='')
cprint('%s ' % CHARACTERS.check, color='green', end='')
cprint('The key ', end='')
cprint('%s' % key_name, color='magenta', end='')
cprint(' does ', end='')
cprint('Exists', color='green')
sys.exit(0)
# Not detected? increase loop
cprint('\r %s ' % next(spinners), color='blue', end='')
cprint('%s' % key_name, color='magenta', end='')
cprint(' is ', end='')
cprint('not Existing', color='magenta', end='')
cprint(' (%d/%d)' % (i, timeout), end='')
# As we did not \n, we must flush stdout to print it
sys.stdout.flush()
time.sleep(1)
cprint("\nThe key %s was not set after %s seconds" % (key_name, timeout))
sys.exit(2)
def do_detect_list():
try:
(code, r) = get_opsbro_local('/agent/detectors')
except get_request_errors() as exp:
logger.error(exp)
return
try:
d = jsoner.loads(r)
except ValueError as exp: # bad json
logger.error('Bad return from the server %s' % exp)
return
print_info_title('Detectors')
logger.debug(str(d))
e = []
d = sorted(d, key=lambda i: i['name'])
for i in d:
# aligne name too
name = '%-20s' % i['name'].split('/')[-1]
groups = i['add_groups']
# align pack level
pack_level = '%-6s' % i['pack_level']
# Aligne pack name
pack_name = '%-10s' % i['pack_name']
print_element_breadcumb(pack_name, pack_level, 'detector', name, set_pack_color=True)
cprint('')
cprint(' if: ', color='grey', end='')
cprint(i['apply_if'], color='green')
cprint(' add groups: ', color='grey', end='')
cprint(','.join(groups), color='magenta')
def do_detect_wait_group(group_name, timeout=30):
import itertools
spinners = itertools.cycle(CHARACTERS.spinners)
for i in range(timeout):
uri = '/agent/detectors/state'
try:
detected_groups = get_opsbro_json(uri)
except get_request_errors() as exp:
logger.error(exp)
return
if group_name in detected_groups:
cprint('\n %s ' % CHARACTERS.arrow_left, color='grey', end='')
cprint('%s ' % CHARACTERS.check, color='green', end='')
cprint('The group ', end='')
cprint('%s' % group_name, color='magenta', end='')
cprint(' is ', end='')
cprint('detected', color='green')
sys.exit(0)
# Not detected? increase loop
cprint('\r %s ' % next(spinners), color='blue', end='') # next=> python3
cprint('%s' % group_name, color='magenta', end='')
cprint(' is ', end='')
cprint('NOT DETECTED', color='magenta', end='')
cprint(' (%d/%d)' % (i, timeout), end='')
# As we did not \n, we must flush stdout to print it
sys.stdout.flush()
time.sleep(1)
cprint("\nThe group %s was not detected after %s seconds" % (group_name, timeout))
sys.exit(2)
def do_detect_history():
uri = '/agent/detectors/history'
try:
(code, r) = get_opsbro_local(uri)
except get_request_errors() as exp:
logger.error(exp)
return
try:
histories = jsoner.loads(r)
except ValueError as exp: # bad json
logger.error('Bad return from the server %s' % exp)
return
print_h1('Detected groups history for this node')
for history in histories:
epoch = history['date']
# We want only group type events
entries = [entry for entry in history['entries'] if entry['type'] in ('group-add', 'group-remove')]
if not entries:
continue
print_h2(' Date: %s ' % time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime(epoch)))
for entry in entries:
_type = entry['type']
op = {'group-add': '+', 'group-remove': '-'}.get(_type, '?')
color = {'group-add': 'green', 'group-remove': 'red'}.get(_type, 'grey')
cprint(' %s %s' % (op, entry['group']), color=color)
def do_collectors_history():
try:
history_entries = get_opsbro_json('/agent/collectors/history')
except get_request_errors() as exp:
logger.error(exp)
return
print_h1('History')
for history_entry in history_entries:
epoch_date = history_entry['date']
entries = history_entry['entries']
print_h2(
' Date: %s ' %
time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime(epoch_date)))
for entry in entries:
name = entry['name']
cprint(' * ', end='')
cprint('%s ' % (name.ljust(20)), color='magenta', end='')
old_state = entry['old_state']
c = COLLECTORS_STATE_COLORS.get(old_state, 'cyan')
cprint('%s' % old_state.ljust(10), color=c, end='')
cprint(' %s ' % CHARACTERS.arrow_left, color='grey', end='')
state = entry['state']
c = COLLECTORS_STATE_COLORS.get(state, 'cyan')
cprint('%s' % state.ljust(10), color=c)
# Now print output the line under
log = entry['log']
if log:
cprint(' ' * 4 + '| ' + log, color='grey')
def do_stop():
try:
(code, r) = get_opsbro_local('/stop')
except get_request_errors() as exp:
logger.error(exp)
return
cprint(r, color='green')
def do_node_public_addr():
try:
d = get_opsbro_json('/agent/info')
except get_request_errors() as exp:
logger.error('Cannot join opsbro agent for info: %s' % exp)
sys.exit(1)
public_addr = d.get('public_addr')
cprint(public_addr)
def do_node_uuid():
try:
d = get_opsbro_json('/agent/info')
except get_request_errors() as exp:
logger.error('Cannot join opsbro agent for info: %s' % exp)
sys.exit(1)
_uuid = d.get('uuid')
cprint(_uuid)
def do_exec(group='*', cmd='uname -a'):
if cmd == '':
logger.error('Missing command')
return
# The information is available only if the agent is started
wait_for_agent_started(visual_wait=True)
try:
(code, r) = get_opsbro_local('/exec/%s?cmd=%s' % (group, cmd))
except get_request_errors() as exp:
logger.error(exp)
return
print(r)
cid = r
print("Command group launch as cid", cid)
time.sleep(5) # TODO: manage a real way to get the result..
try:
(code, r) = get_opsbro_local('/exec-get/%s' % cid)
except get_request_errors() as exp:
logger.error(exp)
return
j = jsoner.loads(r)
res = j['res']
for (uuid, e) in res.items():
node = e['node']
nname = node['name']
color = {
'alive': 'green',
'dead': 'red',
'suspect': 'yellow',
'leave': 'cyan'
}.get(node['state'], 'cyan')
cprint(nname, color=color)
cprint('Return code:', end='')
color = {0: 'green', 1: 'yellow', 2: 'red'}.get(e['rc'], 'cyan')
cprint(e['rc'], color=color)
cprint('Output:', end='')
cprint(e['output'].strip(), color=color)
if e['err']:
cprint('Error:', end='')
cprint(e['err'].strip(), color='red')
cprint('')
def do_list_follow_log():
try:
parts = get_opsbro_json('/log/parts/')
except get_request_errors() as exp:
logger.error('Cannot join opsbro agent to list logs: %s' % exp)
sys.exit(1)
parts.sort()
cprint("Available parts to follow logs:")
for p in parts:
cprint(" * %s" % p)
def do_kv_store_list():
expr_64 = base64.b64encode(expr)
try:
r = post_opsbro_json('/agent/evaluator/eval', {'expr': expr_64},
timeout=30)
except get_request_errors() as exp:
logger.error(exp)
return
print_info_title('Result')
cprint(r)
def do_kv_store_get(key_name):
try:
(code, value) = get_opsbro_local('/kv/%s' % key_name)
except get_request_errors() as exp:
logger.error('Cannot join opsbro agent for info: %s' % exp)
sys.exit(1)
if code == 404:
cprint('ERROR: the key %s does not exist' % key_name, color='red')
sys.exit(2)
value = bytes_to_unicode(value)
cprint("%s::%s" % (key_name, value))
def do_collectors_state():
print_h1('Collectors')
try:
collectors = get_opsbro_json('/collectors')
except get_request_errors() as exp:
logger.error(exp)
return
cnames = list(collectors.keys())
cnames.sort()
for cname in cnames:
d = collectors[cname]
__print_collector_state(d)
def do_exec(group='*', cmd='uname -a'):
if cmd == '':
logger.error('Missing command')
return
# The information is available only if the agent is started
wait_for_agent_started(visual_wait=True)
cmd_64 = bytes_to_unicode(base64.b64encode(unicode_to_bytes(cmd)))
try:
r = get_opsbro_json('/exec/%s?cmd=%s' % (group, cmd_64))
except get_request_errors() as exp:
cprint('ERROR: cannot launch the command: %s' % exp, color='red')
sys.exit(2)
cid = r
print("Command group launch as cid", cid)
time.sleep(5) # TODO: manage a real way to get the result..
try:
r = get_opsbro_json('/exec-get/%s' % cid)
except get_request_errors() as exp:
cprint('ERROR: cannot get execution results: %s' % exp, color='red')
sys.exit(2)
res = r['res']
print('Launched at: %s' % res)
for (uuid, e) in res.items():
node = e['node']
nname = node['name']
color = {'alive': 'green', 'dead': 'red', 'suspect': 'yellow', 'leave': 'cyan'}.get(node['state'], 'cyan')
cprint(nname, color=color)
cprint('Return code for [%s]:' % e['cmd'], end='')
color = {0: 'green', 1: 'yellow', 2: 'red'}.get(e['rc'], 'cyan')
cprint(e['rc'], color=color)
cprint('Output:', end='')
cprint(e['output'].strip(), color=color)
if e['err']:
cprint('Error:', end='')
cprint(e['err'].strip(), color='red')
cprint('')
def do_zone_change(name=''):
if not name:
cprint("Need a zone name")
return
cprint("Switching to zone", name)
try:
r = put_opsbro_json('/agent/zone', name)
except get_request_errors() as exp:
logger.error(exp)
return
print_info_title('Result')
print(r)
def do_evaluator_list(details=False):
try:
(code, r) = get_opsbro_local('/agent/evaluator/list')
except get_request_errors() as exp:
logger.error(exp)
return
try:
d = jsoner.loads(r)
except ValueError as exp: # bad json
logger.error('Bad return from the server %s' % exp)
return
# Group by
groups = {}
for f in d:
fname = f['name']
gname = f['group']
if gname not in groups:
groups[gname] = {}
groups[gname][fname] = f
gnames = groups.keys()
gnames.sort()
for gname in gnames:
print_h1(gname)
group_functions = groups[gname]
fnames = group_functions.keys()
fnames.sort()
for fname in fnames:
f = group_functions[fname]
prototype = f['prototype']
doc = f['doc']
cprint(fname, color='green', end='')
cprint('(', end='')
if prototype:
_s_args = []
for arg in prototype:
kname = arg[0]
def_value = arg[1]
if def_value != '__NO_DEFAULT__':
_s_args.append('%s=%s' % (kname, def_value))
else:
_s_args.append('%s' % kname)
cprint(', '.join(_s_args), color='yellow', end='')
cprint(')')
if details:
cprint(doc, color='grey')
def _save_key(key_string, zone_name, key_path):
with open(key_path, 'wb') as f:
f.write(unicode_to_bytes(key_string))
cprint('%s OK the key is saved as file %s' % (CHARACTERS.check, key_path))
# Try to send the information to the agent, so it can reload the key
try:
get_opsbro_json('/agent/zones-keys/reload/%s' % zone_name)
except get_request_errors():
cprint(
' | The agent seems to not be started. Skipping hot key reload.',
color='grey')
return
def do_leave(nuuid=''):
# The information is available only if the agent is started
wait_for_agent_started(visual_wait=True)
# Lookup at the localhost name first
if not nuuid:
try:
nuuid = get_opsbro_json('/agent/uuid')
except get_request_errors() as exp:
logger.error('Cannot join opsbro agent to get our uuid: %s' % exp)
sys.exit(2)
uri = '/agent/leave/%s' % nuuid
try:
(code, r) = get_opsbro_local(uri)
except get_request_errors() as exp:
logger.error(exp)
return
if code != 200:
logger.error('Node %s is missing (return=%s)' % (nuuid, r))
return
cprint('Node %s is set to leave state' % nuuid, end='')
cprint(': OK', color='green')
def do_ping(node):
# The information is available only if the agent is started
wait_for_agent_started(visual_wait=True)
try:
node_obj = get_opsbro_json('/agent/query/guess/%s' % node)
except get_request_errors() as exp:
logger.error('Cannot query the node: %s' % exp)
sys.exit(2)
if node_obj is None:
cprint('FAILED: cannot find the node %s' % node, color='red')
sys.exit(2)
node_uuid = node_obj['uuid']
try:
ping_result = get_opsbro_json('/agent/ping/%s' % node_uuid)
except get_request_errors() as exp:
logger.error('Cannot launch the node ping: %s' % exp)
sys.exit(2)
if 'error' in ping_result:
cprint('FAILED: %s' % ping_result['error'], color='red')
sys.exit(2)
node_state = ping_result['state']
display_name = node_obj['display_name']
if not display_name:
display_name = node_obj['name']
state_color = NODE_STATE_COLORS.get(node_state)
state_char = NODE_STATE_PREFIXS.get(node_state)
cprint(' %s ' % state_char, color=state_color, end='')
cprint('%-15s ' % display_name, color='magenta', end='')
cprint('is: ', end='')
cprint(node_state, color=state_color)
# If not alive, it's an error
if node_state != NODE_STATES.ALIVE:
sys.exit(2)
def do_members_history():
# The information is available only if the agent is started
wait_for_agent_started(visual_wait=True)
try:
history_entries = get_opsbro_json('/agent/members/history')
except get_request_errors() as exp:
logger.error('Cannot join opsbro agent to show member history: %s' %
exp)
sys.exit(1)
print_h1('History')
for history_entry in history_entries:
epoch_date = history_entry['date']
# We want only group type events
entries = [
entry for entry in history_entry['entries']
if entry['type'] == 'node-state-change'
]
if not entries:
continue
print_h2(
' Date: %s ' %
time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime(epoch_date)))
for entry in entries:
name = entry['name']
if entry.get('display_name', ''):
name = '[ ' + entry.get('display_name') + ' ]'
old_state = entry['old_state']
new_state = entry['state']
old_color = NODE_STATE_COLORS.get(old_state, 'cyan')
old_state_prefix = NODE_STATE_PREFIXS.get(
old_state, CHARACTERS.double_exclamation)
new_color = NODE_STATE_COLORS.get(new_state, 'cyan')
new_state_prefix = NODE_STATE_PREFIXS.get(
new_state, CHARACTERS.double_exclamation)
cprint('%s ' % name.ljust(20), color='magenta', end='')
cprint(('%s %s' % (old_state_prefix, old_state)).ljust(9),
color=old_color,
end='') # 7 for the maximum state string + 2 for prefix
cprint(' %s ' % CHARACTERS.arrow_left, color='grey', end='')
cprint(('%s %s' % (new_state_prefix, new_state)).ljust(9),
color=new_color
) # 7 for the maximum state string + 2 for prefix
def do_wait_ok(check_name, timeout=30):
import itertools
spinners = itertools.cycle(CHARACTERS.spinners)
name = check_name
state_string = 'UNKNOWN'
for i in range(timeout):
uri = '/monitoring/state'
try:
(code, r) = get_opsbro_local(uri)
except get_request_errors() as exp:
logger.error(exp)
return
try:
states = jsoner.loads(r)
except ValueError as exp: # bad json
logger.error('Bad return from the server %s' % exp)
return
checks = states['checks']
check = None
for (cname, c) in checks.items():
if c['display_name'] == name:
check = c
if not check:
logger.error("Cannot find the check '%s'" % name)
sys.exit(2)
state_id = check['state_id']
c = STATE_ID_COLORS.get(state_id, 'cyan')
state_string = STATE_ID_STRINGS.get(state_id, 'UNKNOWN')
cprint('\r %s ' % next(spinners), color='blue', end='')
cprint('%s' % name, color='magenta', end='')
cprint(' is ', end='')
cprint('%s' % state_string.ljust(10), color=c, end='')
cprint(' (%d/%d)' % (i, timeout), end='')
# As we did not \n, we must flush stdout to print it
sys.stdout.flush()
if state_id == 0:
cprint("\nThe check %s is OK" % name)
sys.exit(0)
logger.debug("Current state %s" % state_id)
time.sleep(1)
cprint("\nThe check %s is not OK after %s seconds (currently %s)" % (name, timeout, state_string))
sys.exit(2)
def do_gossip_add_event(event_type):
# The information is available only if the agent is started
wait_for_agent_started(visual_wait=True)
try:
r = post_opsbro_json('/agent/event', {'event_type': event_type})
except get_request_errors() as exp:
logger.error(exp)
sys.exit(2)
cprint('\n %s ' % CHARACTERS.arrow_left, color='grey', end='')
cprint('%s ' % CHARACTERS.check, color='green', end='')
cprint('The event ', end='')
cprint('%s' % event_type, color='magenta', end='')
cprint(' is ', end='')
cprint('added', color='green')
def do_zone_list():
print_h1('Known zones')
try:
zones = get_opsbro_json('/agent/zones')
except get_request_errors() as exp:
logger.error(exp)
return
# We are building the zone tree, so link real object in other zone
for zone in zones.values():
sub_zones = {}
for sub_zone_name in zone.get('sub-zones', []):
sub_zone = zones.get(sub_zone_name, None)
sub_zones[sub_zone_name] = sub_zone
zone['sub-zones'] = sub_zones
# Set if the zone is top/lower if not our own zone
for zone in zones.values():
zone['type'] = _ZONE_TYPES.OTHER
# And finally delete the zone that are not in top level
to_del = set()
for (zname, zone) in zones.items():
for sub_zname in zone['sub-zones']:
to_del.add(sub_zname)
for zname in to_del:
del zones[zname]
for zone in zones.values():
_flag_top_lower_zone(zone)
# Now print it
zone_names = zones.keys()
zone_names.sort()
for zname in zone_names:
zone = zones[zname]
_print_zone(zname, zone, 0)
cprint('')
print_h1('Zones types legend')
for zone_type in _ALL_ZONE_TYPES:
label = _ZONE_TYPE_LABEL[zone_type]
color = _ZONE_TYPE_COLORS[zone_type]
description = _ZONE_TYPE_DESCRIPTION[zone_type]
cprint(' - ', end='')
cprint('%-15s' % label, color=color, end='')
cprint(' : %s' % description)
def do_kv_store_put(key_name, value):
if not key_name:
cprint("Need a key-name parameter", color='red')
sys.exit(2)
cprint(' - Set key ', end='')
cprint(key_name, color='magenta', end='')
cprint(' : ', end='')
sys.stdout.flush()
try:
put_opsbro_json('/kv/%s' % key_name, value)
except get_request_errors() as exp:
cprint(CHARACTERS.cross, color='red')
logger.error(exp)
sys.exit(2)
cprint(CHARACTERS.check, color='green')
def do_compliance_launch(compliance_name, timeout=30):
cprint("Launching compliance %s" % compliance_name)
try:
founded = put_opsbro_json('/compliance/launch', compliance_name)
except get_request_errors() as exp:
logger.error(exp)
return
if not founded:
cprint('ERROR: cannot find the compliance %s (founded=%s)' %
(compliance_name, founded),
color='red')
sys.exit(2)
do_compliance_wait_compliant(compliance_name,
timeout=timeout,
exit_if_ok=False)
do_compliance_state(compliance_name=compliance_name)
def do_compliance_history():
uri = '/compliance/history'
try:
(code, r) = get_opsbro_local(uri)
except get_request_errors() as exp:
logger.error(exp)
return
try:
histories = jsoner.loads(r)
except ValueError as exp: # bad json
logger.error('Bad return from the server %s' % exp)
return
print_h1('Compliance history')
for history in histories:
epoch = history['date']
entries = history['entries']
print_h2(' Date: %s ' %
time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime(epoch)))
entries_by_compliances = {}
for entry in entries:
compliance_name = entry['compliance_name']
pack_name = entry['pack_name']
mode = entry['mode']
if compliance_name not in entries_by_compliances:
entries_by_compliances[compliance_name] = {
'pack_name': pack_name,
'mode': mode,
'entries': []
}
entries_by_compliances[compliance_name]['entries'].append(entry)
for (compliance_name, d) in entries_by_compliances.items():
pack_name = d['pack_name']
mode = d['mode']
entries = d['entries']
cprint(' - %s' % pack_name, color='blue', end='')
cprint(' > compliance > ', color='grey', end='')
cprint('[%s] ' % (compliance_name.ljust(30)),
color='magenta',
end='')
cprint('[mode:%10s] ' % mode, color='magenta')
for entry in entries:
# rule_name = entry['name']
# cprint('[%s] ' % (rule_name.ljust(30)), color='magenta', end='')
__print_rule_entry(entry)
cprint("\n")
def do_zone_list():
print_h1('Known zones')
try:
zones = get_opsbro_json('/agent/zones')
except get_request_errors() as exp:
logger.error(exp)
return
for (zname, zone) in zones.items():
cprint(' * ', end='')
cprint(zname, color='magenta')
sub_zones = zone.get('sub-zones', [])
if not sub_zones:
continue
cprint(' Sub zones:')
for sub_zname in sub_zones:
cprint(' - ', end='')
cprint(sub_zname, color='cyan')