def test_elasticsearch_node_count(local_salt_client):
now = datetime.datetime.now()
today = now.strftime("%Y.%m.%d")
active_nodes = utils.get_active_nodes()
salt_output = local_salt_client.cmd(
'kibana:server',
'pillar.get', ['_param:haproxy_elasticsearch_bind_host'],
expr_form='pillar')
IP = salt_output.values()[0]
resp = json.loads(
requests.post('http://{0}:9200/log-{1}/_search?pretty'.format(
IP, today),
data='{"size": 0, "aggs": '
'{"uniq_hostname": '
'{"terms": {"size": 500, '
'"field": "Hostname.keyword"}}}}').text)
cluster_domain = local_salt_client.cmd('salt:control',
'pillar.get',
['_param:cluster_domain'],
expr_form='pillar').values()[0]
monitored_nodes = []
for item_ in resp['aggregations']['uniq_hostname']['buckets']:
node_name = item_['key']
monitored_nodes.append(node_name + '.' + cluster_domain)
missing_nodes = []
for node in active_nodes.keys():
if node not in monitored_nodes:
missing_nodes.append(node)
assert len(missing_nodes) == 0, \
'Not all nodes are in Elasticsearch. Found {0} keys, ' \
'expected {1}. Missing nodes: \n{2}'. \
format(len(monitored_nodes), len(active_nodes), missing_nodes)
def test_check_ens3_interfaces(local_salt_client):
active_nodes = utils.get_active_nodes()
ens3_interfaces = local_salt_client.cmd(
"L@"+','.join(active_nodes), 'cmd.run', ['ifconfig ens3'], expr_form='compound')
nodes = ens3_interfaces.keys()
count = 0
for node in nodes:
thread = MyThread(node, ens3_interfaces, count)
thread.setName(node)
thread.start()
count += 1
assert len(nodes_without_ens3) == 0, "Node is not contain ens3 interface: ".format(nodes_without_ens3)
def test_etc_hosts(local_salt_client):
active_nodes = utils.get_active_nodes()
nodes_info = local_salt_client.cmd(utils.list_to_target_string(
active_nodes, 'or'),
'cmd.run', ['cat /etc/hosts'],
expr_form='compound')
result = {}
for node in nodes_info.keys():
for nd in nodes_info.keys():
if node not in nodes_info[nd]:
if node in result:
result[node] += ',' + nd
else:
result[node] = nd
assert len(result) <= 1, \
"Some hosts are not presented in /etc/hosts: {0}".format(
json.dumps(result, indent=4))
def test_ntp_sync(local_salt_client):
testname = os.path.basename(__file__).split('.')[0]
active_nodes = utils.get_active_nodes(os.path.basename(__file__))
config = utils.get_configuration()
fail = {}
saltmaster_time = int(
local_salt_client.cmd('salt:master',
'cmd.run', ['date +%s'],
expr_form='pillar').values()[0])
nodes_time = local_salt_client.cmd(utils.list_to_target_string(
active_nodes, 'or'),
'cmd.run', ['date +%s'],
expr_form='compound')
diff = config.get(testname)["time_deviation"] or 30
for node, time in nodes_time.iteritems():
if (int(time) - saltmaster_time) > diff or \
(int(time) - saltmaster_time) < -diff:
fail[node] = time
assert not fail, 'SaltMaster time: {}\n' \
'Nodes with time mismatch:\n {}'.format(saltmaster_time,
fail)