def validate(self, server, since_time=None, events_count=-1):
"""
Validates the cluster events against the given events_list
:param server: Server from which we can get the cluster events
:param since_time: Start time from which the events should be fetched
:param events_count: Number of events to be fetched. '-1' means all
:return list: List containing failures
"""
def extract_req_values(cluster_event, local_event):
output_dict = dict()
for key in local_event.keys():
if key not in cluster_event:
continue
if type(local_event[key]) is dict:
output_dict[key] = extract_req_values(cluster_event[key],
local_event[key])
elif type(cluster_event[key]) is list:
cluster_event[key].sort()
output_dict[key] = cluster_event[key]
local_event[key].sort()
else:
output_dict[key] = cluster_event[key]
return output_dict
failures = list()
# If nothing stored in event list, nothing to validate
if not self.events:
return failures
rest = SystemEventRestHelper([server])
# Fetch all events from the server for generic validation
events = rest.get_events(server=server, events_count=-1)["events"]
# Check for event_id duplications
if self.__duplicate_event_ids_present(events, failures):
failures.append("Duplicate event_ids seen")
# Fetch events from the cluster and validate against the ones the
# test has recorded
v_index = 0
events = rest.get_events(server=server, since_time=since_time,
events_count=events_count)["events"]
for event in events:
if isinstance(self.events[v_index], list):
# Process events which occurred in parallel from test's POV
# TODO: Write a algo to find the match from the subset
pass
else:
dict_to_compare = extract_req_values(event,
self.events[v_index])
if dict_to_compare == self.events[v_index]:
v_index += 1
if v_index == self.__event_counter.counter \
or v_index == EventHelper.max_events:
break
else:
failures.append("Unable to validate event: %s"
% self.events[v_index])
return failures
class QuerySystemEventLogs(QueryTests):
def setUp(self):
super(QuerySystemEventLogs, self).setUp()
self.shell = RemoteMachineShellConnection(self.master)
self.info = self.shell.extract_remote_info()
if self.info.type.lower() == 'windows':
self.curl_path = f"{self.path}curl"
else:
self.curl_path = "curl"
self.event_rest = SystemEventRestHelper([self.master])
def tearDown(self):
super(QuerySystemEventLogs, self).tearDown()
def test_memory_quota(self):
event_seen = False
log_fields = [
"uuid", "component", "event_id", "description", "severity",
"timestamp", "extra_attributes", "node"
]
query = "'statement=select (select * from `default`)&memory_quota=1'"
curl_output = self.shell.execute_command(
f"{self.curl_path} -X POST -u {self.rest.username}:{self.rest.password} http://{self.master.ip}:{self.n1ql_port}/query/service -d {query}"
)
self.log.info(curl_output)
output = self.convert_list_to_json(curl_output[0])
requestID = output['requestID']
events = self.event_rest.get_events(server=self.master,
events_count=-1)["events"]
for event in events:
if event['event_id'] == 1026:
event_seen = True
for field in log_fields:
self.assertTrue(
field in event.keys(),
f"Field {field} is not in the event and it should be, please check the event {event}"
)
self.assertEqual(event['component'], "query")
self.assertEqual(event['description'],
"Request memory quota exceeded")
self.assertEqual(event['severity'], "info")
self.assertEqual(event['node'], self.master.ip)
self.assertEqual(event['extra_attributes']['request-id'],
requestID)
self.assertTrue(
event_seen,
f"We did not see the event id we were looking for: {events}")
def test_change_settings(self):
event_seen = False
log_fields = [
"uuid", "component", "event_id", "description", "severity",
"timestamp", "extra_attributes", "node"
]
settings = '{"auto-prepare":true,"completed-limit":5000,"controls":false}'
curl_output = self.shell.execute_command(
f"{self.curl_path} -X POST -u {self.rest.username}:{self.rest.password} http://{self.servers[1].ip}:{self.n1ql_port}/admin/settings -d '{settings}'"
)
self.log.info(curl_output)
events = self.event_rest.get_events(server=self.master,
events_count=-1)["events"]
for event in events:
if event['event_id'] == 1025:
if "None" in str(event):
pass
else:
for field in log_fields:
self.assertTrue(
field in event.keys(),
f"Field {field} is not in the event and it should be, please check the event {event}"
)
self.assertEqual(event['component'], "query")
self.assertEqual(event['description'],
"Configuration changed")
self.assertEqual(event['severity'], "info")
settings_changed = event['extra_attributes']
# Now check the event we generated
event_seen = True
for setting in settings_changed:
if setting == "auto-prepare":
expected_setting = {'from': False, 'to': True}
elif setting == "completed-limit":
expected_setting = {'from': 4000, 'to': 5000}
else:
self.fail(
f"Unrecognized setting {setting} which should not have been changed! Please check the event {event}"
)
self.assertEqual(event['extra_attributes'][setting],
expected_setting)
self.assertEqual(event['node'], self.servers[1].ip)
self.assertTrue(
event_seen,
f"We did not see the event id we were looking for: {events}")
class SystemEventLogs(ClusterSetup):
def setUp(self):
super(SystemEventLogs, self).setUp()
self.rest = RestConnection(self.cluster.master)
self.event_rest_helper = \
SystemEventRestHelper(self.cluster.nodes_in_cluster)
self.log_setup_status("SystemEventLogs", "started")
def tearDown(self):
self.log_setup_status("SystemEventLogs", "started", stage="tearDown")
self.log_setup_status("SystemEventLogs", "completed", stage="tearDown")
super(SystemEventLogs, self).tearDown()
def get_event_from_cluster(self, last=True):
events = self.event_rest_helper.get_events(
server=self.cluster.master, events_count=-1)["events"]
if last:
return events[-1]
else:
return events
def generic_fields_check(self, event):
event_keys = event.keys()
for param in [Event.Fields.TIMESTAMP, Event.Fields.UUID]:
if param not in event_keys:
self.fail("%s key missing in event" % param)
@staticmethod
def get_default_password_settings():
return {
"minLength": 6,
"must_present": []
}
def test_user_crud_events(self):
domain = "local"
username = "******"
payload = "name=" + username + "&roles=admin"
for action in ["create", "delete"]:
if action == "create":
self.log.info("Creating user")
self.rest.add_set_builtin_user(username, payload)
user_event = SecurityEvents.user_added(self.cluster.master.ip, "local")
else:
self.log.info("Deleting user")
self.rest.delete_builtin_user(username)
user_event = SecurityEvents.user_deleted(self.cluster.master.ip, "local")
# Get the last user created/deleted event
event = self.get_event_from_cluster()
# Test NON Extra Attributes fields & NON generic fields
for param, value in user_event.items():
if param == Event.Fields.EXTRA_ATTRS:
continue
if event[param] != value:
self.fail("Value mismatch for '%s'. Expected %s != %s Actual"
% (param, value, event[param]))
# Test generic fields
self.generic_fields_check(event)
# Test Extra Attributes fields
if event[Event.Fields.EXTRA_ATTRS]["user"] == username:
self.fail("Username got printed in system event log")
if event[Event.Fields.EXTRA_ATTRS]["domain"] != domain:
self.fail("Domain mismatch. Expected %s != %s Actual"
% (event[Event.Fields.EXTRA_ATTRS]["domain"], domain))
def test_group_crud_events(self):
groupname = "cbadmingroup"
payload = "roles=admin"
for action in ["create", "delete"]:
if action == "create":
self.log.info("Creating group")
self.rest.add_set_bulitin_group(groupname, payload)
user_event = SecurityEvents.group_added(self.cluster.master.ip)
else:
self.log.info("Deleting group")
self.rest.delete_builtin_group(groupname)
user_event = SecurityEvents.group_deleted(self.cluster.master.ip)
# Get the last event
event = self.get_event_from_cluster()
# Test NON Extra Attributes fields & NON generic fields
for param, value in user_event.items():
if param == Event.Fields.EXTRA_ATTRS:
continue
if event[param] != value:
self.fail("Value mismatch for '%s'. Expected %s != %s Actual"
% (param, value, event[param]))
# Test generic fields
self.generic_fields_check(event)
# Test Extra Attributes fields
if event[Event.Fields.EXTRA_ATTRS]["group"] == groupname:
self.fail("Groupname got printed in system event log")
def test_password_policy_changed_event(self):
default_settings = self.get_default_password_settings()
status, content = self.rest.change_password_policy(min_length=8, enforce_digits="true")
if not status:
self.fail("Changing password policy failed with {0}".format(content))
new_settings = copy.deepcopy(default_settings)
new_settings["minLength"] = 8
new_settings["must_present"] = ["digits"]
# Get the last event
event = self.get_event_from_cluster()
user_event = SecurityEvents.password_policy_changed(self.cluster.master.ip,
old_min_length=
default_settings["minLength"],
old_must_present=
default_settings["must_present"],
new_min_length=
new_settings["minLength"],
new_must_present=
new_settings["must_present"])
# Test NON Extra Attributes fields & NON generic fields
for param, value in user_event.items():
if param == Event.Fields.EXTRA_ATTRS:
continue
if event[param] != value:
self.fail("Value mismatch for '%s'. Expected %s != %s Actual"
% (param, value, event[param]))
# Test generic fields
self.generic_fields_check(event)
# Test Extra Attributes fields
for param in ["old_settings", "new_settings"]:
exp_val = user_event[Event.Fields.EXTRA_ATTRS][param]
act_val = event[Event.Fields.EXTRA_ATTRS][param]
if act_val != exp_val:
self.fail("Mismatch in %s. Expected %s != %s Actual"
% (param, exp_val, act_val))
def test_ldap_config_changed_event(self):
def validate_ldap_event(old_settings, new_settings):
old_settings_copy = copy.deepcopy(old_settings)
new_settings_copy = copy.deepcopy(new_settings)
old_settings_copy["clientTLSCert"] = "redacted"
old_settings_copy["cacert"] = "redacted"
old_settings_copy["bindDN"] = "redacted"
new_settings_copy["clientTLSCert"] = "redacted"
new_settings_copy["cacert"] = "redacted"
new_settings_copy["bindDN"] = "redacted"
for key, val in old_settings_copy.items():
if val == "false":
old_settings_copy[key] = False
elif val == "true":
old_settings_copy[key] = True
for key, val in new_settings_copy.items():
if val == "false":
new_settings_copy[key] = False
elif val == "true":
new_settings_copy[key] = True
# Get the last event
event = self.get_event_from_cluster()
user_event = SecurityEvents.ldap_config_changed(self.cluster.master.ip,
old_settings_copy, new_settings_copy)
# Test NON Extra Attributes fields & NON generic fields
for param, value in user_event.items():
if param == Event.Fields.EXTRA_ATTRS:
continue
if event[param] != value:
self.fail("Value mismatch for '%s'. Expected %s != %s Actual"
% (param, value, event[param]))
# Test generic fields
self.generic_fields_check(event)
# Test Extra Attributes fields
for param in ["old_settings", "new_settings"]:
exp_val = user_event[Event.Fields.EXTRA_ATTRS][param]
act_val = event[Event.Fields.EXTRA_ATTRS][param]
if act_val != exp_val:
self.fail("Mismatch in %s. Expected %s != %s Actual"
% (param, exp_val, act_val))
old_setting = {"authenticationEnabled": "false", "authorizationEnabled": "false",
"bindDN": "", "bindPass": "", "cacheValueLifetime": 300000,
"encryption": "None", "failOnMaxDepth": "false",
"hosts": [], "maxCacheSize": 10000,
"maxParallelConnections": 100, "nestedGroupsEnabled": "false",
"nestedGroupsMaxDepth": 10, "port": 389, "requestTimeout": 5000,
"serverCertValidation": "true", "userDNMapping": "None"}
new_setting = {"authenticationEnabled": "true", "authorizationEnabled": "true",
"bindDN": "", "bindPass": "", "cacheValueLifetime": 300000,
"encryption": "None", "failOnMaxDepth": "false",
"hosts": [], "maxCacheSize": 10000,
"maxParallelConnections": 100, "nestedGroupsEnabled": "false",
"nestedGroupsMaxDepth": 10, "port": 390, "requestTimeout": 5000,
"serverCertValidation": "true", "userDNMapping": "None"}
ldap_settings = {"authenticationEnabled": "true", "authorizationEnabled": "true",
"port": 390}
status, content, _ = self.rest.configure_ldap_settings(ldap_settings)
if not status:
self.fail("Setting LDAP failed")
validate_ldap_event(old_setting, new_setting)
# revert to old ldap settings and check if event is generated
ldap_settings = {"authenticationEnabled": "false", "authorizationEnabled": "false",
"port": 389}
status, content, _ = self.rest.configure_ldap_settings(ldap_settings)
if not status:
self.fail("Reverting LDAP Setting failed")
validate_ldap_event(new_setting, old_setting)
def test_saslauthd_config_changed_event(self):
def validate_saslauthd_event(event, user_event):
# Test NON Extra Attributes fields & NON generic fields
for param, value in user_event.items():
if param == Event.Fields.EXTRA_ATTRS:
continue
if event[param] != value:
self.fail("Value mismatch for '%s'. Expected %s != %s Actual"
% (param, value, event[param]))
# Test generic fields
self.generic_fields_check(event)
# Test Extra Attributes fields
for param in ["old_settings", "new_settings"]:
exp_val = user_event[Event.Fields.EXTRA_ATTRS][param]
act_val = event[Event.Fields.EXTRA_ATTRS][param]
if act_val != exp_val:
self.fail("Mismatch in %s. Expected %s != %s Actual"
% (param, exp_val, act_val))
# enable sasl authd
settings = {"enabled": "true", "admins": "alice,barry", "roAdmins": "clair,daniel"}
status, content, _ = self.rest.configure_sasl_authd(settings)
if not status:
self.fail("Changing saslauthd settings failed {0}".format(content))
# Get the last event
actual_event = self.get_event_from_cluster()
expected_event = SecurityEvents.sasldauth_config_changed(self.cluster.master.ip,
old_enabled=False,
new_enabled=True)
validate_saslauthd_event(event=actual_event, user_event=expected_event)
# disable the saslauthd
status, content, _ = self.rest.configure_sasl_authd({"enabled": "false"})
if not status:
self.fail("disabling saslauthd failed {0}".format(content))
# Get the last event
actual_event = self.get_event_from_cluster()
expected_event = SecurityEvents.sasldauth_config_changed(self.cluster.master.ip,
old_enabled=True,
new_enabled=False)
validate_saslauthd_event(event=actual_event, user_event=expected_event)
def test_security_config_changed_event(self):
_ = self.rest.update_autofailover_settings(False, 120)
shell_conn = RemoteMachineShellConnection(self.cluster.master)
cb_cli = CbCli(shell_conn)
o = cb_cli.enable_n2n_encryption()
self.log.info(o)
shell_conn.disconnect()
self.rest.set_encryption_level(level="control")
settings = {"tlsMinVersion": "tlsv1.1", "clusterEncryptionLevel": "all"}
self.rest.set_security_settings(settings)
old_settings = {"ssl_minimum_protocol": "tlsv1.2", "cluster_encryption_level": "control"}
new_settings = {"ssl_minimum_protocol": "tlsv1.1", "cluster_encryption_level": "all"}
# Get the last event
event = self.get_event_from_cluster()
user_event = SecurityEvents.security_config_changed(self.cluster.master.ip,
old_settings, new_settings)
# Test NON Extra Attributes fields & NON generic fields
for param, value in user_event.items():
if param == Event.Fields.EXTRA_ATTRS:
continue
if event[param] != value:
self.fail("Value mismatch for '%s'. Expected %s != %s Actual"
% (param, value, event[param]))
# Test generic fields
self.generic_fields_check(event)
# Test Extra Attributes fields
for param in ["old_settings", "new_settings"]:
expected_settings = user_event[Event.Fields.EXTRA_ATTRS][param]
actual_settings = event[Event.Fields.EXTRA_ATTRS][param]
for i_param in ["ssl_minimum_protocol", "cluster_encryption_level"]:
act_val = actual_settings[i_param]
exp_val = expected_settings[i_param]
if act_val != exp_val:
self.fail("Mismatch in %s. Expected %s != %s Actual"
% (param, exp_val, act_val))
def test_audit_enabled_event(self):
status = self.rest.setAuditSettings(enabled='true')
if not status:
self.fail("Enabling audit failed")
# Get the audit event
events = self.get_event_from_cluster(last=False)
events.reverse()
event = {}
for eve in events:
self.log.info("description {0}".format(eve["description"]))
if eve["description"] == "Audit enabled":
event = eve
break
user_event = SecurityEvents.audit_enabled(self.cluster.master.ip, enabled_audit_ids=[],
log_path="/opt/couchbase/var/lib/couchbase/logs",
rotate_interval=86400,
rotate_size=20971520)
# Test NON Extra Attributes fields & NON generic fields
for param, value in user_event.items():
if param == Event.Fields.EXTRA_ATTRS:
continue
if event[param] != value:
self.fail("Value mismatch for '%s'. Expected %s != %s Actual"
% (param, value, event[param]))
# Test generic fields
self.generic_fields_check(event)
# Test Extra Attributes fields
# check if mandatory keys in new settings are present
for param in ["enabled_audit_ids", "log_path", "rotate_interval", "rotate_size"]:
if param not in event[Event.Fields.EXTRA_ATTRS]["new_settings"].keys():
self.fail("Param: {0} not present in new settings".format(param))
def test_audit_disabled_event(self):
status = self.rest.setAuditSettings(enabled='true')
if not status:
self.fail("Enabling audit failed")
status = self.rest.setAuditSettings(enabled='false')
if not status:
self.fail("Disabling audit failed")
# Get the audit event
events = self.get_event_from_cluster(last=False)
events.reverse()
event = {}
for eve in events:
if eve["description"] == "Audit disabled":
event = eve
break
user_event = SecurityEvents.audit_disabled(self.cluster.master.ip,
enabled_audit_ids=[],
log_path="/opt/couchbase/var/lib/couchbase/logs",
rotate_interval=86400,
rotate_size=524288000,
sync=[])
# Test NON Extra Attributes fields & NON generic fields
for param, value in user_event.items():
if param == Event.Fields.EXTRA_ATTRS:
continue
if event[param] != value:
self.fail("Value mismatch for '%s'. Expected %s != %s Actual"
% (param, value, event[param]))
# Test generic fields
self.generic_fields_check(event)
# Test Extra Attributes fields
# check if mandatory keys in old settings are present
for param in ["enabled_audit_ids", "log_path", "rotate_interval", "rotate_size", "sync"]:
if param not in event[Event.Fields.EXTRA_ATTRS]["old_settings"].keys():
self.fail("Param: {0} not present in new settings".format(param))
def test_audit_config_changed_event(self):
status = self.rest.setAuditSettings(enabled='true')
if not status:
self.fail("Enabling audit failed")
status = self.rest.setAuditSettings(enabled='true',
rotateInterval=172800)
if not status:
self.fail("Setting audit failed")
# Get the audit event
events = self.get_event_from_cluster(last=False)
events.reverse()
event = {}
for eve in events:
if eve["description"] == "Audit configuration changed":
event = eve
break
user_event = SecurityEvents.audit_setting_changed(self.cluster.master.ip,
old_enabled_audit_ids=[],
old_log_path="",
old_rotate_interval=86400,
old_rotate_size="", sync=[],
new_enabled_audit_ids=[],
new_log_path="",
new_rotate_interval=172800,
new_rotate_size="")
# Test NON Extra Attributes fields & NON generic fields
for param, value in user_event.items():
if param == Event.Fields.EXTRA_ATTRS:
continue
if event[param] != value:
self.fail("Value mismatch for '%s'. Expected %s != %s Actual"
% (param, value, event[param]))
# Test generic fields
self.generic_fields_check(event)
# Test Extra Attributes fields
for param in ["old_settings", "new_settings"]:
expected_settings = user_event[Event.Fields.EXTRA_ATTRS][param]
actual_settings = event[Event.Fields.EXTRA_ATTRS][param]
for i_param in ["rotate_interval"]:
act_val = actual_settings[i_param]
exp_val = expected_settings[i_param]
if act_val != exp_val:
self.fail("Mismatch in %s. Expected %s != %s Actual"
% (param, exp_val, act_val))
def test_upgrade(self):
update_task = None
internal_setting_error = "eventLogsLimit - not supported " \
"in mixed version clusters"
get_events_error = "This http API endpoint isn't " \
"supported in mixed version clusters"
t_durability_level = choice(
[Bucket.DurabilityLevel.NONE, Bucket.DurabilityLevel.MAJORITY])
if self.upgrade_with_data_load:
self.log.info("Continuous doc updates with durability=%s" %
t_durability_level)
update_task = self.task.async_continuous_doc_ops(
self.cluster,
self.bucket,
self.gen_load,
op_type=DocLoading.Bucket.DocOps.UPDATE,
timeout_secs=30,
process_concurrency=1,
durability=t_durability_level)
self.log.info("Upgrading cluster nodes to target version")
node_to_upgrade = self.fetch_node_to_upgrade()
while node_to_upgrade is not None:
self.log.info("Selected node for upgrade: %s" % node_to_upgrade.ip)
self.upgrade_function[self.upgrade_type](node_to_upgrade,
self.upgrade_version)
self.cluster_util.print_cluster_stats(self.cluster)
node_to_upgrade = self.fetch_node_to_upgrade()
# Create custom event to test system event logs
event = DataServiceEvents.bucket_online(self.cluster.master.ip,
"dummy_bucket",
"dummy_uuid")
event.pop(Event.Fields.EXTRA_ATTRS)
event[Event.Fields.UUID] = self.system_events.get_rand_uuid()
event[Event.Fields.TIMESTAMP] = \
self.system_events.get_timestamp_format(datetime.now())
nodes_in_cluster = \
RestConnection(self.cluster.master).get_nodes()
if node_to_upgrade is None \
or self.cluster_supports_system_event_logs:
# Cluster fully upgraded / already supports system event logs
self.log.info("Creating events for validation")
self.system_events.set_test_start_time()
self.system_events.events = list()
col_name = "collection_1"
self.bucket_util.create_collection(
self.cluster.master,
self.bucket,
collection_spec={"name": col_name})
self.bucket_util.drop_collection(self.cluster.master,
self.bucket,
collection_name=col_name)
self.system_events.add_event(
DataServiceEvents.collection_created(
self.cluster.master.ip, self.bucket.name,
CbServer.default_scope, col_name))
self.system_events.add_event(
DataServiceEvents.collection_dropped(
self.cluster.master.ip, self.bucket.name,
CbServer.default_scope, col_name))
event_helper = SystemEventRestHelper([self.cluster.master])
status, content = event_helper.create_event(event)
if status is False:
self.log_failure("Failed to create event: %s" % content)
self.system_events.add_event(event)
for node in nodes_in_cluster:
self.log.info("Validating events on %s" % node.ip)
self.system_events.validate(node)
else:
# Mixed mode cluster testing
self.log.info("Creating custom event to validate "
"and trying to fetch events from cluster")
for node in nodes_in_cluster:
if float(node.version[:3]) >= 7.1:
event_helper = SystemEventRestHelper([node])
status, content = event_helper.create_event(event)
if not status:
self.log_failure(
"Event creation returned '%s':%s" %
(status, content))
events = event_helper.get_events(events_count=-1)
self.log.critical("Events: %s" % events)
if events != get_events_error:
self.log_failure("Unexpected response: %s" %
events)
status, content = event_helper.update_max_events(
max_event_count=CbServer.sys_event_max_logs)
if status:
self.log_failure("Mixed mode - able to update "
"max_events: %s" % content)
elif content["errors"][0] != internal_setting_error:
self.log_failure(
"Error mismatch in mixed-mode: %s" % content)
# Halt further upgrade if test has failed during current upgrade
if self.test_failure is not None:
break
if self.upgrade_with_data_load:
# Wait for update_task to complete
update_task.end_task()
self.task_manager.get_task_result(update_task)
# Perform final collection/doc_count validation
self.validate_test_failure()