def validate(self, server, since_time=None, events_count=-1): """ Validates the cluster events against the given events_list :param server: Server from which we can get the cluster events :param since_time: Start time from which the events should be fetched :param events_count: Number of events to be fetched. '-1' means all :return list: List containing failures """ def extract_req_values(cluster_event, local_event): output_dict = dict() for key in local_event.keys(): if key not in cluster_event: continue if type(local_event[key]) is dict: output_dict[key] = extract_req_values(cluster_event[key], local_event[key]) elif type(cluster_event[key]) is list: cluster_event[key].sort() output_dict[key] = cluster_event[key] local_event[key].sort() else: output_dict[key] = cluster_event[key] return output_dict failures = list() # If nothing stored in event list, nothing to validate if not self.events: return failures rest = SystemEventRestHelper([server]) # Fetch all events from the server for generic validation events = rest.get_events(server=server, events_count=-1)["events"] # Check for event_id duplications if self.__duplicate_event_ids_present(events, failures): failures.append("Duplicate event_ids seen") # Fetch events from the cluster and validate against the ones the # test has recorded v_index = 0 events = rest.get_events(server=server, since_time=since_time, events_count=events_count)["events"] for event in events: if isinstance(self.events[v_index], list): # Process events which occurred in parallel from test's POV # TODO: Write a algo to find the match from the subset pass else: dict_to_compare = extract_req_values(event, self.events[v_index]) if dict_to_compare == self.events[v_index]: v_index += 1 if v_index == self.__event_counter.counter \ or v_index == EventHelper.max_events: break else: failures.append("Unable to validate event: %s" % self.events[v_index]) return failures
def setUp(self): super(QuerySystemEventLogs, self).setUp() self.shell = RemoteMachineShellConnection(self.master) self.info = self.shell.extract_remote_info() if self.info.type.lower() == 'windows': self.curl_path = f"{self.path}curl" else: self.curl_path = "curl" self.event_rest = SystemEventRestHelper([self.master])
class QuerySystemEventLogs(QueryTests): def setUp(self): super(QuerySystemEventLogs, self).setUp() self.shell = RemoteMachineShellConnection(self.master) self.info = self.shell.extract_remote_info() if self.info.type.lower() == 'windows': self.curl_path = f"{self.path}curl" else: self.curl_path = "curl" self.event_rest = SystemEventRestHelper([self.master]) def tearDown(self): super(QuerySystemEventLogs, self).tearDown() def test_memory_quota(self): event_seen = False log_fields = [ "uuid", "component", "event_id", "description", "severity", "timestamp", "extra_attributes", "node" ] query = "'statement=select (select * from `default`)&memory_quota=1'" curl_output = self.shell.execute_command( f"{self.curl_path} -X POST -u {self.rest.username}:{self.rest.password} http://{self.master.ip}:{self.n1ql_port}/query/service -d {query}" ) self.log.info(curl_output) output = self.convert_list_to_json(curl_output[0]) requestID = output['requestID'] events = self.event_rest.get_events(server=self.master, events_count=-1)["events"] for event in events: if event['event_id'] == 1026: event_seen = True for field in log_fields: self.assertTrue( field in event.keys(), f"Field {field} is not in the event and it should be, please check the event {event}" ) self.assertEqual(event['component'], "query") self.assertEqual(event['description'], "Request memory quota exceeded") self.assertEqual(event['severity'], "info") self.assertEqual(event['node'], self.master.ip) self.assertEqual(event['extra_attributes']['request-id'], requestID) self.assertTrue( event_seen, f"We did not see the event id we were looking for: {events}") def test_change_settings(self): event_seen = False log_fields = [ "uuid", "component", "event_id", "description", "severity", "timestamp", "extra_attributes", "node" ] settings = '{"auto-prepare":true,"completed-limit":5000,"controls":false}' curl_output = self.shell.execute_command( f"{self.curl_path} -X POST -u {self.rest.username}:{self.rest.password} http://{self.servers[1].ip}:{self.n1ql_port}/admin/settings -d '{settings}'" ) self.log.info(curl_output) events = self.event_rest.get_events(server=self.master, events_count=-1)["events"] for event in events: if event['event_id'] == 1025: if "None" in str(event): pass else: for field in log_fields: self.assertTrue( field in event.keys(), f"Field {field} is not in the event and it should be, please check the event {event}" ) self.assertEqual(event['component'], "query") self.assertEqual(event['description'], "Configuration changed") self.assertEqual(event['severity'], "info") settings_changed = event['extra_attributes'] # Now check the event we generated event_seen = True for setting in settings_changed: if setting == "auto-prepare": expected_setting = {'from': False, 'to': True} elif setting == "completed-limit": expected_setting = {'from': 4000, 'to': 5000} else: self.fail( f"Unrecognized setting {setting} which should not have been changed! Please check the event {event}" ) self.assertEqual(event['extra_attributes'][setting], expected_setting) self.assertEqual(event['node'], self.servers[1].ip) self.assertTrue( event_seen, f"We did not see the event id we were looking for: {events}")
def setUp(self): super(SystemEventLogs, self).setUp() self.rest = RestConnection(self.cluster.master) self.event_rest_helper = \ SystemEventRestHelper(self.cluster.nodes_in_cluster) self.log_setup_status("SystemEventLogs", "started")
class SystemEventLogs(ClusterSetup): def setUp(self): super(SystemEventLogs, self).setUp() self.rest = RestConnection(self.cluster.master) self.event_rest_helper = \ SystemEventRestHelper(self.cluster.nodes_in_cluster) self.log_setup_status("SystemEventLogs", "started") def tearDown(self): self.log_setup_status("SystemEventLogs", "started", stage="tearDown") self.log_setup_status("SystemEventLogs", "completed", stage="tearDown") super(SystemEventLogs, self).tearDown() def get_event_from_cluster(self, last=True): events = self.event_rest_helper.get_events( server=self.cluster.master, events_count=-1)["events"] if last: return events[-1] else: return events def generic_fields_check(self, event): event_keys = event.keys() for param in [Event.Fields.TIMESTAMP, Event.Fields.UUID]: if param not in event_keys: self.fail("%s key missing in event" % param) @staticmethod def get_default_password_settings(): return { "minLength": 6, "must_present": [] } def test_user_crud_events(self): domain = "local" username = "******" payload = "name=" + username + "&roles=admin" for action in ["create", "delete"]: if action == "create": self.log.info("Creating user") self.rest.add_set_builtin_user(username, payload) user_event = SecurityEvents.user_added(self.cluster.master.ip, "local") else: self.log.info("Deleting user") self.rest.delete_builtin_user(username) user_event = SecurityEvents.user_deleted(self.cluster.master.ip, "local") # Get the last user created/deleted event event = self.get_event_from_cluster() # Test NON Extra Attributes fields & NON generic fields for param, value in user_event.items(): if param == Event.Fields.EXTRA_ATTRS: continue if event[param] != value: self.fail("Value mismatch for '%s'. Expected %s != %s Actual" % (param, value, event[param])) # Test generic fields self.generic_fields_check(event) # Test Extra Attributes fields if event[Event.Fields.EXTRA_ATTRS]["user"] == username: self.fail("Username got printed in system event log") if event[Event.Fields.EXTRA_ATTRS]["domain"] != domain: self.fail("Domain mismatch. Expected %s != %s Actual" % (event[Event.Fields.EXTRA_ATTRS]["domain"], domain)) def test_group_crud_events(self): groupname = "cbadmingroup" payload = "roles=admin" for action in ["create", "delete"]: if action == "create": self.log.info("Creating group") self.rest.add_set_bulitin_group(groupname, payload) user_event = SecurityEvents.group_added(self.cluster.master.ip) else: self.log.info("Deleting group") self.rest.delete_builtin_group(groupname) user_event = SecurityEvents.group_deleted(self.cluster.master.ip) # Get the last event event = self.get_event_from_cluster() # Test NON Extra Attributes fields & NON generic fields for param, value in user_event.items(): if param == Event.Fields.EXTRA_ATTRS: continue if event[param] != value: self.fail("Value mismatch for '%s'. Expected %s != %s Actual" % (param, value, event[param])) # Test generic fields self.generic_fields_check(event) # Test Extra Attributes fields if event[Event.Fields.EXTRA_ATTRS]["group"] == groupname: self.fail("Groupname got printed in system event log") def test_password_policy_changed_event(self): default_settings = self.get_default_password_settings() status, content = self.rest.change_password_policy(min_length=8, enforce_digits="true") if not status: self.fail("Changing password policy failed with {0}".format(content)) new_settings = copy.deepcopy(default_settings) new_settings["minLength"] = 8 new_settings["must_present"] = ["digits"] # Get the last event event = self.get_event_from_cluster() user_event = SecurityEvents.password_policy_changed(self.cluster.master.ip, old_min_length= default_settings["minLength"], old_must_present= default_settings["must_present"], new_min_length= new_settings["minLength"], new_must_present= new_settings["must_present"]) # Test NON Extra Attributes fields & NON generic fields for param, value in user_event.items(): if param == Event.Fields.EXTRA_ATTRS: continue if event[param] != value: self.fail("Value mismatch for '%s'. Expected %s != %s Actual" % (param, value, event[param])) # Test generic fields self.generic_fields_check(event) # Test Extra Attributes fields for param in ["old_settings", "new_settings"]: exp_val = user_event[Event.Fields.EXTRA_ATTRS][param] act_val = event[Event.Fields.EXTRA_ATTRS][param] if act_val != exp_val: self.fail("Mismatch in %s. Expected %s != %s Actual" % (param, exp_val, act_val)) def test_ldap_config_changed_event(self): def validate_ldap_event(old_settings, new_settings): old_settings_copy = copy.deepcopy(old_settings) new_settings_copy = copy.deepcopy(new_settings) old_settings_copy["clientTLSCert"] = "redacted" old_settings_copy["cacert"] = "redacted" old_settings_copy["bindDN"] = "redacted" new_settings_copy["clientTLSCert"] = "redacted" new_settings_copy["cacert"] = "redacted" new_settings_copy["bindDN"] = "redacted" for key, val in old_settings_copy.items(): if val == "false": old_settings_copy[key] = False elif val == "true": old_settings_copy[key] = True for key, val in new_settings_copy.items(): if val == "false": new_settings_copy[key] = False elif val == "true": new_settings_copy[key] = True # Get the last event event = self.get_event_from_cluster() user_event = SecurityEvents.ldap_config_changed(self.cluster.master.ip, old_settings_copy, new_settings_copy) # Test NON Extra Attributes fields & NON generic fields for param, value in user_event.items(): if param == Event.Fields.EXTRA_ATTRS: continue if event[param] != value: self.fail("Value mismatch for '%s'. Expected %s != %s Actual" % (param, value, event[param])) # Test generic fields self.generic_fields_check(event) # Test Extra Attributes fields for param in ["old_settings", "new_settings"]: exp_val = user_event[Event.Fields.EXTRA_ATTRS][param] act_val = event[Event.Fields.EXTRA_ATTRS][param] if act_val != exp_val: self.fail("Mismatch in %s. Expected %s != %s Actual" % (param, exp_val, act_val)) old_setting = {"authenticationEnabled": "false", "authorizationEnabled": "false", "bindDN": "", "bindPass": "", "cacheValueLifetime": 300000, "encryption": "None", "failOnMaxDepth": "false", "hosts": [], "maxCacheSize": 10000, "maxParallelConnections": 100, "nestedGroupsEnabled": "false", "nestedGroupsMaxDepth": 10, "port": 389, "requestTimeout": 5000, "serverCertValidation": "true", "userDNMapping": "None"} new_setting = {"authenticationEnabled": "true", "authorizationEnabled": "true", "bindDN": "", "bindPass": "", "cacheValueLifetime": 300000, "encryption": "None", "failOnMaxDepth": "false", "hosts": [], "maxCacheSize": 10000, "maxParallelConnections": 100, "nestedGroupsEnabled": "false", "nestedGroupsMaxDepth": 10, "port": 390, "requestTimeout": 5000, "serverCertValidation": "true", "userDNMapping": "None"} ldap_settings = {"authenticationEnabled": "true", "authorizationEnabled": "true", "port": 390} status, content, _ = self.rest.configure_ldap_settings(ldap_settings) if not status: self.fail("Setting LDAP failed") validate_ldap_event(old_setting, new_setting) # revert to old ldap settings and check if event is generated ldap_settings = {"authenticationEnabled": "false", "authorizationEnabled": "false", "port": 389} status, content, _ = self.rest.configure_ldap_settings(ldap_settings) if not status: self.fail("Reverting LDAP Setting failed") validate_ldap_event(new_setting, old_setting) def test_saslauthd_config_changed_event(self): def validate_saslauthd_event(event, user_event): # Test NON Extra Attributes fields & NON generic fields for param, value in user_event.items(): if param == Event.Fields.EXTRA_ATTRS: continue if event[param] != value: self.fail("Value mismatch for '%s'. Expected %s != %s Actual" % (param, value, event[param])) # Test generic fields self.generic_fields_check(event) # Test Extra Attributes fields for param in ["old_settings", "new_settings"]: exp_val = user_event[Event.Fields.EXTRA_ATTRS][param] act_val = event[Event.Fields.EXTRA_ATTRS][param] if act_val != exp_val: self.fail("Mismatch in %s. Expected %s != %s Actual" % (param, exp_val, act_val)) # enable sasl authd settings = {"enabled": "true", "admins": "alice,barry", "roAdmins": "clair,daniel"} status, content, _ = self.rest.configure_sasl_authd(settings) if not status: self.fail("Changing saslauthd settings failed {0}".format(content)) # Get the last event actual_event = self.get_event_from_cluster() expected_event = SecurityEvents.sasldauth_config_changed(self.cluster.master.ip, old_enabled=False, new_enabled=True) validate_saslauthd_event(event=actual_event, user_event=expected_event) # disable the saslauthd status, content, _ = self.rest.configure_sasl_authd({"enabled": "false"}) if not status: self.fail("disabling saslauthd failed {0}".format(content)) # Get the last event actual_event = self.get_event_from_cluster() expected_event = SecurityEvents.sasldauth_config_changed(self.cluster.master.ip, old_enabled=True, new_enabled=False) validate_saslauthd_event(event=actual_event, user_event=expected_event) def test_security_config_changed_event(self): _ = self.rest.update_autofailover_settings(False, 120) shell_conn = RemoteMachineShellConnection(self.cluster.master) cb_cli = CbCli(shell_conn) o = cb_cli.enable_n2n_encryption() self.log.info(o) shell_conn.disconnect() self.rest.set_encryption_level(level="control") settings = {"tlsMinVersion": "tlsv1.1", "clusterEncryptionLevel": "all"} self.rest.set_security_settings(settings) old_settings = {"ssl_minimum_protocol": "tlsv1.2", "cluster_encryption_level": "control"} new_settings = {"ssl_minimum_protocol": "tlsv1.1", "cluster_encryption_level": "all"} # Get the last event event = self.get_event_from_cluster() user_event = SecurityEvents.security_config_changed(self.cluster.master.ip, old_settings, new_settings) # Test NON Extra Attributes fields & NON generic fields for param, value in user_event.items(): if param == Event.Fields.EXTRA_ATTRS: continue if event[param] != value: self.fail("Value mismatch for '%s'. Expected %s != %s Actual" % (param, value, event[param])) # Test generic fields self.generic_fields_check(event) # Test Extra Attributes fields for param in ["old_settings", "new_settings"]: expected_settings = user_event[Event.Fields.EXTRA_ATTRS][param] actual_settings = event[Event.Fields.EXTRA_ATTRS][param] for i_param in ["ssl_minimum_protocol", "cluster_encryption_level"]: act_val = actual_settings[i_param] exp_val = expected_settings[i_param] if act_val != exp_val: self.fail("Mismatch in %s. Expected %s != %s Actual" % (param, exp_val, act_val)) def test_audit_enabled_event(self): status = self.rest.setAuditSettings(enabled='true') if not status: self.fail("Enabling audit failed") # Get the audit event events = self.get_event_from_cluster(last=False) events.reverse() event = {} for eve in events: self.log.info("description {0}".format(eve["description"])) if eve["description"] == "Audit enabled": event = eve break user_event = SecurityEvents.audit_enabled(self.cluster.master.ip, enabled_audit_ids=[], log_path="/opt/couchbase/var/lib/couchbase/logs", rotate_interval=86400, rotate_size=20971520) # Test NON Extra Attributes fields & NON generic fields for param, value in user_event.items(): if param == Event.Fields.EXTRA_ATTRS: continue if event[param] != value: self.fail("Value mismatch for '%s'. Expected %s != %s Actual" % (param, value, event[param])) # Test generic fields self.generic_fields_check(event) # Test Extra Attributes fields # check if mandatory keys in new settings are present for param in ["enabled_audit_ids", "log_path", "rotate_interval", "rotate_size"]: if param not in event[Event.Fields.EXTRA_ATTRS]["new_settings"].keys(): self.fail("Param: {0} not present in new settings".format(param)) def test_audit_disabled_event(self): status = self.rest.setAuditSettings(enabled='true') if not status: self.fail("Enabling audit failed") status = self.rest.setAuditSettings(enabled='false') if not status: self.fail("Disabling audit failed") # Get the audit event events = self.get_event_from_cluster(last=False) events.reverse() event = {} for eve in events: if eve["description"] == "Audit disabled": event = eve break user_event = SecurityEvents.audit_disabled(self.cluster.master.ip, enabled_audit_ids=[], log_path="/opt/couchbase/var/lib/couchbase/logs", rotate_interval=86400, rotate_size=524288000, sync=[]) # Test NON Extra Attributes fields & NON generic fields for param, value in user_event.items(): if param == Event.Fields.EXTRA_ATTRS: continue if event[param] != value: self.fail("Value mismatch for '%s'. Expected %s != %s Actual" % (param, value, event[param])) # Test generic fields self.generic_fields_check(event) # Test Extra Attributes fields # check if mandatory keys in old settings are present for param in ["enabled_audit_ids", "log_path", "rotate_interval", "rotate_size", "sync"]: if param not in event[Event.Fields.EXTRA_ATTRS]["old_settings"].keys(): self.fail("Param: {0} not present in new settings".format(param)) def test_audit_config_changed_event(self): status = self.rest.setAuditSettings(enabled='true') if not status: self.fail("Enabling audit failed") status = self.rest.setAuditSettings(enabled='true', rotateInterval=172800) if not status: self.fail("Setting audit failed") # Get the audit event events = self.get_event_from_cluster(last=False) events.reverse() event = {} for eve in events: if eve["description"] == "Audit configuration changed": event = eve break user_event = SecurityEvents.audit_setting_changed(self.cluster.master.ip, old_enabled_audit_ids=[], old_log_path="", old_rotate_interval=86400, old_rotate_size="", sync=[], new_enabled_audit_ids=[], new_log_path="", new_rotate_interval=172800, new_rotate_size="") # Test NON Extra Attributes fields & NON generic fields for param, value in user_event.items(): if param == Event.Fields.EXTRA_ATTRS: continue if event[param] != value: self.fail("Value mismatch for '%s'. Expected %s != %s Actual" % (param, value, event[param])) # Test generic fields self.generic_fields_check(event) # Test Extra Attributes fields for param in ["old_settings", "new_settings"]: expected_settings = user_event[Event.Fields.EXTRA_ATTRS][param] actual_settings = event[Event.Fields.EXTRA_ATTRS][param] for i_param in ["rotate_interval"]: act_val = actual_settings[i_param] exp_val = expected_settings[i_param] if act_val != exp_val: self.fail("Mismatch in %s. Expected %s != %s Actual" % (param, exp_val, act_val))
def test_upgrade(self): update_task = None internal_setting_error = "eventLogsLimit - not supported " \ "in mixed version clusters" get_events_error = "This http API endpoint isn't " \ "supported in mixed version clusters" t_durability_level = choice( [Bucket.DurabilityLevel.NONE, Bucket.DurabilityLevel.MAJORITY]) if self.upgrade_with_data_load: self.log.info("Continuous doc updates with durability=%s" % t_durability_level) update_task = self.task.async_continuous_doc_ops( self.cluster, self.bucket, self.gen_load, op_type=DocLoading.Bucket.DocOps.UPDATE, timeout_secs=30, process_concurrency=1, durability=t_durability_level) self.log.info("Upgrading cluster nodes to target version") node_to_upgrade = self.fetch_node_to_upgrade() while node_to_upgrade is not None: self.log.info("Selected node for upgrade: %s" % node_to_upgrade.ip) self.upgrade_function[self.upgrade_type](node_to_upgrade, self.upgrade_version) self.cluster_util.print_cluster_stats(self.cluster) node_to_upgrade = self.fetch_node_to_upgrade() # Create custom event to test system event logs event = DataServiceEvents.bucket_online(self.cluster.master.ip, "dummy_bucket", "dummy_uuid") event.pop(Event.Fields.EXTRA_ATTRS) event[Event.Fields.UUID] = self.system_events.get_rand_uuid() event[Event.Fields.TIMESTAMP] = \ self.system_events.get_timestamp_format(datetime.now()) nodes_in_cluster = \ RestConnection(self.cluster.master).get_nodes() if node_to_upgrade is None \ or self.cluster_supports_system_event_logs: # Cluster fully upgraded / already supports system event logs self.log.info("Creating events for validation") self.system_events.set_test_start_time() self.system_events.events = list() col_name = "collection_1" self.bucket_util.create_collection( self.cluster.master, self.bucket, collection_spec={"name": col_name}) self.bucket_util.drop_collection(self.cluster.master, self.bucket, collection_name=col_name) self.system_events.add_event( DataServiceEvents.collection_created( self.cluster.master.ip, self.bucket.name, CbServer.default_scope, col_name)) self.system_events.add_event( DataServiceEvents.collection_dropped( self.cluster.master.ip, self.bucket.name, CbServer.default_scope, col_name)) event_helper = SystemEventRestHelper([self.cluster.master]) status, content = event_helper.create_event(event) if status is False: self.log_failure("Failed to create event: %s" % content) self.system_events.add_event(event) for node in nodes_in_cluster: self.log.info("Validating events on %s" % node.ip) self.system_events.validate(node) else: # Mixed mode cluster testing self.log.info("Creating custom event to validate " "and trying to fetch events from cluster") for node in nodes_in_cluster: if float(node.version[:3]) >= 7.1: event_helper = SystemEventRestHelper([node]) status, content = event_helper.create_event(event) if not status: self.log_failure( "Event creation returned '%s':%s" % (status, content)) events = event_helper.get_events(events_count=-1) self.log.critical("Events: %s" % events) if events != get_events_error: self.log_failure("Unexpected response: %s" % events) status, content = event_helper.update_max_events( max_event_count=CbServer.sys_event_max_logs) if status: self.log_failure("Mixed mode - able to update " "max_events: %s" % content) elif content["errors"][0] != internal_setting_error: self.log_failure( "Error mismatch in mixed-mode: %s" % content) # Halt further upgrade if test has failed during current upgrade if self.test_failure is not None: break if self.upgrade_with_data_load: # Wait for update_task to complete update_task.end_task() self.task_manager.get_task_result(update_task) # Perform final collection/doc_count validation self.validate_test_failure()