def get_health(self, filter_node) -> Dict: """Get nodes and synchronization information. Parameters ---------- filter_node : dict Whether to filter by a node or return all health information. Returns ------- dict Dict object containing nodes information. """ workers_info = {key: val.to_dict() for key, val in self.clients.items() if filter_node is None or filter_node == {} or key in filter_node} n_connected_nodes = len(workers_info) if filter_node is None or self.configuration['node_name'] in filter_node: workers_info.update({self.configuration['node_name']: self.to_dict()}) # Get active agents by node and format last keep alive date format for node_name in workers_info.keys(): workers_info[node_name]["info"]["n_active_agents"] = Agent.get_agents_overview(filters={'status': 'active', 'node_name': node_name})['totalItems'] if workers_info[node_name]['info']['type'] != 'master': workers_info[node_name]['status']['last_keep_alive'] = str( datetime.fromtimestamp(workers_info[node_name]['status']['last_keep_alive'])) return {"n_connected_nodes": n_connected_nodes, "nodes": workers_info}
def get_health(self, filter_node) -> Dict: """ Return healthcheck data :param filter_node: Node to filter by :return: Dictionary """ workers_info = { key: val.to_dict() for key, val in self.clients.items() if filter_node is None or filter_node == {} or key in filter_node } n_connected_nodes = len(workers_info) if filter_node is None or self.configuration[ 'node_name'] in filter_node: workers_info.update( {self.configuration['node_name']: self.to_dict()}) # Get active agents by node and format last keep alive date format for node_name in workers_info.keys(): workers_info[node_name]["info"][ "n_active_agents"] = Agent.get_agents_overview( filters={ 'status': 'active', 'node_name': node_name })['totalItems'] if workers_info[node_name]['info']['type'] != 'master': workers_info[node_name]['status']['last_keep_alive'] = str( datetime.fromtimestamp( workers_info[node_name]['status']['last_keep_alive'])) return {"n_connected_nodes": n_connected_nodes, "nodes": workers_info}
def get_health(self, filter_node) -> Dict: """Get nodes and synchronization information. Parameters ---------- filter_node : dict Whether to filter by a node or return all health information. Returns ------- dict Dict object containing nodes information. """ workers_info = { key: val.to_dict() for key, val in self.clients.items() if filter_node is None or filter_node == {} or key in filter_node } n_connected_nodes = len(workers_info) if filter_node is None or self.configuration[ 'node_name'] in filter_node: workers_info.update( {self.configuration['node_name']: self.to_dict()}) # Get active agents by node and format last keep alive date format active_agents = Agent.get_agents_overview(filters={ 'status': 'active', 'node_name': filter_node })['items'] for agent in active_agents: if (agent_node := agent["node_name"]) in workers_info.keys(): workers_info[agent_node]["info"]["n_active_agents"] = \ workers_info[agent_node]["info"].get("n_active_agents", 0) + 1
def remove_bulk_agents(agent_ids_list: KeysView, logger): """ Removes files created by agents in worker nodes. This function doesn't remove agents from client.keys since the client.keys file is overwritten by the master node. :param agent_ids_list: List of agents ids to remove. :param logger: Logger to use :return: None. """ def remove_agent_file_type(agent_files: List[str]): """ Removes files if they exist :param agent_files: Path regexes of the files to remove :return: None """ for filetype in agent_files: filetype_glob = filetype.format(ossec_path=common.ossec_path, id='*', name='*', ip='*') filetype_agent = {filetype.format(ossec_path=common.ossec_path, id=a['id'], name=a['name'], ip=a['ip']) for a in agent_info} for agent_file in set(glob.iglob(filetype_glob)) & filetype_agent: logger.debug2("Removing {}".format(agent_file)) if os.path.isdir(agent_file): shutil.rmtree(agent_file) else: os.remove(agent_file) if not agent_ids_list: return # the function doesn't make sense if there is no agents to remove logger.info("Removing files from {} agents".format(len(agent_ids_list))) logger.debug("Agents to remove: {}".format(', '.join(agent_ids_list))) # Remove agents in group of 500 elements (so wazuh-db socket is not saturated) for agents_ids_sublist in itertools.zip_longest(*itertools.repeat(iter(agent_ids_list), 500), fillvalue='0'): agents_ids_sublist = list(filter(lambda x: x != '0', agents_ids_sublist)) # Get info from DB agent_info = Agent.get_agents_overview(q=",".join(["id={}".format(i) for i in agents_ids_sublist]), select=['ip', 'id', 'name'], limit=None)['items'] logger.debug2("Removing files from agents {}".format(', '.join(agents_ids_sublist))) files_to_remove = ['{ossec_path}/queue/rootcheck/({name}) {ip}->rootcheck', '{ossec_path}/queue/diff/{name}', '{ossec_path}/queue/agent-groups/{id}', '{ossec_path}/queue/rids/{id}', '{ossec_path}/var/db/agents/{name}-{id}.db'] remove_agent_file_type(files_to_remove) logger.debug2("Removing agent group assigments from database") # remove agent from groups wdb_conn = WazuhDBConnection() query_to_execute = 'global sql delete from belongs where {}'.format(' or '.join([ 'id_agent = {}'.format(agent_id) for agent_id in agents_ids_sublist ])) wdb_conn.run_wdb_command(query_to_execute) logger.info("Agent files removed")
async def process_files_from_worker(self, files_metadata: Dict, decompressed_files_path: str, logger): """Iterate over received files from worker and updates the local ones. Parameters ---------- files_metadata : dict Dictionary containing file metadata (each key is a filepath and each value its metadata). decompressed_files_path : str Filepath of the decompressed received zipfile. logger : Logger object The logger to use. """ async def update_file(name: str, data: Dict): """Update a local file with one received from a worker. The modification date is checked to decide whether to update ir or not. Parameters ---------- name : str Relative path of the file. data : dict Metadata of the file (MD5, merged, etc). """ # Full path full_path, error_updating_file = os.path.join( common.wazuh_path, name), False try: # Only valid client.keys is the local one (master). if os.path.basename(name) == 'client.keys': self.logger.warning( "Client.keys received in a master node") raise exception.WazuhClusterError(3007) # If the file is merged, create individual files from it. if data['merged']: for file_path, file_data, file_time in wazuh.core.cluster.cluster.unmerge_info( data['merge_type'], decompressed_files_path, data['merge_name']): # Destination path. full_unmerged_name = os.path.join( common.wazuh_path, file_path) # Path where to create the file before moving it to the destination path (with safe_move). tmp_unmerged_path = os.path.join( common.wazuh_path, 'queue', 'cluster', self.name, os.path.basename(file_path)) try: agent_id = os.path.basename(file_path) # If the agent does not exist on the master, do not copy its file from the worker. if agent_id not in agent_ids: n_errors['warnings'][data['cluster_item_key']] = 1 \ if n_errors['warnings'].get(data['cluster_item_key']) is None \ else n_errors['warnings'][data['cluster_item_key']] + 1 self.logger.debug2( f"Received group of an non-existent agent '{agent_id}'" ) continue # Format the file_data specified inside the merged file. try: mtime = datetime.strptime( file_time, '%Y-%m-%d %H:%M:%S.%f') except ValueError: mtime = datetime.strptime( file_time, '%Y-%m-%d %H:%M:%S') # If the file already existed, check if it is older than the one to be copied from worker. if os.path.isfile(full_unmerged_name): local_mtime = datetime.utcfromtimestamp( int(os.stat(full_unmerged_name).st_mtime)) if local_mtime > mtime: logger.debug2( f"Receiving an old file ({file_path})") continue # Create file in temporal path and safe move it to the destination path. with open(tmp_unmerged_path, 'wb') as f: f.write(file_data) mtime_epoch = timegm(mtime.timetuple()) utils.safe_move( tmp_unmerged_path, full_unmerged_name, ownership=(common.ossec_uid(), common.ossec_gid()), permissions=self.cluster_items['files'][ data['cluster_item_key']]['permissions'], time=(mtime_epoch, mtime_epoch)) self.integrity_sync_status[ 'total_extra_valid'] += 1 except Exception as e: self.logger.error( f"Error updating agent group/status ({tmp_unmerged_path}): {e}" ) n_errors['errors'][data['cluster_item_key']] = 1 \ if n_errors['errors'].get(data['cluster_item_key']) is None \ else n_errors['errors'][data['cluster_item_key']] + 1 await asyncio.sleep(0.0001) # If the file is not merged, move it directly to the destination path. else: zip_path = os.path.join(decompressed_files_path, name) utils.safe_move(zip_path, full_path, ownership=(common.ossec_uid(), common.ossec_gid()), permissions=self.cluster_items['files'] [data['cluster_item_key']]['permissions']) except exception.WazuhException as e: logger.debug2(f"Warning updating file '{name}': {e}") error_tag = 'warnings' error_updating_file = True except Exception as e: logger.debug2(f"Error updating file '{name}': {e}") error_tag = 'errors' error_updating_file = True if error_updating_file: n_errors[error_tag][data['cluster_item_key']] = 1 if not n_errors[error_tag].get( data['cluster_item_key']) \ else n_errors[error_tag][data['cluster_item_key']] + 1 n_errors = {'errors': {}, 'warnings': {}} # Get ID of all agents. try: agents = Agent.get_agents_overview(select=['name'], limit=None)['items'] agent_ids = set(map(operator.itemgetter('id'), agents)) except Exception as e: logger.debug2(f"Error getting agent ids: {e}") agent_ids = {} # Iterate and update each file specified in 'files_metadata' if conditions are meets. try: for filename, data in files_metadata.items(): await update_file(data=data, name=filename) except Exception as e: self.logger.error(f"Error updating worker files: '{e}'.") raise e # Log errors if any. if sum(n_errors['errors'].values()) > 0: logger.error("Errors updating worker files: {}".format(' | '.join([ '{}: {}'.format(key, value) for key, value in n_errors['errors'].items() ]))) if sum(n_errors['warnings'].values()) > 0: for key, value in n_errors['warnings'].items(): if key == 'queue/agent-groups/': logger.debug2( f"Received {value} group assignments for non-existent agents. Skipping." )
async def process_files_from_worker(self, files_checksums: Dict, decompressed_files_path: str, logger): """ Iterates over received files from worker and updates the local ones :param files_checksums: A dictionary containing file metadata :param decompressed_files_path: Filepath of the decompressed received zipfile :param logger: The logger to use :return: None """ async def update_file(name: str, data: Dict): """ Updates a file from the worker. It checks the modification date to decide whether to update it or not. If it's a merged file, it unmerges it. :param name: Filename to update :param data: File metadata :return: None """ # Full path full_path, error_updating_file, n_merged_files = common.ossec_path + name, False, 0 # Cluster items information: write mode and permissions lock_full_path = "{}/queue/cluster/lockdir/{}.lock".format(common.ossec_path, os.path.basename(full_path)) lock_file = open(lock_full_path, 'a+') try: fcntl.lockf(lock_file, fcntl.LOCK_EX) if os.path.basename(name) == 'client.keys': self.logger.warning("Client.keys received in a master node") raise exception.WazuhClusterError(3007) if data['merged']: self.sync_extra_valid_status['total_extra_valid'] = len(agent_ids) for file_path, file_data, file_time in wazuh.core.cluster.cluster.unmerge_info(data['merge_type'], decompressed_files_path, data['merge_name']): full_unmerged_name = os.path.join(common.ossec_path, file_path) tmp_unmerged_path = os.path.join(common.ossec_path, 'queue/cluster', self.name, os.path.basename(file_path)) try: agent_id = os.path.basename(file_path) if agent_id not in agent_ids: n_errors['warnings'][data['cluster_item_key']] = 1 \ if n_errors['warnings'].get(data['cluster_item_key']) is None \ else n_errors['warnings'][data['cluster_item_key']] + 1 self.logger.debug2("Received group of an non-existent agent '{}'".format(agent_id)) continue try: mtime = datetime.strptime(file_time, '%Y-%m-%d %H:%M:%S.%f') except ValueError: mtime = datetime.strptime(file_time, '%Y-%m-%d %H:%M:%S') if os.path.isfile(full_unmerged_name): local_mtime = datetime.utcfromtimestamp(int(os.stat(full_unmerged_name).st_mtime)) # check if the date is older than the manager's date if local_mtime > mtime: logger.debug2("Receiving an old file ({})".format(file_path)) continue with open(tmp_unmerged_path, 'wb') as f: f.write(file_data) mtime_epoch = timegm(mtime.timetuple()) utils.safe_move(tmp_unmerged_path, full_unmerged_name, ownership=(common.ossec_uid(), common.ossec_gid()), permissions=self.cluster_items['files'][data['cluster_item_key']]['permissions'], time=(mtime_epoch, mtime_epoch) ) except Exception as e: self.logger.error("Error updating agent group/status ({}): {}".format(tmp_unmerged_path, e)) self.sync_extra_valid_status['total_extra_valid'] -= 1 n_errors['errors'][data['cluster_item_key']] = 1 \ if n_errors['errors'].get(data['cluster_item_key']) is None \ else n_errors['errors'][data['cluster_item_key']] + 1 await asyncio.sleep(0.0001) else: zip_path = "{}{}".format(decompressed_files_path, name) utils.safe_move(zip_path, full_path, ownership=(common.ossec_uid(), common.ossec_gid()), permissions=self.cluster_items['files'][data['cluster_item_key']]['permissions'] ) except exception.WazuhException as e: logger.debug2("Warning updating file '{}': {}".format(name, e)) error_tag = 'warnings' error_updating_file = True except Exception as e: logger.debug2("Error updating file '{}': {}".format(name, e)) error_tag = 'errors' error_updating_file = True if error_updating_file: n_errors[error_tag][data['cluster_item_key']] = 1 if not n_errors[error_tag].get( data['cluster_item_key']) \ else n_errors[error_tag][data['cluster_item_key']] + 1 fcntl.lockf(lock_file, fcntl.LOCK_UN) lock_file.close() # tmp path tmp_path = "/queue/cluster/{}/tmp_files".format(self.name) n_merged_files = 0 n_errors = {'errors': {}, 'warnings': {}} # create temporary directory for lock files lock_directory = "{}/queue/cluster/lockdir".format(common.ossec_path) if not os.path.exists(lock_directory): utils.mkdir_with_mode(lock_directory) try: agents = Agent.get_agents_overview(select=['name'], limit=None)['items'] agent_ids = set(map(operator.itemgetter('id'), agents)) except Exception as e: logger.debug2("Error getting agent ids: {}".format(e)) agent_ids = {} try: for filename, data in files_checksums.items(): await update_file(data=data, name=filename) except Exception as e: self.logger.error("Error updating worker files: '{}'.".format(e)) raise e if sum(n_errors['errors'].values()) > 0: logger.error("Errors updating worker files: {}".format(' | '.join( ['{}: {}'.format(key, value) for key, value in n_errors['errors'].items()]) )) if sum(n_errors['warnings'].values()) > 0: for key, value in n_errors['warnings'].items(): if key == '/queue/agent-groups/': logger.debug2("Received {} group assignments for non-existent agents. Skipping.".format(value))
def remove_bulk_agents(agent_ids_list: KeysView, logger): """ Removes files created by agents in worker nodes. This function doesn't remove agents from client.keys since the client.keys file is overwritten by the master node. :param agent_ids_list: List of agents ids to remove. :param logger: Logger to use :return: None. """ def remove_agent_file_type(agent_files: List[str]): """ Removes files if they exist :param agent_files: Path regexes of the files to remove :return: None """ for filetype in agent_files: filetype_glob = filetype.format(ossec_path=common.ossec_path, id='*', name='*', ip='*') filetype_agent = { filetype.format(ossec_path=common.ossec_path, id=a['id'], name=a['name'], ip=a['ip']) for a in agent_info } for agent_file in set( glob.iglob(filetype_glob)) & filetype_agent: logger.debug2("Removing {}".format(agent_file)) if os.path.isdir(agent_file): shutil.rmtree(agent_file) else: os.remove(agent_file) if not agent_ids_list: return # the function doesn't make sense if there is no agents to remove logger.info("Removing files from {} agents".format( len(agent_ids_list))) logger.debug("Agents to remove: {}".format(', '.join(agent_ids_list))) # the agents must be removed in groups of 997: 999 is the limit of SQL variables per query. Limit and offset are # always included in the SQL query, so that leaves 997 variables as limit. for agents_ids_sublist in itertools.zip_longest(*itertools.repeat( iter(agent_ids_list), 997), fillvalue='0'): agents_ids_sublist = list( filter(lambda x: x != '0', agents_ids_sublist)) # Get info from DB agent_info = Agent.get_agents_overview(q=",".join( ["id={}".format(i) for i in agents_ids_sublist]), select=['ip', 'id', 'name'], limit=None)['items'] logger.debug2("Removing files from agents {}".format( ', '.join(agents_ids_sublist))) files_to_remove = [ '{ossec_path}/queue/agent-info/{name}-{ip}', '{ossec_path}/queue/rootcheck/({name}) {ip}->rootcheck', '{ossec_path}/queue/diff/{name}', '{ossec_path}/queue/agent-groups/{id}', '{ossec_path}/queue/rids/{id}', '{ossec_path}/var/db/agents/{name}-{id}.db' ] remove_agent_file_type(files_to_remove) logger.debug2("Removing agent group assigments from database") # remove agent from groups db_global = glob.glob(common.database_path_global) if not db_global: raise WazuhInternalError(1600) conn = Connection(db_global[0]) agent_ids_db = { 'id_agent{}'.format(i): int(i) for i in agents_ids_sublist } conn.execute( 'delete from belongs where {}'.format(' or '.join([ 'id_agent = :{}'.format(i) for i in agent_ids_db.keys() ])), agent_ids_db) conn.commit() # Tell wazuhbd to delete agent database wdb_conn = WazuhDBConnection() wdb_conn.delete_agents_db(agents_ids_sublist) logger.info("Agent files removed")
# - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/var/ossec/framework/lib from sys import path, exit import json # cwd = /var/ossec/api/framework/examples #framework_path = '{0}'.format(path[0][:-9]) # cwd = /var/ossec/api #framework_path = '{0}/framework'.format(path[0]) # Default path framework_path = '/var/ossec/api/framework' path.append(framework_path) try: from wazuh import Wazuh from wazuh.core.agent import Agent except Exception as e: print("No module 'wazuh' found.") exit() if __name__ == "__main__": # Creating wazuh object # It is possible to specify the ossec path (path argument) or get /etc/ossec-init.conf (get_init argument) print("\nWazuh:") myWazuh = Wazuh() print(myWazuh) print("\nAgents:") agents = Agent.get_agents_overview() print(json.dumps(agents, indent=4, sort_keys=True))