class EventgenParser: """ This class represents the entire eventgen.conf file and handles parsing mechanism of eventgen and the rules. Args: addon_path (str): Path to the Splunk App """ conf_name = " " def __init__(self, addon_path, config_path=None): self._app = App(addon_path, python_analyzer_enable=False) self.config_path = config_path self._eventgen = None self.addon_path = addon_path self.match_stanzas = set() @property def path_to_samples(self): if os.path.exists(os.path.join(self.config_path, "samples")): LOGGER.info( "Samples path is: {}".format(os.path.join(self.config_path, "samples")) ) return os.path.join(self.config_path, "samples") elif os.path.exists( os.path.join( os.path.abspath(os.path.join(self.config_path, os.pardir)), "samples" ) ): LOGGER.info( "Samples path is: {}".format( os.path.join( os.path.abspath(os.path.join(self.config_path, os.pardir)), "samples", ) ) ) return os.path.join( os.path.abspath(os.path.join(self.config_path, os.pardir)), "samples" ) else: LOGGER.info( "Samples path is: {}".format(os.path.join(self.addon_path, "samples")) ) return os.path.join(self.addon_path, "samples") @property def eventgen(self): try: relative_path = os.path.relpath(self.config_path, self.addon_path) if os.path.exists( os.path.join(self.config_path, "pytest-splunk-addon-data.conf") ): self._eventgen = self._app.get_config( "pytest-splunk-addon-data.conf", dir=relative_path ) self.conf_name = "psa-data-gen" path = self._app.get_filename( relative_path, "pytest-splunk-addon-data.conf" ) elif os.path.exists(os.path.join(self.config_path, "eventgen.conf")): self._eventgen = self._app.get_config( "eventgen.conf", dir=relative_path ) self.conf_name = "eventgen" path = self._app.get_filename(relative_path, "eventgen.conf") else: self._eventgen = self._app.get_config("eventgen.conf") self.conf_name = "eventgen" path = self._app.get_filename("default", "eventgen.conf") LOGGER.info( "Using Eventgen path: {e}\nUsing Conf file name: {c}".format( e=path, c=self.conf_name ) ) return self._eventgen except OSError: LOGGER.warning("pytest-splunk-addon-data.conf/eventgen.conf not Found") raise FileNotFoundError( "pytest-splunk-addon-data.conf/eventgen.conf not Found" ) def get_sample_stanzas(self): """ Converts a stanza in eventgen.conf to an object of SampleStanza. Yields: SampleStanza Object """ eventgen_dict = self.get_eventgen_stanzas() self.check_samples() for sample_name, stanza_params in sorted(eventgen_dict.items()): sample_path = os.path.join(self.path_to_samples, sample_name) yield SampleStanza( sample_path, stanza_params, ) def get_eventgen_stanzas(self): """ Parses the eventgen.conf file and converts it into a dictionary. Format:: { "sample_file_name": # Not Stanza name { "input_type": "str", "tokens": { 1: { token: #One# replacementType: random replacement: static } } } } Return: Dictionary representing eventgen.conf in the above format. """ eventgen_dict = {} if os.path.exists(self.path_to_samples): for sample_file in os.listdir(self.path_to_samples): for stanza in sorted(self.eventgen.sects): stanza_match_obj = re.search(stanza, sample_file) if stanza_match_obj and stanza_match_obj.group(0) == sample_file: self.match_stanzas.add(stanza) eventgen_sections = self.eventgen.sects[stanza] eventgen_dict.setdefault((sample_file), {"tokens": {}}) for stanza_param in eventgen_sections.options: eventgen_property = eventgen_sections.options[stanza_param] if eventgen_property.name.startswith("token"): _, token_id, token_param = eventgen_property.name.split( "." ) token_key = "{}_{}".format(stanza, token_id) if ( not token_key in eventgen_dict[sample_file]["tokens"].keys() ): eventgen_dict[sample_file]["tokens"][token_key] = {} eventgen_dict[sample_file]["tokens"][token_key][ token_param ] = eventgen_property.value else: eventgen_dict[sample_file][ eventgen_property.name ] = eventgen_property.value return eventgen_dict def check_samples(self): """ Gives a user warning when sample file is not found for the stanza peresent in the configuration file. """ if os.path.exists(self.path_to_samples): for stanza in self.eventgen.sects: if stanza not in self.match_stanzas: raise_warning("No sample file found for stanza : {}".format(stanza)) LOGGER.info("Sample file found for stanza : {}".format(stanza))
class UpdateEventgen: """Update eventgen file""" def __init__(self, addon_path): self._app = App(addon_path, python_analyzer_enable=False) self._eventgen = None self.path_to_samples = os.path.join(addon_path, "samples") @property def eventgen(self): try: if not self._eventgen: self._eventgen = self._app.get_config("eventgen.conf") return self._eventgen except OSError: LOGGER.error("Eventgen.conf not found") raise Exception("Eventgen.conf not found") def get_eventgen_stanzas(self): """ To get eventgen stanza and create a dictionary. If stanza contains regex for multiple sample files, then it creates stanza for each sample file. Return: eventgen_dict (dict): { "stanza_name": { "other metadata": "source, sourcetype, etc." "sample_count" : int "tokens": { 0: { token: #One# replacementType: random replacement: static } } } } """ eventgen_dict = {} for stanza in self.eventgen.sects: eventgen_sections = self.eventgen.sects[stanza] eventgen_dict.setdefault( (stanza), { "tokens": {}, }, ) try: events_in_file = len( open(os.path.join(self.path_to_samples, stanza)).readlines()) eventgen_dict[stanza]["sample_count"] = events_in_file except: pass for stanza_param in eventgen_sections.options: eventgen_property = eventgen_sections.options[stanza_param] if eventgen_property.name.startswith("token"): _, token_id, token_param = eventgen_property.name.split( ".") if not token_id in eventgen_dict[stanza]["tokens"].keys(): eventgen_dict[stanza]["tokens"][token_id] = {} eventgen_dict[stanza]["tokens"][token_id][ token_param] = eventgen_property.value else: eventgen_dict[stanza][ eventgen_property.name] = eventgen_property.value for sample_file in os.listdir(self.path_to_samples): if re.search(stanza, sample_file): events_in_file = len( open(os.path.join(self.path_to_samples, sample_file)).readlines()) if sample_file not in eventgen_dict.keys(): eventgen_dict.setdefault((sample_file), {}) eventgen_dict[sample_file][ "sample_count"] = events_in_file eventgen_dict[sample_file]["add_comment"] = True eventgen_dict[sample_file]["tokens"] = {} return eventgen_dict # update the stanzas in dict def update_eventgen_stanzas(self, eventgen_dict): """ Updates the eventgen_dict by adding new metadata New Metadata: ["input_type", "host_type", "sourcetype_to_search", "timestamp_type"] And update the tokens if possible based on the new Data-Generator rules. Input: eventgen_dict (dict) : eventgen dictionary in following format. Return: eventgen_dict (dict): Updated Eventgen stanzas dictionary """ metadata = [ "input_type", "host_type", "sourcetype_to_search", "timestamp_type" ] review_comments = { "metadata": "#REVIEW : Update metadata as per addon's requirement", "replacement": "# REVIEW : Possible value in list : ", "field": "# REVIEW : Check if the field is extracted from the events, else remove this field parameter", "mapping": "# REVIEW : Please check if it can be replace with %s rule", "sample_count": "# REVIEW : Please check for the events per stanza and update sample_count accordingly", } for stanza_name, stanza_data in eventgen_dict.items(): # adding metadata for data in metadata: eventgen_dict[stanza_name][data] = ( f"<<{data}>> " f"{review_comments['metadata']}") if eventgen_dict[stanza_name].get("index"): eventgen_dict[stanza_name]["index"] = ( f"{eventgen_dict[stanza_name]['index']} " f"{review_comments['metadata']}") eventgen_dict[stanza_name]["source"] = eventgen_dict[stanza_name].get( "source", f"pytest-splunk-addon:{eventgen_dict[stanza_name]['input_type']}", ) for _, token_data in stanza_data.get("tokens", {}).items(): token_name = token_data.get("token").strip("#()").lower() for _, new_token_values in FIELD_MAPPING.items(): if token_name in new_token_values.get("token"): new_replacement_type = new_token_values.get( "replacementType") new_replacement = new_token_values.get("replacement") token_data["replacementType"] = new_replacement_type token_data["replacement"] = new_replacement if new_token_values.get("possible_replacement"): token_data["replacement"] = ( f"{new_replacement} " f"{review_comments['replacement']} " f"{new_token_values.get('possible_replacement')}" ) if new_token_values.get("field"): token_data["field"] = ( f"{new_token_values.get('field')} " f"{review_comments['field']}") if token_data.get("replacementType").lower() == "timestamp": token_data["field"] = f"_time {review_comments['field']}" elif token_data.get("replacementType").lower() in [ "file", "mvfile" ]: file_name = (token_data.get("replacement").split("/") [-1].split(":")[0]) token_data[ "replacement"] = f"file[{token_data.get('replacement')}]" token_data["replacementType"] = "random" for key_fields, mapped_files in FILE_MAPPING.items(): replacement_type = FIELD_MAPPING.get(key_fields).get( "replacementType") replacement = FIELD_MAPPING.get(key_fields).get( "replacement") replacement_type_values = FIELD_MAPPING.get( key_fields).get("possible_replacement") field_value = FIELD_MAPPING.get(key_fields).get( "field") if file_name in mapped_files: if "SA-Eventgen" in token_data["replacement"]: token_data["replacementType"] = ( f"{replacement_type} " f"{review_comments['mapping']%key_fields}") token_data["replacement"] = ( f"{replacement} " f"{review_comments['mapping']%key_fields}") if replacement_type_values: token_data["replacement"] = ( f"{token_data['replacement']} " f"{review_comments['replacement']} " f"{replacement_type_values}") if field_value: token_data["field"] = ( f"{field_value} " f"{review_comments['mapping']%key_fields}") # for assigning sample_count at the end of metadata if eventgen_dict.get(stanza_name).get("sample_count"): event_count = eventgen_dict[stanza_name].pop("sample_count") eventgen_dict[stanza_name]["sample_count"] = ( f"{event_count}" f" {review_comments['sample_count']}") # for assigning tokens at the end of metadata if eventgen_dict.get(stanza_name).get("tokens"): token_dict = eventgen_dict[stanza_name].pop("tokens") eventgen_dict[stanza_name]["tokens"] = token_dict return eventgen_dict def create_new_eventgen(self, updated_eventgen_dict, new_conf_path): """ Writes the new values in a new conf file params: updated_eventgen_dict (dict) : Containing all the new values for eventgen.conf new_conf_path : file path for creating new conf file """ with open(new_conf_path, "w") as new_eventgen: LOGGER.info("created new file {}".format(new_conf_path)) # writing file metadata in new eventgen file comment = "## Stanza gets metadata from main stanza" for file_metadata in self.eventgen.headers: new_eventgen.write(file_metadata + "\n") for stanza_name, stanza_data in updated_eventgen_dict.items(): new_eventgen.write(f"\n[{stanza_name}]\n") for metadata_name, metadata_value in stanza_data.items(): if metadata_name == "add_comment": new_eventgen.write(f"{comment}\n") elif metadata_name != "tokens": new_eventgen.write( f"{metadata_name} = {metadata_value}\n") else: new_eventgen.write("\n") for tokens_id, tokens_value in stanza_data.get( "tokens").items(): new_eventgen.write( f"token.{tokens_id}.token = {tokens_value['token']}\n" ) new_eventgen.write( f"token.{tokens_id}.replacementType = {tokens_value['replacementType']}\n" ) new_eventgen.write( f"token.{tokens_id}.replacement = {tokens_value['replacement']}\n" ) if tokens_value.get("field"): new_eventgen.write( f'token.{tokens_id}.field = {tokens_value.get("field")}\n' ) new_eventgen.write("\n")