def __init__(self, addon_path, config_path=None):
self._app = App(addon_path, python_analyzer_enable=False)
self.config_path = config_path
self._eventgen = None
self.addon_path = addon_path
self.match_stanzas = set()
class EventgenParser:
"""
This class represents the entire eventgen.conf file and handles parsing mechanism of eventgen and the rules.
Args:
addon_path (str): Path to the Splunk App
"""
conf_name = " "
def __init__(self, addon_path, config_path=None):
self._app = App(addon_path, python_analyzer_enable=False)
self.config_path = config_path
self._eventgen = None
self.addon_path = addon_path
self.match_stanzas = set()
@property
def path_to_samples(self):
if os.path.exists(os.path.join(self.config_path, "samples")):
LOGGER.info(
"Samples path is: {}".format(os.path.join(self.config_path, "samples"))
)
return os.path.join(self.config_path, "samples")
elif os.path.exists(
os.path.join(
os.path.abspath(os.path.join(self.config_path, os.pardir)), "samples"
)
):
LOGGER.info(
"Samples path is: {}".format(
os.path.join(
os.path.abspath(os.path.join(self.config_path, os.pardir)),
"samples",
)
)
)
return os.path.join(
os.path.abspath(os.path.join(self.config_path, os.pardir)), "samples"
)
else:
LOGGER.info(
"Samples path is: {}".format(os.path.join(self.addon_path, "samples"))
)
return os.path.join(self.addon_path, "samples")
@property
def eventgen(self):
try:
relative_path = os.path.relpath(self.config_path, self.addon_path)
if os.path.exists(
os.path.join(self.config_path, "pytest-splunk-addon-data.conf")
):
self._eventgen = self._app.get_config(
"pytest-splunk-addon-data.conf", dir=relative_path
)
self.conf_name = "psa-data-gen"
path = self._app.get_filename(
relative_path, "pytest-splunk-addon-data.conf"
)
elif os.path.exists(os.path.join(self.config_path, "eventgen.conf")):
self._eventgen = self._app.get_config(
"eventgen.conf", dir=relative_path
)
self.conf_name = "eventgen"
path = self._app.get_filename(relative_path, "eventgen.conf")
else:
self._eventgen = self._app.get_config("eventgen.conf")
self.conf_name = "eventgen"
path = self._app.get_filename("default", "eventgen.conf")
LOGGER.info(
"Using Eventgen path: {e}\nUsing Conf file name: {c}".format(
e=path, c=self.conf_name
)
)
return self._eventgen
except OSError:
LOGGER.warning("pytest-splunk-addon-data.conf/eventgen.conf not Found")
raise FileNotFoundError(
"pytest-splunk-addon-data.conf/eventgen.conf not Found"
)
def get_sample_stanzas(self):
"""
Converts a stanza in eventgen.conf to an object of SampleStanza.
Yields:
SampleStanza Object
"""
eventgen_dict = self.get_eventgen_stanzas()
self.check_samples()
for sample_name, stanza_params in sorted(eventgen_dict.items()):
sample_path = os.path.join(self.path_to_samples, sample_name)
yield SampleStanza(
sample_path,
stanza_params,
)
def get_eventgen_stanzas(self):
"""
Parses the eventgen.conf file and converts it into a dictionary.
Format::
{
"sample_file_name": # Not Stanza name
{
"input_type": "str",
"tokens":
{
1:
{
token: #One#
replacementType: random
replacement: static
}
}
}
}
Return:
Dictionary representing eventgen.conf in the above format.
"""
eventgen_dict = {}
if os.path.exists(self.path_to_samples):
for sample_file in os.listdir(self.path_to_samples):
for stanza in sorted(self.eventgen.sects):
stanza_match_obj = re.search(stanza, sample_file)
if stanza_match_obj and stanza_match_obj.group(0) == sample_file:
self.match_stanzas.add(stanza)
eventgen_sections = self.eventgen.sects[stanza]
eventgen_dict.setdefault((sample_file), {"tokens": {}})
for stanza_param in eventgen_sections.options:
eventgen_property = eventgen_sections.options[stanza_param]
if eventgen_property.name.startswith("token"):
_, token_id, token_param = eventgen_property.name.split(
"."
)
token_key = "{}_{}".format(stanza, token_id)
if (
not token_key
in eventgen_dict[sample_file]["tokens"].keys()
):
eventgen_dict[sample_file]["tokens"][token_key] = {}
eventgen_dict[sample_file]["tokens"][token_key][
token_param
] = eventgen_property.value
else:
eventgen_dict[sample_file][
eventgen_property.name
] = eventgen_property.value
return eventgen_dict
def check_samples(self):
"""
Gives a user warning when sample file is not found for the stanza peresent in the configuration file.
"""
if os.path.exists(self.path_to_samples):
for stanza in self.eventgen.sects:
if stanza not in self.match_stanzas:
raise_warning("No sample file found for stanza : {}".format(stanza))
LOGGER.info("Sample file found for stanza : {}".format(stanza))
def __init__(self, addon_path):
self._app = App(addon_path, python_analyzer_enable=False)
self._eventgen = None
self.path_to_samples = os.path.join(addon_path, "samples")
def app(self):
if not self._app:
self._app = App(self.splunk_app_path, python_analyzer_enable=False)
return self._app
class UpdateEventgen:
"""Update eventgen file"""
def __init__(self, addon_path):
self._app = App(addon_path, python_analyzer_enable=False)
self._eventgen = None
self.path_to_samples = os.path.join(addon_path, "samples")
@property
def eventgen(self):
try:
if not self._eventgen:
self._eventgen = self._app.get_config("eventgen.conf")
return self._eventgen
except OSError:
LOGGER.error("Eventgen.conf not found")
raise Exception("Eventgen.conf not found")
def get_eventgen_stanzas(self):
"""
To get eventgen stanza and create a dictionary.
If stanza contains regex for multiple sample files, then it creates stanza for each sample file.
Return:
eventgen_dict (dict):
{
"stanza_name":
{
"other metadata": "source, sourcetype, etc."
"sample_count" : int
"tokens":
{
0: {
token: #One#
replacementType: random
replacement: static
}
}
}
}
"""
eventgen_dict = {}
for stanza in self.eventgen.sects:
eventgen_sections = self.eventgen.sects[stanza]
eventgen_dict.setdefault(
(stanza),
{
"tokens": {},
},
)
try:
events_in_file = len(
open(os.path.join(self.path_to_samples,
stanza)).readlines())
eventgen_dict[stanza]["sample_count"] = events_in_file
except:
pass
for stanza_param in eventgen_sections.options:
eventgen_property = eventgen_sections.options[stanza_param]
if eventgen_property.name.startswith("token"):
_, token_id, token_param = eventgen_property.name.split(
".")
if not token_id in eventgen_dict[stanza]["tokens"].keys():
eventgen_dict[stanza]["tokens"][token_id] = {}
eventgen_dict[stanza]["tokens"][token_id][
token_param] = eventgen_property.value
else:
eventgen_dict[stanza][
eventgen_property.name] = eventgen_property.value
for sample_file in os.listdir(self.path_to_samples):
if re.search(stanza, sample_file):
events_in_file = len(
open(os.path.join(self.path_to_samples,
sample_file)).readlines())
if sample_file not in eventgen_dict.keys():
eventgen_dict.setdefault((sample_file), {})
eventgen_dict[sample_file][
"sample_count"] = events_in_file
eventgen_dict[sample_file]["add_comment"] = True
eventgen_dict[sample_file]["tokens"] = {}
return eventgen_dict
# update the stanzas in dict
def update_eventgen_stanzas(self, eventgen_dict):
"""
Updates the eventgen_dict by adding new metadata
New Metadata: ["input_type", "host_type", "sourcetype_to_search", "timestamp_type"]
And update the tokens if possible based on the new Data-Generator rules.
Input:
eventgen_dict (dict) : eventgen dictionary in following format.
Return:
eventgen_dict (dict): Updated Eventgen stanzas dictionary
"""
metadata = [
"input_type", "host_type", "sourcetype_to_search", "timestamp_type"
]
review_comments = {
"metadata":
"#REVIEW : Update metadata as per addon's requirement",
"replacement":
"# REVIEW : Possible value in list : ",
"field":
"# REVIEW : Check if the field is extracted from the events, else remove this field parameter",
"mapping":
"# REVIEW : Please check if it can be replace with %s rule",
"sample_count":
"# REVIEW : Please check for the events per stanza and update sample_count accordingly",
}
for stanza_name, stanza_data in eventgen_dict.items():
# adding metadata
for data in metadata:
eventgen_dict[stanza_name][data] = (
f"<<{data}>> "
f"{review_comments['metadata']}")
if eventgen_dict[stanza_name].get("index"):
eventgen_dict[stanza_name]["index"] = (
f"{eventgen_dict[stanza_name]['index']} "
f"{review_comments['metadata']}")
eventgen_dict[stanza_name]["source"] = eventgen_dict[stanza_name].get(
"source",
f"pytest-splunk-addon:{eventgen_dict[stanza_name]['input_type']}",
)
for _, token_data in stanza_data.get("tokens", {}).items():
token_name = token_data.get("token").strip("#()").lower()
for _, new_token_values in FIELD_MAPPING.items():
if token_name in new_token_values.get("token"):
new_replacement_type = new_token_values.get(
"replacementType")
new_replacement = new_token_values.get("replacement")
token_data["replacementType"] = new_replacement_type
token_data["replacement"] = new_replacement
if new_token_values.get("possible_replacement"):
token_data["replacement"] = (
f"{new_replacement} "
f"{review_comments['replacement']} "
f"{new_token_values.get('possible_replacement')}"
)
if new_token_values.get("field"):
token_data["field"] = (
f"{new_token_values.get('field')} "
f"{review_comments['field']}")
if token_data.get("replacementType").lower() == "timestamp":
token_data["field"] = f"_time {review_comments['field']}"
elif token_data.get("replacementType").lower() in [
"file", "mvfile"
]:
file_name = (token_data.get("replacement").split("/")
[-1].split(":")[0])
token_data[
"replacement"] = f"file[{token_data.get('replacement')}]"
token_data["replacementType"] = "random"
for key_fields, mapped_files in FILE_MAPPING.items():
replacement_type = FIELD_MAPPING.get(key_fields).get(
"replacementType")
replacement = FIELD_MAPPING.get(key_fields).get(
"replacement")
replacement_type_values = FIELD_MAPPING.get(
key_fields).get("possible_replacement")
field_value = FIELD_MAPPING.get(key_fields).get(
"field")
if file_name in mapped_files:
if "SA-Eventgen" in token_data["replacement"]:
token_data["replacementType"] = (
f"{replacement_type} "
f"{review_comments['mapping']%key_fields}")
token_data["replacement"] = (
f"{replacement} "
f"{review_comments['mapping']%key_fields}")
if replacement_type_values:
token_data["replacement"] = (
f"{token_data['replacement']} "
f"{review_comments['replacement']} "
f"{replacement_type_values}")
if field_value:
token_data["field"] = (
f"{field_value} "
f"{review_comments['mapping']%key_fields}")
# for assigning sample_count at the end of metadata
if eventgen_dict.get(stanza_name).get("sample_count"):
event_count = eventgen_dict[stanza_name].pop("sample_count")
eventgen_dict[stanza_name]["sample_count"] = (
f"{event_count}"
f" {review_comments['sample_count']}")
# for assigning tokens at the end of metadata
if eventgen_dict.get(stanza_name).get("tokens"):
token_dict = eventgen_dict[stanza_name].pop("tokens")
eventgen_dict[stanza_name]["tokens"] = token_dict
return eventgen_dict
def create_new_eventgen(self, updated_eventgen_dict, new_conf_path):
"""
Writes the new values in a new conf file
params:
updated_eventgen_dict (dict) : Containing all the new values for eventgen.conf
new_conf_path : file path for creating new conf file
"""
with open(new_conf_path, "w") as new_eventgen:
LOGGER.info("created new file {}".format(new_conf_path))
# writing file metadata in new eventgen file
comment = "## Stanza gets metadata from main stanza"
for file_metadata in self.eventgen.headers:
new_eventgen.write(file_metadata + "\n")
for stanza_name, stanza_data in updated_eventgen_dict.items():
new_eventgen.write(f"\n[{stanza_name}]\n")
for metadata_name, metadata_value in stanza_data.items():
if metadata_name == "add_comment":
new_eventgen.write(f"{comment}\n")
elif metadata_name != "tokens":
new_eventgen.write(
f"{metadata_name} = {metadata_value}\n")
else:
new_eventgen.write("\n")
for tokens_id, tokens_value in stanza_data.get(
"tokens").items():
new_eventgen.write(
f"token.{tokens_id}.token = {tokens_value['token']}\n"
)
new_eventgen.write(
f"token.{tokens_id}.replacementType = {tokens_value['replacementType']}\n"
)
new_eventgen.write(
f"token.{tokens_id}.replacement = {tokens_value['replacement']}\n"
)
if tokens_value.get("field"):
new_eventgen.write(
f'token.{tokens_id}.field = {tokens_value.get("field")}\n'
)
new_eventgen.write("\n")