def parse_conf(conf_file, parts=list()):
"""Parse searchbnf.conf file.
The provided searchbnf.conf file is processed and parsed to return all
contained SPL terms. Optionally, only a subset of the file may be processed,
by specifying a list containing names of the sections to be processed.
Args:
conf_file (str): path to the searchbnf.conf file
parts (list, optional): names of parts to be processed. Defaults to\
empty list(), meaning the whole file.
Returns:
dict: dictionary of SPLTerms objects
"""
confparser = configparser.RawConfigParser(strict=False)
confparser.read(conf_file)
spl_terms = dict()
if parts:
for part in parts:
try:
spl_term = parse_spl_term(part, confparser[part])
spl_terms[spl_term.name] = spl_term
except KeyError:
pass
else:
for section in confparser.sections():
try:
spl_term = parse_spl_term(section, confparser[section])
spl_terms[spl_term.name] = spl_term
except KeyError:
log_message("WARNING", f"{section} has incorrect format.")
return spl_terms
async def async_fetch_spl_terms(self, spl_terms):
"""Asynchronously fetch details about SPL terms.
Args:
spl_terms (list): SPL terms to retrieve
"""
self.session = aiohttp.ClientSession(headers=self.headers,
connector=aiohttp.TCPConnector(verify_ssl=False),
auth=self.auth, raise_for_status=True)
async with self.session:
try:
if not spl_terms:
json_data = await self.async_get(self.build_url())
for term in json_data["entry"]:
try:
spl_terms.append(term["name"])
except KeyError:
pass
await asyncio.gather(*[self.async_fetch_spl_term(term)\
for term in spl_terms])
except aiohttp.client_exceptions.ClientResponseError as e:
if e.code == 401:
raise AuthenticationError()
else:
log_message("DEBUG", str(e))
raise ConnectionError()
except aiohttp.client_exceptions.ClientConnectorError as e:
log_message("DEBUG", str(e))
raise ConnectionError()
def build_syntax_trees(pseudo_bnf, spl_terms):
"""Build syntax trees from Pseudo-BNF definitions.
Provided SPLTerm objects should contain Pseudo-PNF definition of the syntax.
This function builds syntax trees from the syntax definitions, based on
the provided Lark grammar.
Args:
pseudo_bnf (str): loaded contents of the file defining Lark grammar\
for Pseudo-BNF.
spl_terms (dict): dictionary containing initialized SPLTerm objects
Raises:
InitError: if an error with the Lark grammar occurs
Returns:
dict: dictionary containing lark.Tree objects, representing syntax of\
the SPL terms
"""
try:
parser = LarkParser(pseudo_bnf)
except GrammarError:
raise InitError()
trees = dict()
for name, term in spl_terms.items():
try:
tree = parser.parse(term.get_bnf_syntax())
trees[name] = tree
except (UnexpectedCharacters, UnexpectedEOF):
log_message("DEBUG", f"<{name}> could not be parsed correctly")
return trees
def parse_json(json_data, parts=list()):
"""Parse searchbnf data in json format.
The provided searchbnf data in json is processed and parsed to return all
contained SPL terms. Optionally, only a subset of the file may be processed,
by specifying a list containing names of the terms to be processed.
Args:
json_data (dict): json data containing searchbnf definitions
parts (list, optional): names of parts to be processed. Defaults to\
empty list(), meaning all the data.
Returns:
dict: dictionary of SPLTerms objects
"""
spl_terms = dict()
if parts:
json_terms = [x for x in json_data["entry"] if x["name"] in parts]
else:
json_terms = json_data["entry"]
for json_term in json_terms:
try:
spl_term = parse_spl_term(json_term["name"], json_term["content"])
spl_terms[spl_term.name] = spl_term
except KeyError:
log_message("WARNING", f"{json_term['name']} has incorrect format.")
return spl_terms
def generate_grammar(self, outfile):
"""Generate a tmLanguage grammar for SPL.
Performs all necessary steps to fetch, process, build and generate
a tmLanguage grammar file for SPL. Informational messages are printed
to terminal during the generation process.
Args:
outfile (str): name of the file to save the generated grammar into
"""
log_message("INFO", f"Fetching all SPL terms...")
self.fetch_spl_terms()
pseudo_bnf = pkg_resources.read_text("spl_parser.grammars",
"pseudo_bnf.lark")
tm_template = pkg_resources.read_text("spl_parser.templates",
"template.tmLanguage.json")
generator = TmLanguageGenerator(tm_template)
log_message("INFO", "Building syntax trees...")
trees = build_syntax_trees(pseudo_bnf, self.spl_terms)
log_message("INFO", "Parsing SPL commands...")
commands = [x for x in self.spl_terms.values()\
if isinstance(x, SPLCommand)]
for command in commands:
command.parse(trees)
generator.add_command(command)
log_message("INFO", f"Saving grammar to file {outfile}...")
generator.save_grammar(outfile)
async def async_fetch_spl_term(self, spl_term):
"""Asynchronously fetch details about an SPL term.
Args:
spl_term (SPLTerm): SPL term to retrieve
"""
url = self.build_url(spl_term)
try:
json_data = await self.async_get(url)
log_message("DEBUG", f"<{spl_term}> was fetched")
spl_terms = parse_json(json_data, [spl_term])
self.spl_terms.update(spl_terms)
except aiohttp.client_exceptions.ClientResponseError as e:
if e.code == 404:
log_message("DEBUG", f"<{spl_term}> was NOT fetched")
else:
raise e
def parse(self, syntax_trees):
"""Parse full properties of the SPL command from syntax trees.
Details for the SPL command (its arguments, functions, operators) are
parsed from syntax trees. The trees related to the processed SPL command
are chosen from all available trees. If the command cannot be parsed
correctly, a warning is printed to the terminal.
Args:
syntax_trees (dict): dictionary containing pre-built syntax trees
"""
related_trees = find_related_trees(self.name, syntax_trees)
if related_trees:
for tree in related_trees.values():
(arguments, functions, operators) = parse_syntax_tree(tree)
self.arguments.update(arguments)
self.functions.update(functions)
self.operators.update(operators)
else:
log_message("WARNING",
f"{self.name} command may not work properly")
def view_command(self, command):
"""View details about an SPL command.
Performs all necessary steps to fetch, process and view information
about the provided command. The command details and informational
messages are printed to terminal.
Args:
command (str): name of the command to view
Raises:
CommandNotFoundError: if the specified command was not found
"""
log_message("INFO", f"Fetching details about {command} command...")
self.fetch_spl_terms([f"{command}-command"])
if self.spl_terms:
log_message("INFO", f"Found results:")
for spl_term in self.spl_terms.values():
spl_term.print()
else:
raise CommandNotFoundError(command)
def generate(ctx, outfile):
try:
ctx.obj.generate_grammar(outfile)
log_message("INFO", "SPL grammar was generated successfully!")
except SplParserError as e:
log_message("ERROR", e)
log_message("ERROR", "SPL grammar was not generated!")
sys.exit(2)
def remote(ctx, url, username, password):
ctx.obj = RemoteSplResource(url, username, password)
log_message("INFO", f"Using remote Splunk server: {url}")
def local(ctx, file):
ctx.obj = LocalSplResource(file)
log_message("INFO", f"Using local searchbnf file: {file}")
def view(ctx, spl_command):
try:
ctx.obj.view_command(spl_command)
except SplParserError as e:
log_message("ERROR", e)
sys.exit(1)
