def compiler(self, pattern, flags):
# pylint: disable=no-self-use
try:
return re.compile(pattern, flags)
except re.error as e:
raise ParseException(str(e))
def parseImpl(self, instring, loc, doActions=True):
test = instring[loc:loc + self.matchLen]
if test.upper() == self.match:
return loc + self.matchLen, test
#~ raise ParseException( instring, loc, self.errmsg )
raise ParseException(instring, loc, self.errmsg, self)
def _check_n_tokens(tokens, n_tokens, name):
if not len(tokens) == n_tokens:
err = "{} take {} values. You gave {}"
err = err.format(name, n_tokens, len(tokens))
raise ParseException(err)
def _check_keyword(tokens):
if tokens[0][-1] == '-':
raise ParseException("'-' found at the end of keyword.")
return None
def post_process(res, allow_implicit=True):
"""
Perform post-processing on the results of the date range parsing.
At the moment this consists mainly of ensuring that any missing information is filled in.
For example, if no years are specified at all in the string then both years are set to the
current year, and if one part of the string includes no month or year then these are
filled in from the other part of the string.
:param res: The results from the parsing operation, as returned by the parseString function
:param allow_implicit: If implicit dates are allowed
:return: the results with populated date information
"""
# Get current date
today = datetime.date.today()
if not allow_implicit:
if ('start' in res and 'day' not in res.start) or ('end' in res and 'day' not in res.end):
raise ParseException("Couldn't parse resulting datetime")
if 'start' not in res:
# We have a single date, not a range
res['start'] = {}
if 'month' not in res.end and 'day' not in res.end:
# We have only got a year, so go from start to end of the year
res['start']['year'] = res.end.year
res['start']['month'] = 1
res['start']['day'] = 1
res['end']['month'] = 12
res['end']['day'] = 31
return res
elif 'month' in res.end and 'day' not in res.end:
if not isinstance(res.end.month, int):
raise ParseException("Couldn't parse resulting datetime")
# special case - treat bare month as a range from start to end of month
if 'year' not in res.end or res.end.year == "":
res['start']['year'] = today.year
res['end']['year'] = today.year
else:
res['start']['year'] = res.end.year
res['start']['day'] = 1
res['start']['month'] = res.end.month
res['end']['day'] = calendar.monthrange(
res['start']['year'], res.end.month)[1]
else:
res['start']['day'] = res.end.day
res['start']['month'] = res.end.month
if 'year' not in res.end:
res['start']['year'] = today.year
else:
res['start']['year'] = res.end.year
res['end'] = None
return res
if 'month' not in res.end and 'month' not in res.start and \
'day' not in res.end and 'day' not in res.start:
# No months or days given, just years
res['start']['month'] = 1
res['start']['day'] = 1
res['end']['month'] = 12
res['end']['day'] = 31
return res
# Sort out years
if 'year' not in res.end:
res.end['year'] = today.year
res.start['year'] = today.year
elif 'year' not in res.start:
res.start['year'] = res.end.year
# Sort out months
if 'month' not in res.start:
res.start['month'] = res.end.month
if 'day' not in res.start or res.start['day'] == '':
res.start['day'] = 1
if res.end.month and ('day' not in res.end or res.end['day'] == ''):
res.end['day'] = calendar.monthrange(res.end.year, res.end.month)[1]
return res
def containerIdParseAction(s, loc, tokens):
v = int(tokens[0])
if not v in container_ids:
raise ParseException(s, loc, "Not a valid container id")
return v
def format_and_raise_parse_error(exc):
msg = ParseException.explain(exc, depth=0)
raise ParseError(msg)
def assertParserResultDict(parser_element,
test_strings,
expected_results,
assert_flag=True,
message=''):
"""
A nice unit test tool which provides an assert()-like function
that takes an string, parse the string, takes its computed
Pythonized list/dict and compares the result against its
expected Pythonized result.
:param parser_element: ParserElement class to exercise
:param test_strings: A string in which to be parsed by parser_element.
Or it can be a list of strings ['a', 'b'].
:param expected_results: A Python list in which to expect
If a_test_data is a list, then this argument
shall also be a list of expected result
:param assert_flag: If True, then expected result must match or an
exception gets raised.
If False, then parse MUST fail or expected
result does not match, else an exception
gets raised
:return: Always returns True (exception handles the False, like
an assert() class would do)
"""
retsts = None
def incr_pos(fn):
def _inner(*args):
global pos
pos += 1
print("\t" * pos, end="")
return fn(*args)
return _inner
def decr_pos(fn):
def _inner(*args):
global pos
print("\t" * pos, end="")
pos -= 1
return fn(*args)
return _inner
import pyparsing
pyparsing._defaultStartDebugAction = incr_pos(pyparsing._defaultStartDebugAction)
pyparsing._defaultSuccessDebugAction = decr_pos(pyparsing._defaultSuccessDebugAction)
pyparsing._defaultExceptionDebugAction = decr_pos(pyparsing._defaultExceptionDebugAction)
try:
parser_element = parser_element.setDebug(True)
result = parser_element.parseString(test_strings, parseAll=True)
from pprint import PrettyPrinter
pp = PrettyPrinter(indent=2, width=66, compact=False)
if result.asDict() == {}:
print('Dict() empty; BAD result:', end='')
pp.pprint(result)
retsts = False
else:
print('Good result:')
pp.pprint(result.asDict())
# Convert ParserElement into Python List[]
retsts = (result.asDict() == expected_results)
print('expecting: ')
pp.pprint(expected_results)
except ParseException as pe:
print('ParseException:')
print(pe.line) # affected data content
print(' ' * (pe.column - 1) + '^') # Show where the error occurred
print(pe)
ParseException.explain(pe)
retsts = False
except ParseSyntaxException as pe:
print('ParseSyntaxException:')
print(test_strings) # affected data content
print(' ' * (pe.column - 1) + '^') # Show where the error occurred
print(pe)
# print(parser_element.errmsg)
retsts = False
# raise InvalidConfiguration(error_msg)
if retsts == assert_flag:
print('assert(True)')
return True
else:
errmsg = 'Error(assert=' + str(False) + '): ' + message + '\"' + test_strings + '\".'
raise SyntaxError(errmsg)
def test_main(tm_parse_element):
"""
python_script # defaults to STDIN for input a_file (good for quick test or cut-n-paste)
python_script -t # Exercise built-in unit test
python_script <filespec> # Read a_file and syntax-check it
python_script -v # Increase verbosity level
python_script -d # Increase PyParsing debugging
:return:
"""
pgm_basename = os.path.basename(sys.argv[0])
prgm_desc = 'Exercise or test the {} function in {} python script against ParserElement "{}"'.format(
tm_parse_element.__class__.__name__, pgm_basename, tm_parse_element)
parser = argparse.ArgumentParser(description=prgm_desc)
parser.add_argument('-v', '--verbose', action='store_true', help='Run with extra messages')
parser.add_argument('-d', '--debug', action='store_true',
help='Run with PyParsing debugging enabled; outputs "Match/Matched" a_debug lines')
if unix_pipe_support:
default_arg1 = '-'
else:
default_arg1 = None
# nagrs='?' is zero or one argument exactly
parser.add_argument('filespec',
type=argparse.FileType('r'),
default=default_arg1,
nargs='?',
help='Input a_file to read and parse')
args = parser.parse_args()
if args.verbose:
print('"Number of arguments: ', len(sys.argv))
print('The arguments are: ', str(sys.argv))
print('argparse.args:', args)
retsts = 0
test_data = None
if not unix_pipe_support:
# test if file exist
if args.filespec is None:
retsts = errno.ENOTTY # Most people don't want UNIX pipe support
elif not os.access(args.filespec.name, os.R_OK):
print("Cannot read {} file. Exiting...".format(args.filespec.name))
# If no a_file given, we default to STDIN as a a_file to be opened
# Naturally, you'll have to press Ctrl-D to close the a_file.
if args.verbose:
print('parser_element:', tm_parse_element)
try:
test_data = args.filespec.read()
args.filespec.close()
except Exception as pe:
print('Exception:')
print(pe)
retsts = errno.EBADFD
if test_data:
if args.debug:
tm_parse_element.setDebug()
try:
tm_parse_element.ignore(cppStyleComment)
tm_parse_element.ignore(pythonStyleComment)
result = tm_parse_element.parseString(test_data, parseAll=True)
result_text = result.asList()
print("Result: ", result_text)
if len(result) == 0:
retsts = errno.EBADE
else:
retsts = 0
except ParseException as pe:
print('ParseException:')
print(pe.line) # affected data content
print(' ' * (pe.column - 1) + '^') # Show where the error occurred
print(pe)
ParseException.explain(pe)
retsts = errno.ELIBSCN
except ParseSyntaxException as pe:
print('ParseSyntaxException:')
print(test_data) # affected data content
print(' ' * (pe.column - 1) + '^') # Show where the error occurred
print(pe)
retsts = errno.ELIBBAD
else:
retsts = errno.ENODATA
if args.verbose:
print("test_data is empty")
# Build return values
return_list = {}
return_list['verbosity'] = args.verbose
return_list['a_debug'] = args.debug
return_list['filespec'] = args.filespec
return_list['errcode'] = retsts
return return_list
def validate_and_convert_number(tokens):
try:
return float(tokens[0])
except ValueError:
raise ParseException("Invalid number (%s)" % tokens[0])
def no_keywords_allowed(s, l, t):
wd = t[0]
if wd in pythonKeywords:
errmsg = "cannot not use keyword '%s' " \
"as an identifier" % wd
raise ParseException(s, l, errmsg)
def rangeCheckParseAction(string, loc, tokens):
parsedval = tokens[0]
if not inRangeFn(parsedval):
raise ParseException(string, loc, outOfRangeMessage % parsedval)
def assertParseElement(a_parse_element,
a_test_data,
a_expected_result,
a_assert_flag=True):
"""
A nice unit test tool which provides an assert()-like function
that takes an string, parse the string, takes its computed
Pythonized list/dict and compares the result against its
expected Pythonized result.
:param a_parse_element: ParserElement class to exercise
:param a_test_data: A string in which to be parsed by a_parse_element
:param a_expected_result: A Python list in which to expect
:param a_assert_flag: If True, then expected result must match or an
exception gets raised.
If False, then parse MUST fail or expected
result does not match, else an exception
gets raised
:return: Always returns True (exception handles the False, like
an assert() class would do)
"""
retsts = None
try:
a_parse_element = a_parse_element.setDebug(True)
result = a_parse_element.parseString(a_test_data, parseAll=True)
pp = PrettyPrinter(indent=2, width=66, compact=False)
if result.asDict() == {}:
print('***BAD***-Python-Dict result:', end='')
pp.pprint(result)
else:
print('Good-Python-Dict result:')
pp.pprint(result.asDict())
print('expecting: ')
pp.pprint(a_expected_result)
# Convert ParserElement into Python List[] and compare
retsts = (result.asDict() == a_expected_result)
except ParseException as pe:
print('ParseException:')
print(pe.line) # affected data content
print(' ' * (pe.column - 1) + '^') # Show where the error occurred
print(pe)
ParseException.explain(pe)
retsts = False
except ParseBaseException as pe:
print('ParseBaseException:')
print(a_test_data) # affected data content
print(' ' * (pe.column - 1) + '^') # Show where the error occurred
print(pe)
retsts = False
except ParseSyntaxException as pe:
print('ParseSyntaxException:')
print(a_test_data) # affected data content
print(' ' * (pe.column - 1) + '^') # Show where the error occurred
print(pe)
retsts = False
if retsts == a_assert_flag:
print('assert(True)')
return True
else:
print('assert(***FALSE***)')
errmsg = 'Error(assert=' + str(False) + '): \"' + a_test_data + '\".'
raise SyntaxError(errmsg)
def __init__(self, t):
if t[0] not in ['unit', 'global']:
raise ParseException(f'Only `unit` and `global` aggregation types are supported but `{t[0]}` received.')
self.agg_type = t[0]
def __repr__(self):
# print([t for t in self.tokens.items()])
if 'singleterm' in self.tokens:
if self.tokens.fieldname == '_exists_':
return '{{ "attributes.{}": {{ "$exists": true }} }}'.format(
self.tokens.singleterm)
else:
if self.tokens.field[0] == '__default_field__':
return '{{ "{}": {{ "{}": "{}" }} }}'.format(
'__default_field__', '__default_operator__',
self.tokens.singleterm)
else:
return '{{ "{}": {{ "$regex": "{}" }} }}'.format(
self.tokens.field[0], self.tokens.singleterm)
if 'phrase' in self.tokens:
if self.tokens.field[0] == '__default_field__':
return '{{ "{}": {{ "{}": "{}" }} }}'.format(
'__default_field__', '__default_operator__',
self.tokens.phrase)
else:
return '{{ "{}": {{ "$regex": "{}" }} }}'.format(
self.tokens.field[0], self.tokens.phrase)
if 'wildcard' in self.tokens:
return '{{ "{}": {{ "$regex": "\\\\b{}\\\\b" }} }}'.format(
self.tokens.field[0], self.tokens.wildcard)
if 'regex' in self.tokens:
return '{{ "{}": {{ "$regex": "{}" }} }}'.format(
self.tokens.field[0], self.tokens.regex)
def range_term(field, operator, range):
if field in ['duplicateCount', 'timeout']:
range = int(range)
else:
range = '"{}"'.format(range)
return '{{ "{}": {{ "{}": {} }} }}'.format(field, operator, range)
if 'range' in self.tokens:
if self.tokens.range[0].lowerbound == '*':
lower_term = '{}'
else:
lower_term = range_term(
self.tokens.field[0],
'$gte' if 'inclusive' in self.tokens.range[0] else '$gt',
self.tokens.range[0].lowerbound)
if self.tokens.range[2].upperbound == '*':
upper_term = '{}'
else:
upper_term = range_term(
self.tokens.field[0],
'$lte' if 'inclusive' in self.tokens.range[2] else '$lt',
self.tokens.range[2].upperbound)
return '{{ "$and": [ {}, {} ] }}'.format(lower_term, upper_term)
if 'onesidedrange' in self.tokens:
return range_term(self.tokens.field[0],
self.tokens.onesidedrange.op,
self.tokens.onesidedrange.bound)
if 'subquery' in self.tokens:
if self.tokens.field[0] != '__default_field__':
return '{}'.format(self.tokens.subquery[0])\
.replace('__default_field__', self.tokens.field[0])\
.replace('__default_operator__', '$regex')
else:
return '{}'.format(self.tokens.subquery[0])
raise ParseException('Search term did not match query syntax: %s' %
self.tokens)
def parse_hand(s, parseAll=False):
tokens = hand.parseString(s, parseAll=parseAll)
if tokens is not None and len(tokens) > 0:
return tokens[0]
else:
raise ParseException('Error parsing hand string \'' + s + '\'')
def secretIdNumberParseAction(s, loc, tokens):
v = int(tokens[0])
if not v in secret_ids:
raise ParseException(s, loc, "Not a valid secret id")
return v
def __init__(self, t):
if t[0] not in ["unit", "global"]:
raise ParseException(f"Only `unit` and `global` aggregation types are supported but `{t[0]}` received.")
self.agg_type = t[0]
def mustMatch(tokens):
if tokens[1] != opentag:
raise ParseException("", 0, "")
def get_named_var(var_name):
try:
get_type(var_name)
except ParseException:
return _() % var_name
raise ParseException('var name clashes with type: %s' % var_name)
def parse_file(p: ParserElement, file_name: PathLike) -> ParseResults:
"""Apply parser `p` on file `file_name`."""
try:
return p.parseFile(file_name)
except ParseException as ex:
raise ParseException(f"Error Trying to parse: {p} in file: {file_name}") from ex
def dont_allow_non_comparing_terms(self, s, loc, toks):
if isinstance(toks[0], self.BoolOperand):
raise ParseException("Failed")
return toks
def parse(text, allow_implicit=True):
"""
Parses a date range string and returns the start and end as datetimes.
**Accepted formats:**
This parsing routine works with date ranges and single dates, and should
work with a wide variety of human-style string formats, including:
- 27th-29th June 2010
- 30 May to 9th Aug
- 3rd Jan 1980 - 2nd Jan 2013
- Wed 23 Jan - Sat 16 February 2013
- Tuesday 29 May -> Sat 2 June 2012
- From 27th to 29th March 1999
- 1--9 Jul
- 14th July 1988
- 23rd October 7:30pm
- From 07:30 18th Nov to 17:00 24th Nov
**Notes:**
- If an error encountered while parsing the date range then a
`pyparsing.ParseException` will be raised.
- If no year is specified then the current year is used.
- All day names are ignored, so there is no checking to see whether,
for example, the 23rd Jan 2013 is actually a Wednesday.
- All times are ignored, assuming they are placed either before or after
each date, otherwise they will cause an error.
- The separators that are allows as part of the date range are `to`,
`until`, `-`, `--` and `->`, plus the unicode em and en dashes.
- Other punctuation, such as commas, is ignored.
:param text: The string to parse
:param allow_implicit: If implicit dates are allowed. For example,
string 'May' by default treated as range
from May, 1st to May, 31th. Setting allow_implicit to False helps avoid it.
:return: A tuple ``(start, end)`` where each element is a datetime object.
If the string only defines a single date then the tuple is ``(date, None)``.
All times in the datetime objects are set to 00:00 as this function only parses dates.
"""
parser = create_parser()
# print text
result = parser.parseString(text)
# print result.dump()
# print "----------"
res = post_process(result, allow_implicit)
# print res.dump()
# Create standard dd/mm/yyyy strings and then convert to Python datetime
# objects
if 'year' not in res.start:
# in case only separator was given
raise ParseException("Couldn't parse resulting datetime")
try:
start_str = "%(day)s/%(month)s/%(year)s" % res.start
start_datetime = datetime.datetime.strptime(start_str, "%d/%m/%Y")
except ValueError:
raise ParseException("Couldn't parse resulting datetime")
if res.end is None:
return start_datetime, None
elif not res.end:
raise ParseException("Couldn't parse resulting datetime")
else:
try:
if "month" not in res.end:
res.end["month"] = res.start["month"]
end_str = "%(day)s/%(month)s/%(year)s" % res.end
end_datetime = datetime.datetime.strptime(end_str, "%d/%m/%Y")
except ValueError:
raise ParseException("Couldn't parse resulting datetime")
if end_datetime < start_datetime:
# end is before beginning!
# This is probably caused by a date straddling the change of year
# without the year being given
# So, we assume that the start should be the previous year
res.start['year'] = res.start['year'] - 1
start_str = "%(day)s/%(month)s/%(year)s" % res.start
start_datetime = datetime.datetime.strptime(start_str, "%d/%m/%Y")
return start_datetime, end_datetime
def check_sub_indent(str, location, tokens):
cur_col = col(location, str)
if cur_col > indent_stack[-1]:
indent_stack.append(cur_col)
else:
raise ParseException(str, location, "not a subentry")
def _check_toplabel(tokens):
if tokens[0][-1] == '-':
raise ParseException("Top level ending in '-'")
return None
def check_unindent(str, location, tokens):
if location >= len(str):
return
cur_col = col(location, str)
if not (cur_col < indent_stack[-1] and cur_col <= indent_stack[-2]):
raise ParseException(str, location, "not an unindent")
def convertToFloat(s, loc, toks):
try:
return float(toks[0])
except:
raise ParseException(loc, "invalid float format %s" % toks[0])
def __repr__(self):
# print([t for t in self.tokens.items()])
if 'singleterm' in self.tokens:
if self.tokens.fieldname == '_exists_':
return '"attributes"::jsonb ? \'{}\''.format(
self.tokens.singleterm)
elif self.tokens.fieldname in ['correlate', 'service', 'tags']:
return '\'{}\'=ANY("{}")'.format(self.tokens.singleterm,
self.tokens.field[0])
elif self.tokens.attr:
tokens_attr = self.tokens.attr.replace('_', 'attributes')
return '"{}"::jsonb ->>\'{}\' ILIKE \'%%{}%%\''.format(
tokens_attr, self.tokens.fieldname, self.tokens.singleterm)
else:
return '"{}" ILIKE \'%%{}%%\''.format(self.tokens.field[0],
self.tokens.singleterm)
if 'phrase' in self.tokens:
if self.tokens.field[0] == '__default_field__':
return '"{}" ~* \'\\y{}\\y\''.format('__default_field__',
self.tokens.phrase)
elif self.tokens.field[0] in ['correlate', 'service', 'tags']:
return '\'{}\'=ANY("{}")'.format(self.tokens.term,
self.tokens.field[0])
else:
return '"{}" ~* \'\\y{}\\y\''.format(self.tokens.field[0],
self.tokens.phrase)
if 'wildcard' in self.tokens:
return '"{}" ~* \'\\y{}\\y\''.format(self.tokens.field[0],
self.tokens.wildcard)
if 'regex' in self.tokens:
return '"{}" ~* \'{}\''.format(self.tokens.field[0],
self.tokens.regex)
if 'range' in self.tokens:
if self.tokens.range[0].lowerbound == '*':
lower_term = '1=1'
else:
lower_term = '"{}" {} \'{}\''.format(
self.tokens.field[0],
'>=' if 'inclusive' in self.tokens.range[0] else '>',
self.tokens.range[0].lowerbound)
if self.tokens.range[2].upperbound == '*':
upper_term = '1=1'
else:
upper_term = '"{}" {} \'{}\''.format(
self.tokens.field[0],
'<=' if 'inclusive' in self.tokens.range[2] else '<',
self.tokens.range[2].upperbound)
return '({} AND {})'.format(lower_term, upper_term)
if 'onesidedrange' in self.tokens:
return '("{}" {} \'{}\')'.format(self.tokens.field[0],
self.tokens.onesidedrange.op,
self.tokens.onesidedrange.bound)
if 'subquery' in self.tokens:
if self.tokens.attr:
tokens_attr = 'attributes' if self.tokens.attr == '_' else self.tokens.attr
tokens_fieldname = '"{}"::jsonb ->>\'{}\''.format(
tokens_attr, self.tokens.fieldname)
else:
tokens_fieldname = '"{}"'.format(self.tokens.fieldname
or self.tokens.field[0])
return '{}'.format(self.tokens.subquery[0]).replace(
'"__default_field__"', tokens_fieldname)
raise ParseException('Search term did not match query syntax: %s' %
self.tokens)
def convertToExpressionTree(tokens=None):
if not tokens:
raise ParseException("Failed to parse expression")
return ParseExpression(' '.join(tokens[0]))
def parse(self, filepath: str) -> None:
with open(filepath, "r") as filehandle:
try:
self.__grammar.parseFile(filehandle)
except (ParseException, ParseFatalException) as e:
raise ParserError("\n" + ParseException.explain(e, 0))
