def _resolve_variable(cls, config, substitution):
"""
:param config:
:param substitution:
:return: (is_resolved, resolved_variable)
"""
variable = substitution.variable
try:
return True, config.get(variable)
except ConfigMissingException:
# default to environment variable
value = os.environ.get(variable)
if value is None:
if substitution.optional:
return False, None
else:
raise ConfigSubstitutionException(
"Cannot resolve variable ${{{variable}}} (line: {line}, col: {col})"
.format(variable=variable,
line=lineno(substitution.loc,
substitution.instring),
col=col(substitution.loc,
substitution.instring)))
elif isinstance(value, ConfigList) or isinstance(
value, ConfigTree):
raise ConfigSubstitutionException(
"Cannot substitute variable ${{{variable}}} because it does not point to a "
"string, int, float, boolean or null {type} (line:{line}, col: {col})"
.format(variable=variable,
type=value.__class__.__name__,
line=lineno(substitution.loc,
substitution.instring),
col=col(substitution.loc, substitution.instring)))
return True, value
def _resolve_variable(config, substitution):
"""
:param config:
:param substitution:
:return: (is_resolved, resolved_variable)
"""
variable = substitution.variable
try:
return True, config.get(variable)
except ConfigMissingException:
# default to environment variable
value = os.environ.get(variable)
if value is None:
if substitution.optional:
return False, None
else:
raise ConfigSubstitutionException(
"Cannot resolve variable ${{{variable}}} (line: {line}, col: {col})".format(
variable=variable,
line=lineno(substitution.loc, substitution.instring),
col=col(substitution.loc, substitution.instring)))
elif isinstance(value, ConfigList) or isinstance(value, ConfigTree):
raise ConfigSubstitutionException(
"Cannot substitute variable ${{{variable}}} because it does not point to a "
"string, int, float, boolean or null {type} (line:{line}, col: {col})".format(
variable=variable,
type=value.__class__.__name__,
line=lineno(substitution.loc, substitution.instring),
col=col(substitution.loc, substitution.instring)))
return True, value
def pyparsingLoggingSuccessDebugAction(instring, startloc, endloc, expr, toks):
row_start = pyparsing.lineno(startloc, instring)
col_start = pyparsing.col(startloc, instring)
row_end = pyparsing.lineno(endloc, instring)
col_end = pyparsing.col(endloc, instring)
pplogger.debug(
strip_margin('''Match Success: expr: `%s`, startLoc: `%s`,
| startRow: `%s`, startCol: `%s`, endLoc: `%s`, endRow: `%s`, endCol:
| `%s`, -> toks: `%s` tokens'''), expr, startloc, row_start, col_start,
endloc, row_end, col_end, len(toks.asList()))
def expand_state_definition(source, loc, tokens):
indent = " " * (col(loc, source) - 1)
statedef = []
# build list of states
states = set()
fromTo = {}
for tn in tokens.transitions:
states.add(tn.fromState)
states.add(tn.toState)
fromTo[tn.fromState] = tn.toState
# define base class for state classes
baseStateClass = tokens.name + "State"
statedef.extend([
"class %s(object):" % baseStateClass, " def __str__(self):",
" return self.__class__.__name__", " def next_state(self):",
" return self._next_state_class()"
])
# define all state classes
statedef.extend("class {}({}): pass".format(s, baseStateClass)
for s in states)
statedef.extend("{}._next_state_class = {}".format(s, fromTo[s])
for s in states if s in fromTo)
return indent + ("\n" + indent).join(statedef) + "\n"
def _setLineAndLineNumberParseAction(
self, s: str, loc: int,
toks: pyparsing.ParseResults) -> pyparsing.ParseResults:
'''
helper that is meant to be used with `<ParserElement>.setParseAction`, which means this function gets called
whenever a match for each line of the Telegram TL file gets hit.
Here we add stuff to the ParseResults that it passes in
we add the source line and source line number to each match
@param s is the original parse string
@param loc is the location in the string where matching started
@param toks is the list of the matched tokens, packaged as a ParseResults object
@return a ParseResults if you are modifying it, else None
'''
t = toks
orig_line = pyparsing.line(loc, s)
line_number = pyparsing.lineno(loc, s)
column = pyparsing.col(loc, s)
logger.debug(
"_setLineAndLineNumberParseAction: line_number: `%s`, column: `%s`, line: `%s`",
line_number, column, orig_line)
t[constants.RESULT_NAME_SOURCE_LINE] = orig_line
t[constants.RESULT_NAME_SOURCE_LINE_NUMBER] = line_number
return t
def check_sub_indent(string, location, tokens):
"""Check the indentation."""
cur_col = col(location, string)
if cur_col > indent_stack[-1]:
indent_stack.append(cur_col)
else:
raise ParseException(string, location, "not a subentry")
def check_unindent(string, location, tokens):
"""Check the 'undentation'."""
if location >= len(string):
return
cur_col = col(location, string)
if not (cur_col < indent_stack[-1] and cur_col <= indent_stack[-2]):
raise ParseException(string, location, "not an unindent")
def check_unindent(string, location, tokens):
"""Check the 'undentation'."""
if location >= len(string):
return
cur_col = col(location, string)
if not(cur_col < indent_stack[-1] and cur_col <= indent_stack[-2]):
raise ParseException(string, location, "not an unindent")
def check_sub_indent(string, location, tokens):
"""Check the indentation."""
cur_col = col(location, string)
if cur_col > indent_stack[-1]:
indent_stack.append(cur_col)
else:
raise ParseException(string, location, "not a subentry")
def check_unindent(self, s, l, t):
if l >= len(s):
return
curCol = col(l, s)
if not(curCol < self.indentStack[-1]
and curCol <= self.indentStack[-2]):
raise ParseException(s, l, "not an unindent")
def resolve_substitutions(config):
ConfigParser._fixup_self_references(config)
substitutions = ConfigParser._find_substitutions(config)
if len(substitutions) > 0:
unresolved = True
any_unresolved = True
_substitutions = []
while any_unresolved and len(substitutions) > 0 and set(substitutions) != set(_substitutions):
unresolved = False
any_unresolved = True
_substitutions = substitutions[:]
for substitution in _substitutions:
is_optional_resolved, resolved_value = ConfigParser._resolve_variable(config, substitution)
# if the substitution is optional
if not is_optional_resolved and substitution.optional:
resolved_value = None
unresolved, new_substitutions, result = ConfigParser._do_substitute(substitution, resolved_value, is_optional_resolved)
any_unresolved = unresolved or any_unresolved
substitutions.extend(new_substitutions)
if not isinstance(result, ConfigValues):
substitutions.remove(substitution)
ConfigParser._final_fixup(config)
if unresolved:
raise ConfigSubstitutionException("Cannot resolve {variables}. Check for cycles.".format(
variables=', '.join('${{{variable}}}: (line: {line}, col: {col})'.format(
variable=substitution.variable,
line=lineno(substitution.loc, substitution.instring),
col=col(substitution.loc, substitution.instring)) for substitution in substitutions)))
return config
def expand_state_definition(source, loc, tokens):
indent = " " * (col(loc,source)-1)
statedef = []
# build list of states
states = set()
fromTo = {}
for tn in tokens.transitions:
states.add(tn.fromState)
states.add(tn.toState)
fromTo[tn.fromState] = tn.toState
# define base class for state classes
baseStateClass = tokens.name + "State"
statedef.extend([
"class %s(object):" % baseStateClass,
" def __str__(self):",
" return self.__class__.__name__",
" def next_state(self):",
" return self._next_state_class()" ])
# define all state classes
statedef.extend(
"class %s(%s): pass" % (s,baseStateClass)
for s in states )
statedef.extend(
"%s._next_state_class = %s" % (s,fromTo[s])
for s in states if s in fromTo )
return indent + ("\n"+indent).join(statedef)+"\n"
def __init__(self, st, locn, tokString):
self.token_string = tokString
self.loc = locn
self.before_line = line(locn - 1, st)
self.source_line = line(locn, st)
self.line_num = lineno(locn, st)
self.col = col(locn, st)
def transform(self):
def determine_type(token):
return ConfigTree if isinstance(
token,
ConfigTree) else ConfigList if isinstance(token, list) else str
def format_str(v):
return '' if v is None else str(v)
if self.has_substitution():
return self
# remove None tokens
tokens = [token for token in self.tokens if token is not None]
if not tokens:
return None
# check if all tokens are compatible
first_tok_type = determine_type(tokens[0])
for index, token in enumerate(tokens[1:]):
tok_type = determine_type(token)
if first_tok_type is not tok_type:
raise ConfigWrongTypeException(
"Token '{token}' of type {tok_type} (index {index}) must be of type {req_tok_type} (line: {line}, col: {col})"
.format(token=token,
index=index + 1,
tok_type=tok_type.__name__,
req_tok_type=first_tok_type.__name__,
line=lineno(self._loc, self._instring),
col=col(self._loc, self._instring)))
if first_tok_type is ConfigTree:
result = ConfigTree()
for token in tokens:
for key, val in token.items():
# update references for substituted contents
if isinstance(val, ConfigValues):
val.parent = result
val.key = key
result[key] = val
return result
elif first_tok_type is ConfigList:
result = []
for sublist in tokens:
sublist_result = ConfigList()
for index, token in enumerate(sublist):
if isinstance(token, ConfigValues):
token.parent = result
token.key = index
sublist_result.append(token)
result.extend(sublist_result)
return [result]
else:
if len(tokens) == 1:
return tokens[0]
else:
return ''.join(
token if isinstance(token, str) else format_str(token) +
' ' for token in tokens[:-1]) + format_str(tokens[-1])
def transform(self):
def determine_type(token):
return ConfigTree if isinstance(token, ConfigTree) else ConfigList if isinstance(token, list) else str
def format_str(v):
return "" if v is None else str(v)
if self.has_substitution():
return self
# remove None tokens
tokens = [token for token in self.tokens if token is not None]
if not tokens:
return None
# check if all tokens are compatible
first_tok_type = determine_type(tokens[0])
for index, token in enumerate(tokens[1:]):
tok_type = determine_type(token)
if first_tok_type is not tok_type:
raise ConfigWrongTypeException(
"Token '{token}' of type {tok_type} (index {index}) must be of type {req_tok_type} (line: {line}, col: {col})".format(
token=token,
index=index + 1,
tok_type=tok_type.__name__,
req_tok_type=first_tok_type.__name__,
line=lineno(self._loc, self._instring),
col=col(self._loc, self._instring),
)
)
if first_tok_type is ConfigTree:
result = ConfigTree()
for token in tokens:
for key, val in token.items():
# update references for substituted contents
if isinstance(val, ConfigValues):
val.parent = result
val.key = key
result[key] = val
return result
elif first_tok_type is ConfigList:
result = []
main_index = 0
for sublist in tokens:
sublist_result = ConfigList()
for token in sublist:
if isinstance(token, ConfigValues):
token.parent = result
token.key = main_index
main_index += 1
sublist_result.append(token)
result.extend(sublist_result)
return [result]
else:
if len(tokens) == 1:
return tokens[0]
else:
return "".join(format_str(token) for token in tokens[:-1]) + format_str(tokens[-1])
def line_col(primitive):
loc = location(primitive)
src = source(primitive)
if src and loc:
return lineno(loc, src), col(loc, src)
else:
return None, None
def checkPeerIndent( s, l, t ): c = p.col( l, s ) print( c, indentStack[-1] ) if( c != indentStack[-1] ): if( ( not indentStack ) or c > indentStack[-1] ): raise p.ParseFatalException( s, l, "illegal nesting" ) raise p.ParseException( s, l, "not a peer entry" )
def pyparsingLoggingStartDebugAction(instring, loc, expr):
row = pyparsing.lineno(loc, instring)
col = pyparsing.col(loc, instring)
pplogger.debug("Match Start: expr: `%s` at loc `%s`, row: `%d`, col: `%d`",
expr, loc, row, col)
def perform_anomality_check_structured_stream(self):
process_row_udf = udf(lambda heart_beat: AnomalyDetector.check_anomality(self, heart_beat))
spark_session = SparkSession.builder \
.appName('EKG structured streaming') \
.getOrCreate()
ekg_signal_schema = StructType(
[StructField('timestamp', StringType()),
StructField('signal_values', ArrayType(FloatType()))
])
ekg_stream_df = spark_session \
.readStream \
.format('kafka') \
.option('kafka.bootstrap.servers', 'localhost:9092') \
.option('subscribe', 'ekg-stream') \
.option('startingOffsets', 'earliest') \
.load() \
.select(from_json(col('value').cast('string'), ekg_signal_schema).alias('value'))
parsed_ekg_stream_df = ekg_stream_df.select(['value.timestamp', 'value.signal_values'])
# .withColumn('signal_values', process_row_udf('signal_values'))
parsed_ekg_stream_df.printSchema()
query = parsed_ekg_stream_df \
.writeStream \
.format('console') \
.option('truncate', False) \
.start()
query.awaitTermination()
def getMessage(pstr, pos, filepath=''):
line = pyparsing.line(pos, pstr);
lineno = pyparsing.lineno(pos, pstr);
col = pyparsing.col(pos, pstr)
arrow = ( '-' * (col-1) + '^');
ls = os.linesep + ' ';
return ' in file {filepath} (line: {lineno}, col: {col}){ls}{line}{ls}{arrow}'.format(**locals());
def _parse_action_obj(self, source, idx, tokin):
value = tokin[0]
return [{
'type': 'obj',
'line': pp.lineno(idx, source),
'col': pp.col(idx, source),
'key': (value[0], value[1])
}]
def _parse_action_group(self, source, idx, tokin):
value = tokin
return [{
'type': 'group',
'line': pp.lineno(idx, source),
'col': pp.col(idx, source),
'key': (value[0], )
}]
def _parse_action_block(self, source, idx, tokin):
value = tokin[0].asList()
return [{
'type': 'block',
'line': pp.lineno(idx, source),
'col': pp.col(idx, source),
'statements': value
}]
def find_cursor(source):
"""Return (source, line, col) based on the | character, stripping the source."""
source = source.strip()
i = source.index('|')
assert i != -1
l = pyparsing.lineno(i, source)
c = pyparsing.col(i, source)
return source.replace('|', ''), l, c
def _location (orig_string, locn):
if not _debug_info_enabled:
return None
assert _source_file is not None
return instruction.SourceLocation (srcfile=os.path.abspath (_source_file),
line=pp.lineno (locn, orig_string),
column=pp.col (locn, orig_string))
def pyparsingLoggingExceptionDebugAction(instring, loc, expr, exc):
row = pyparsing.lineno(loc, instring)
col = pyparsing.col(loc, instring)
pplogger.error(
"Caught Exception: expr: `%s`, loc: `%s`, row: `%d`, col: `%d`, exception: `%s`",
expr, loc, row, col, exc)
def _parse_action_obj(self, source, idx, tokin):
value = tokin[0].asDict()
return [{'type': 'obj',
'line': pp.lineno(idx, source),
'col': pp.col(idx, source),
'id_type': value.get('id_type'),
'id_fixed': value.get('id_fixed'),
'key': (value.get('class'), value.get('id_fixed', 'xxx'))}]
def checkUnindent( s, l, t ): if( l >= len( s ) ): return c = p.col( l, s ) if( not ( c < indentStack[-1] and c <= indentStack[-2] ) ): raise p.ParseException( s, l, "not an unindent" )
def transform(self):
def determine_type(token):
return ConfigTree if isinstance(token, ConfigTree) else ConfigList if isinstance(token, list) else str
def format_str(v, last=False):
if isinstance(v, ConfigQuotedString):
return v.value + ('' if last else v.ws)
else:
return '' if v is None else str(v)
if self.has_substitution():
return self
# remove None tokens
tokens = [token for token in self.tokens if token is not None]
if not tokens:
return None
# check if all tokens are compatible
first_tok_type = determine_type(tokens[0])
for index, token in enumerate(tokens[1:]):
tok_type = determine_type(token)
if first_tok_type is not tok_type:
raise ConfigWrongTypeException(
"Token '{token}' of type {tok_type} (index {index}) must be of type {req_tok_type} (line: {line}, col: {col})".format(
token=token,
index=index + 1,
tok_type=tok_type.__name__,
req_tok_type=first_tok_type.__name__,
line=lineno(self._loc, self._instring),
col=col(self._loc, self._instring)))
if first_tok_type is ConfigTree:
result = ConfigTree()
for token in tokens:
ConfigTree.merge_configs(result, token, copy_trees=True)
return result
elif first_tok_type is ConfigList:
result = []
main_index = 0
for sublist in tokens:
sublist_result = ConfigList()
for token in sublist:
if isinstance(token, ConfigValues):
token.parent = result
token.key = main_index
main_index += 1
sublist_result.append(token)
result.extend(sublist_result)
return result
else:
if len(tokens) == 1:
if isinstance(tokens[0], ConfigQuotedString):
return tokens[0].value
return tokens[0]
else:
return ''.join(format_str(token) for token in tokens[:-1]) + format_str(tokens[-1], True)
def transform(self):
def determine_type(token):
return ConfigTree if isinstance(token, ConfigTree) else ConfigList if isinstance(token, list) else str
def format_str(v, last=False):
if isinstance(v, ConfigQuotedString):
return v.value + ('' if last else v.ws)
else:
return '' if v is None else str(v)
if self.has_substitution():
return self
# remove None tokens
tokens = [token for token in self.tokens if token is not None]
if not tokens:
return None
# check if all tokens are compatible
first_tok_type = determine_type(tokens[0])
for index, token in enumerate(tokens[1:]):
tok_type = determine_type(token)
if first_tok_type is not tok_type:
raise ConfigWrongTypeException(
"Token '{token}' of type {tok_type} (index {index}) must be of type {req_tok_type} (line: {line}, col: {col})".format(
token=token,
index=index + 1,
tok_type=tok_type.__name__,
req_tok_type=first_tok_type.__name__,
line=lineno(self._loc, self._instring),
col=col(self._loc, self._instring)))
if first_tok_type is ConfigTree:
result = ConfigTree()
for token in tokens:
ConfigTree.merge_configs(result, token, copy_trees=True)
return result
elif first_tok_type is ConfigList:
result = []
main_index = 0
for sublist in tokens:
sublist_result = ConfigList()
for token in sublist:
if isinstance(token, ConfigValues):
token.parent = result
token.key = main_index
main_index += 1
sublist_result.append(token)
result.extend(sublist_result)
return result
else:
if len(tokens) == 1:
if isinstance(tokens[0], ConfigQuotedString):
return tokens[0].value
return tokens[0]
else:
return ''.join(format_str(token) for token in tokens[:-1]) + format_str(tokens[-1], True)
def start_action(self, instring, loc, expr):
self.writer.startElement('attempt',
attributes={
'class': str(expr.__class__.__name__),
'loc': str(repr(loc)),
'expr': str(repr(expr)),
'lineno': str(lineno(loc, instring)),
'col': str(col(loc, instring)),
})
def test(session, f, prefix):
colName = f.name
if prefix is not None:
colName = prefix + "." + f.name
if f.dataType == StructType:
return flattenSchema(session, f.dataType, colName)
else:
return [col(colName)]
def _parse_action_attr(self, source, idx, tokin):
value = tokin[0]
tokout = {'type': 'attr',
'line': pp.lineno(idx, source),
'col': pp.col(idx, source),
'key': (value[0], value[1])}
if len(value) > 2:
tokout['comment'] = value[2][1:].strip()
return [tokout]
def expand_state_definition(source, loc, tokens):
"""
Parse action to convert statemachine to corresponding Python classes and methods
"""
indent = " " * (pp.col(loc, source) - 1)
statedef = []
# build list of states
states = set()
fromTo = {}
for tn in tokens.transitions:
states.add(tn.from_state)
states.add(tn.to_state)
fromTo[tn.from_state] = tn.to_state
# define base class for state classes
baseStateClass = tokens.name
statedef.extend([
"class %s(object):" % baseStateClass,
" def __str__(self):",
" return self.__class__.__name__",
" @classmethod",
" def states(cls):",
" return list(cls.__subclasses__())",
" def next_state(self):",
" return self._next_state_class()",
])
# define all state classes
statedef.extend("class {}({}): pass".format(s, baseStateClass)
for s in states)
# define state->state transitions
statedef.extend("{}._next_state_class = {}".format(s, fromTo[s])
for s in states if s in fromTo)
statedef.extend([
"class {baseStateClass}Mixin:".format(baseStateClass=baseStateClass),
" def __init__(self):",
" self._state = None",
" def initialize_state(self, init_state):",
" if issubclass(init_state, {baseStateClass}):".format(
baseStateClass=baseStateClass),
" init_state = init_state()",
" self._state = init_state",
" @property",
" def state(self):",
" return self._state",
" # get behavior/properties from current state",
" def __getattr__(self, attrname):",
" attr = getattr(self._state, attrname)",
" return attr",
" def __str__(self):",
" return '{0}: {1}'.format(self.__class__.__name__, self._state)",
])
return ("\n" + indent).join(statedef) + "\n"
def action(s, loc, token):
_lineno = lineno(loc, s)
_colno = col(loc, s) - 1
logger.info("log={0} (lineno={1}, col={2})".format(loc, _lineno, _colno))
logger.debug("comment: {0}".format(token[0]))
logger.debug("comment: {0}".format(nodeInfo(token)))
_comment = ast.Comment(token[0][2:])
_comment.loc_info = (_lineno, _colno)
return _comment
def checkUnindent(s, l, t): # pylint: disable=invalid-name
del t
if l >= len(s):
return
curCol = pp.col(l, s) # pylint: disable=invalid-name
if not (_indentation_stack and curCol < _indentation_stack[-1]
and curCol <= _indentation_stack[-2]):
raise pp.ParseException(s, l, 'not an unindent')
_indentation_stack.pop()
def resolve_substitutions(cls, config, accept_unresolved=False):
has_unresolved = False
cls._fixup_self_references(config, accept_unresolved)
substitutions = cls._find_substitutions(config)
if len(substitutions) > 0:
unresolved = True
any_unresolved = True
_substitutions = []
cache = {}
while any_unresolved and len(substitutions) > 0 and set(substitutions) != set(_substitutions):
unresolved = False
any_unresolved = True
_substitutions = substitutions[:]
for substitution in _substitutions:
is_optional_resolved, resolved_value = cls._resolve_variable(config, substitution)
# if the substitution is optional
if not is_optional_resolved and substitution.optional:
resolved_value = None
if isinstance(resolved_value, ConfigValues):
parents = cache.get(resolved_value)
if parents is None:
parents = []
link = resolved_value
while isinstance(link, ConfigValues):
parents.append(link)
link = link.overriden_value
cache[resolved_value] = parents
if isinstance(resolved_value, ConfigValues) \
and substitution.parent in parents \
and hasattr(substitution.parent, 'overriden_value') \
and substitution.parent.overriden_value:
# self resolution, backtrack
resolved_value = substitution.parent.overriden_value
unresolved, new_substitutions, result = cls._do_substitute(
substitution, resolved_value, is_optional_resolved)
any_unresolved = unresolved or any_unresolved
substitutions.extend(new_substitutions)
if not isinstance(result, ConfigValues):
substitutions.remove(substitution)
cls._final_fixup(config)
if unresolved:
has_unresolved = True
if not accept_unresolved:
raise ConfigSubstitutionException("Cannot resolve {variables}. Check for cycles.".format(
variables=', '.join('${{{variable}}}: (line: {line}, col: {col})'.format(
variable=substitution.variable,
line=lineno(substitution.loc, substitution.instring),
col=col(substitution.loc, substitution.instring)) for substitution in substitutions)))
cls._final_fixup(config)
return has_unresolved
def resolve_substitutions(cls, config, accept_unresolved=False):
has_unresolved = False
cls._fixup_self_references(config, accept_unresolved)
substitutions = cls._find_substitutions(config)
if len(substitutions) > 0:
unresolved = True
any_unresolved = True
_substitutions = []
cache = {}
while any_unresolved and len(substitutions) > 0 and set(substitutions) != set(_substitutions):
unresolved = False
any_unresolved = True
_substitutions = substitutions[:]
for substitution in _substitutions:
is_optional_resolved, resolved_value = cls._resolve_variable(config, substitution)
# if the substitution is optional
if not is_optional_resolved and substitution.optional:
resolved_value = None
if isinstance(resolved_value, ConfigValues):
parents = cache.get(resolved_value)
if parents is None:
parents = []
link = resolved_value
while isinstance(link, ConfigValues):
parents.append(link)
link = link.overriden_value
cache[resolved_value] = parents
if isinstance(resolved_value, ConfigValues) \
and substitution.parent in parents \
and hasattr(substitution.parent, 'overriden_value') \
and substitution.parent.overriden_value:
# self resolution, backtrack
resolved_value = substitution.parent.overriden_value
unresolved, new_substitutions, result = cls._do_substitute(substitution, resolved_value, is_optional_resolved)
any_unresolved = unresolved or any_unresolved
substitutions.extend(new_substitutions)
if not isinstance(result, ConfigValues):
substitutions.remove(substitution)
cls._final_fixup(config)
if unresolved:
has_unresolved = True
if not accept_unresolved:
raise ConfigSubstitutionException("Cannot resolve {variables}. Check for cycles.".format(
variables=', '.join('${{{variable}}}: (line: {line}, col: {col})'.format(
variable=substitution.variable,
line=lineno(substitution.loc, substitution.instring),
col=col(substitution.loc, substitution.instring)) for substitution in substitutions)))
cls._final_fixup(config)
return has_unresolved
def pos_str(self):
if self.text:
lineno_ = lineno(self.loc, self.text)
col_ = col(self.loc, self.text)
pos = f'{lineno_}:{col_}'
else:
pos = self.loc
return f'{self.filename}:{pos}'
def _parse_action_obj(self, source, idx, tokin):
value = tokin[0].asDict()
return [{
'type': 'obj',
'line': pp.lineno(idx, source),
'col': pp.col(idx, source),
'id_type': value.get('id_type'),
'id_fixed': value.get('id_fixed'),
'key': (value.get('class'), value.get('id_fixed', 'xxx'))
}]
def __str__(self):
#TODO: create error message which is understood by Pydev. Format:
#File "/home/eike/codedir/freeode/trunk/freeode_py/simlparser.py", line 956, in createProcess
if self.str == None:
return 'Error! ' + self.message + '\n At position: ' + str(self.loc)
else:
lineno = pyparsing.lineno(self.loc, self.str)
col = pyparsing.col(self.loc, self.str)
return 'Error! %s \n' % self.message +\
'Line: %d, Column: %d' % (lineno, col)
def start_action(self, instring, loc, expr):
self.writer.startElement(
u"attempt",
attributes={
u"class": unicode(expr.__class__.__name__),
u"loc": unicode(repr(loc)),
u"expr": unicode(repr(expr)),
u"lineno": unicode(lineno(loc, instring)),
u"col": unicode(col(loc, instring)),
},
)
def start_action(self, instring, loc, expr):
self.writer.startElement(u'attempt',
attributes={
u'class':
unicode(expr.__class__.__name__),
u'loc': unicode(repr(loc)),
u'expr': unicode(repr(expr)),
u'lineno':
unicode(lineno(loc, instring)),
u'col': unicode(col(loc, instring)),
})
def _parse_action_attr(self, source, idx, tokin):
value = tokin[0]
tokout = {
'type': 'attr',
'line': pp.lineno(idx, source),
'col': pp.col(idx, source),
'key': (value[0], value[1])
}
if len(value) > 2:
tokout['comment'] = value[2][1:].strip()
return [tokout]
def __str__(self):
#TODO: create error message which is understood by Pydev.
errMsg = 'Error!\n'
for msg1, loc1 in self.errTupList:
lineno, col = 0, 0
if self.str:
lineno = pyparsing.lineno(loc1, self.str)
col = pyparsing.col(loc1, self.str)
errMsg += '%s \n' % msg1 +\
'Line: %d, Column: %d\n' % (lineno, col)
return errMsg
def in_string(location, code):
"""Determines if the given location is in a string inside of code.
Does not detect triple-quoted multi-line strings."""
str_char = None
for c in line(location, code)[:col(location, code) - 1]:
if c == str_char:
str_char = None
elif c in "\"'":
str_char = c
return str_char is not None
def __init__(self, message, print_location=True):
super(SemanticException, self).__init__()
self._message = message
self.location = exshared.location
self.print_location = print_location
if exshared.location is not None:
self.line = lineno(exshared.location, exshared.text)
self.col = col(exshared.location, exshared.text)
self.text = line(exshared.location, exshared.text)
else:
self.line = self.col = self.text = None
def parse(self,string):
try:
self.gramma().parseString(string, parseAll=True)
except ParseException as e:
raise
seen_procs = [str(x) for x in self._processes]
for seen in self._seen:
if seen not in seen_procs:
line = lineno(self._seen[seen][0], self._seen[seen][1])
column =col(self._seen[seen][0], self._seen[seen][1])
raise ProcessNotDefinedError("{} process not defined - possible deadlock, line {}, col {}".format(seen,line, column))
return (self._processes, self._var_stack, self._systemeq, self._actions)
def read_include_contents(s, l, t):
include_file_ref = t.include_file_name
include_echo = "/* {} */".format(pp.line(l, s).strip())
# guard against recursive includes
if include_file_ref not in seen:
seen.add(include_file_ref)
included_file_contents = Path(include_file_ref).read_text()
return (include_echo + '\n' +
include_directive.transformString(included_file_contents))
else:
lead = ' ' * (pp.col(l, s) - 1)
return "/* recursive include! */\n{}{}".format(lead, include_echo)
def __init__(self, filename, string,
character=None, line=None, column=None):
self.filename = filename
self.string = string
if character is None:
assert line is not None and column is not None
self.line = line
self.col = column
self.character = None
else:
assert line is None and column is None
self.character = character
self.line = lineno(character, string)
self.col = col(character, string)
def assertParses(self, syntax, text):
try:
return self.parse(syntax, text)
except p.ParseBaseException as exc:
if hasattr(exc, 'col') and hasattr(exc, 'lineno'):
lineno = exc.lineno
col = exc.col
else:
lineno = p.lineno(exc.loc, text)
col = p.col(exc.loc, text)
print()
print("Parse error on line {} column {}: {}".format(
lineno, col, exc))
self._show_text(text, lineno, col, context=3)
raise
def remove_indent(multiline, indent):
"""
Generate the lines removing the indent
"""
for line in multiline.splitlines():
if line and not line[:indent].isspace():
warn("%s: %s: under-indented multiline string "
"truncated: '%s'" %
(lineno(loc, s), col(loc, s), line),
LettuceSyntaxWarning)
# for those who are surprised by this, slicing a string
# shorter than indent will yield empty string, not IndexError
yield line[indent:]
def _parse_action_attr(self, source, idx, tokin):
value = tokin[0].asDict()
ref_path = value.get('ref_path')
if ref_path is not None:
ref_path = " ".join(ref_path.asList())
tokout = {'type': 'attr',
'line': pp.lineno(idx, source),
'col': pp.col(idx, source),
'ref_path': ref_path,
'val_type': value.get('val_type'),
'val_dfl': value.get('val_dfl', NO_VALUE),
'required': value.get('val_dfl', NO_VALUE) == NO_VALUE,
'comment': value.get('comment'),
'key': (value.get('name'), 'xxx')}
return [tokout]
def __init__(self, s, loc, tokens):
super(TaggedBlock, self).__init__(s, loc, tokens)
token = tokens[0]
self._tags = list(token.tags)
self.keyword = token.keyword
self.name = token.name.strip()
if self.name == '':
raise LettuceSyntaxError(
None,
"{line}:{col} {klass} must have a name".format(
line=lineno(loc, s),
col=col(loc, s),
klass=self.__class__.__name__))
def FailFunc(s, loc, expr, err, expected=''):
#print '*'*60 + " FailFunc called " + '*'*60
if False:
#print "Expected =", expected
print 'got here'
#print s
print loc
print repr(s[loc:])
#print expr
#print err, repr(err)
print err
print pyparsing.lineno(loc, s)
# Get the remaining string to parse
remainingString = s[loc:]
# This function will always get called at the end of the input string, however, all that is
# left will be some empty lines, so only check for errors if there is valid text left.
if not remainingString.strip():
return
# Check the keyword of the remaining input string
keyword = GetKeyword(remainingString)
# Handle the comment header
if keyword.startswith( '/**' ) :
if False:
print 'Got comment header'
print "expected =", expected
print err
print repr(remainingString)
print '-'*40
# Skip over the comment header and continue. Note that 'loc' must be updated to reflect the
# new starting position, but 'err' has the correct position, if it is used.
initialCommentHeader, remainingString = remainingString.split('*/', 1)
keyword = GetKeyword(remainingString)
#print 'keyword = ', keyword
# Update 'loc':
# - +2 since the '*/' is not part of the initial comment header
# - additional offset required, since keyword may not be at the start of remainingString,
# if there is whitespace between '*/' and keyword.
loc += ( len(initialCommentHeader) + 2 + remainingString.index(keyword) )
#print 'keyword = ', keyword
# The 'ADD_HANDLER_PARAMS' and 'HANDLER_PARAMS' keywords are optional within a HANDLER expression
# and can appear in any order within the expression. Thus, the associated parse expressions
# could fail when we reach the end of the HANDLER definition, and so this is not an error.
if expected in [ StringHandlerParams, StringAddHandlerParams ] and keyword[0] == '}':
#print "Got %s special case" % expected
#print '*'*80
return
# Set default error message; may get overwritten below
errMsg = err.msg
# If it is a valid keyword but not the expected one, then this is not an error.
if keyword in KeywordList:
if keyword != expected:
if False:
print "Expected '%s', got '%s'" % (expected, keyword)
print '='*80
return
# A valid failure/error has occurred.
# If the 'err' data is valid, it will give a more accurate location, as long as the 'loc'
# attribute has been set. For exceptions generated within this file, 'loc' is not set.
# The default value of 'loc' is zero, so checking against zero should be a valid way of
# determining if it was set, since we should never get here if 'loc' is legitimately zero.
if err and err.loc != 0:
lineno = err.lineno
col = err.col
else:
lineno = pyparsing.lineno(loc, s)
col = pyparsing.col(loc, s)
# This is an unknown keyword, which is also an error
else:
#print "Unknown keyword '%s'" % keyword
lineno = pyparsing.lineno(loc, s)
col = pyparsing.col(loc, s)
errMsg = "unknown keyword '%s'" % keyword
# It is a real error, so print out the error message, and exit right away
PrintErrorMessage(s, lineno, col, errMsg)
sys.exit(1)
def check_unindent(str, location, tokens):
if location >= len(str):
return
cur_col = col(location, str)
if not(cur_col < indent_stack[-1] and cur_col <= indent_stack[-2]):
raise ParseException(str, location, "not an unindent")
def __init__(self, text, loc, msg):
self.line = line(loc, text)
self.col = col(loc, text)
self.lineno = lineno(loc, text)
self.msg = msg
