def check_equals_true(self, code): keyword = Literal("true") | Literal("false") statement_parser = Group("==" + keyword) | Group(keyword + "==") if len(statement_parser.searchString(code)): self.add_error(label="EQUALS_TRUE")
def check_equals_true(self, code): variable = Word(alphanums) keyword = Literal("true") | Literal("false") statement_parser = Group(variable + "==" + keyword) | Group(keyword + "==" + variable) if len(statement_parser.searchString(code)): self.add_error("EQUALS_TRUE")
class Parser(object): """A parser class for solving simple data accesses and super-indexing data :param data: Trace Object :type data: instance of :mod:`trappy.ftrace.BareTrace` or a child class (like :mod:`trappy.ftrace.FTrace`) :param pvars: A dictionary of variables that need to be accessed from within the grammar :type pvars: dict :param method: The method to be used for reindexing data This can be one of the standas :mod:`pandas.DataFrame` methods (eg. pad, bfill, nearest). The default is pad or use the last valid observation. :type method: str :param limit: The number of indices a value will be propagated when reindexing. The default is None :type limit: int :param fill: Whether to fill the NaNs in the data. The default value is True. :type fill: bool :param window: A window of time in which to apply the data accesses. By default the data accesses happen accross the whole trace. With the window parameter you can limit it to a window of time inside the trace. The first element of the tuple is the starting time and the second the ending time (set to None for end of trace). :type window: tuple - **Operators** +----------------+----------------------+---------------+ | Operation | operator | Associativity | +================+======================+===============+ | Exponentiation | \*\* | Left | +----------------+----------------------+---------------+ |Unary | \- | Right | +----------------+----------------------+---------------+ | Multiply/Divide| \*, /, //, % | Left | +----------------+----------------------+---------------+ | Add/Subtract | +, \-, | Left | +----------------+----------------------+---------------+ | Comparison | >, <, >=, <=, ==, != | Left | +----------------+----------------------+---------------+ | Logical | &&, ||, \|, & | Left | +----------------+----------------------+---------------+ - **Data Accessors** Since the goal of the grammar is to provide an easy language to access and compare data from a :mod:`trappy.trace.FTrace` object. The parser provides a simple notation to access this data. *Statically Defined Events* :: import trappy from trappy.stats.grammar import Parser trace = trappy.FTrace("path/to/trace/file") parser = Parser(trace) parser.solve("trappy.thermal.Thermal:temp * 2") *Aliasing* :: import trappy from trappy.stats.grammar import Parser pvars = {} pvars["THERMAL"] = trappy.thermal.Thermal trace = trappy.FTrace("path/to/trace/file") parser = Parser(trace) parser.solve("THERMAL:temp * 2") *Using Event Name* :: import trappy from trappy.stats.grammar import Parser trace = trappy.FTrace("path/to/trace/file") parser = Parser(trace) parser.solve("thermal:temp * 2") The event :mod:`trappy.thermal.Thermal` is aliased as **THERMAL** in the grammar *Dynamic Events* :: import trappy from trappy.stats.grammar import Parser # Register Dynamic Event cls = trappy.register_dynamic_ftrace("my_unique_word", "event_name") pvars = {} pvars["CUSTOM"] = cls trace = trappy.FTrace("path/to/trace/file") parser = Parser(trace) parser.solve("CUSTOM:col * 2") .. seealso:: :mod:`trappy.dynamic.register_dynamic_ftrace` """ def __init__(self, data, pvars=None, window=(0, None), **kwargs): if pvars is None: pvars = {} self.data = data self._pvars = pvars self._accessor = Group( FUNC_NAME + COLON + IDENTIFIER).setParseAction(self._pre_process) self._inspect = Group( FUNC_NAME + COLON + IDENTIFIER).setParseAction(self._parse_for_info) self._parse_expr = get_parse_expression( self._parse_func, self._parse_var_id) self._agg_df = pd.DataFrame() self._pivot_set = set() self._limit = kwargs.get("limit", StatConf.REINDEX_LIMIT_DEFAULT) self._method = kwargs.get("method", StatConf.REINDEX_METHOD_DEFAULT) self._fill = kwargs.get("fill", StatConf.NAN_FILL_DEFAULT) self._window = window def solve(self, expr): """Parses and solves the input expression :param expr: The input expression :type expr: str :return: The return type may vary depending on the expression. For example: **Vector** :: import trappy from trappy.stats.grammar import Parser trace = trappy.FTrace("path/to/trace/file") parser = Parser(trace) parser.solve("trappy.thermal.Thermal:temp * 2") **Scalar** :: import trappy from trappy.stats.grammar import Parser trace = trappy.FTrace("path/to/trace/file") parser = Parser(trace) parser.solve("numpy.mean(trappy.thermal.Thermal:temp)") **Vector Mask** :: import trappy from trappy.stats.grammar import Parser trace = trappy.FTrace("path/to/trace/file") parser = Parser(trace) parser.solve("trappy.thermal.Thermal:temp > 65000") """ # Pre-process accessors for indexing self._accessor.searchString(expr) return self._parse_expr.parseString(expr)[0] """ # Pre-process accessors for indexing self._accessor.searchString(expr) return self._parse_expr.parseString(expr)[0] """ # Pre-process accessors for indexing self._accessor.searchString(expr) return self._parse_expr.parseString(expr)[0] def _pivot(self, cls, column): """Pivot Data for concatenation""" data_frame = self._get_data_frame(cls) data_frame = handle_duplicate_index(data_frame) new_index = self._agg_df.index.union(data_frame.index) if hasattr(cls, "pivot") and cls.pivot: pivot = cls.pivot pivot_vals = list(np.unique(data_frame[pivot].values)) data = {} for val in pivot_vals: data[val] = data_frame[data_frame[pivot] == val][[column]] if len(self._agg_df): data[val] = data[val].reindex( index=new_index, method=self._method, limit=self._limit) return pd.concat(data, axis=1).swaplevel(0, 1, axis=1) if len(self._agg_df): data_frame = data_frame.reindex( index=new_index, method=self._method, limit=self._limit) return pd.concat({StatConf.GRAMMAR_DEFAULT_PIVOT: data_frame[ [column]]}, axis=1).swaplevel(0, 1, axis=1) def _pre_process(self, tokens): """Pre-process accessors for super-indexing""" params = tokens[0] if params[1] in self._agg_df.columns: return self._agg_df[params[1]] cls = params[0] column = params[1] if cls in self._pvars: cls = self._pvars[cls] elif cls in self.data.class_definitions: cls = self.data.class_definitions[cls] else: cls = str_to_attr(cls) data_frame = self._pivot(cls, column) self._agg_df = pd.concat( [self._agg_df, data_frame], axis=1) if self._fill: self._agg_df = self._agg_df.fillna(method="pad") return self._agg_df[params[1]] def _parse_for_info(self, tokens): """Parse Action for inspecting data accessors""" params = tokens[0] cls = params[0] column = params[1] info = {} info["pivot"] = None info["pivot_values"] = None if cls in self._pvars: cls = self._pvars[cls] elif cls in self.data.class_definitions: cls = self.data.class_definitions[cls] else: cls = str_to_attr(cls) data_frame = self._get_data_frame(cls) info["class"] = cls info["length"] = len(data_frame) if hasattr(cls, "pivot") and cls.pivot: info["pivot"] = cls.pivot info["pivot_values"] = list(np.unique(data_frame[cls.pivot])) info["column"] = column info["column_present"] = column in data_frame.columns return info def _parse_var_id(self, tokens): """A function to parse a variable identifier """ params = tokens[0] try: return float(params) except (ValueError, TypeError): try: return self._pvars[params] except KeyError: return self._agg_df[params[1]] def _parse_func(self, tokens): """A function to parse a function string""" params = tokens[0] func_name = params[0] if func_name in self._pvars and isinstance( self._pvars[func_name], types.FunctionType): func = self._pvars[func_name] else: func = str_to_attr(params[0]) return func(*params[1]) def _get_data_frame(self, cls): """Get the data frame from the BareTrace object, applying the window if set""" data_frame = getattr(self.data, cls.name).data_frame if self._window[1] is None: data_frame = data_frame.loc[self._window[0]:] else: data_frame = data_frame.loc[self._window[0]:self._window[1]] return data_frame def ref(self, mask): """Reference super indexed data with a boolean mask :param mask: A boolean :mod:`pandas.Series` that can be used to reference the aggregated data in the parser :type mask: :mod:`pandas.Series` :return: aggregated_data[mask] """ return self._agg_df[mask] def inspect(self, accessor): """A function to inspect the accessor for information :param accessor: A data accessor of the format <event>:<column> :type accessor: str :return: A dictionary of information """ return self._inspect.parseString(accessor)[0]
class Parser(object): """A parser class for solving simple data accesses and super-indexing data :param pvars: A dictionary of variables that need to be accessed from within the grammar :type pvars: dict :param topology: Future support for the usage of topologies in grammar :type topology: :mod:`trappy.stats.Topology` - **Operators** +----------------+----------------------+---------------+ | Operation | operator | Associativity | +================+======================+===============+ |Unary | \- | Right | +----------------+----------------------+---------------+ | Multiply/Divide| \*, / | Left | +----------------+----------------------+---------------+ | Add/Subtract | +, \-, | Left | +----------------+----------------------+---------------+ | Comparison | >, <, >=, <=, ==, != | Left | +----------------+----------------------+---------------+ | Logical | &&, ||, \|, & | Left | +----------------+----------------------+---------------+ - **Data Accessors** Since the goal of the grammar is to provide an easy language to access and compare data from a :mod:`trappy.run.Run` object. The parser provides a simple notation to access this data. *Statically Defined Events* :: import trappy from trappy.stats.grammar import Parser run = trappy.Run("path/to/trace/file") parser = Parser(run) parser.solve("trappy.thermal.Thermal:temp * 2") *Aliasing* :: import trappy from trappy.stats.grammar import Parser pvars = {} pvars["THERMAL"] = trappy.thermal.Thermal run = trappy.Run("path/to/trace/file") parser = Parser(run) parser.solve("THERMAL:temp * 2") The event :mod:`trappy.thermal.Thermal` is aliased as **THERMAL** in the grammar *Dynamic Events* :: import trappy from trappy.stats.grammar import Parser # Register Dynamic Event cls = trappy.register_dynamic("my_unique_word", "event_name") pvars = {} pvars["CUSTOM"] = cls run = trappy.Run("path/to/trace/file") parser = Parser(run) parser.solve("CUSTOM:col * 2") .. seealso:: :mod:`trappy.dynamic.register_dynamic` """ def __init__(self, data, pvars=None, topology=None): if pvars is None: pvars = {} self.data = data self._pvars = pvars self._accessor = Group(FUNC_NAME + COLON + IDENTIFIER).setParseAction( self._pre_process) self._parse_expr = get_parse_expression(self._parse_func, self._parse_var_id) self._agg_df = pd.DataFrame() if not topology: self.topology = Topology() else: self.topology = topology self._pivot_set = set() self._index_limit = None def solve(self, expr): """Parses and solves the input expression :param expr: The input expression :type expr: str :return: The return type may vary depending on the expression. For example: **Vector** :: import trappy from trappy.stats.grammar import Parser run = trappy.Run("path/to/trace/file") parser = Parser(run) parser.solve("trappy.thermal.Thermal:temp * 2") **Scalar** :: import trappy from trappy.stats.grammar import Parser run = trappy.Run("path/to/trace/file") parser = Parser(run) parser.solve("numpy.mean(trappy.thermal.Thermal:temp)") **Vector Mask** :: import trappy from trappy.stats.grammar import Parser run = trappy.Run("path/to/trace/file") parser = Parser(run) parser.solve("trappy.thermal.Thermal:temp > 65000") """ # Pre-process accessors for indexing self._accessor.searchString(expr) return self._parse_expr.parseString(expr)[0] """ # Pre-process accessors for indexing self._accessor.searchString(expr) return self._parse_expr.parseString(expr)[0] """ # Pre-process accessors for indexing self._accessor.searchString(expr) return self._parse_expr.parseString(expr)[0] def _pivot(self, cls, column): """Pivot Data for concatenation""" data_frame = getattr(self.data, cls.name).data_frame if hasattr(cls, "pivot") and cls.pivot: pivot = cls.pivot pivot_vals = list(np.unique(data_frame[pivot].values)) data = {} for val in pivot_vals: data[val] = data_frame[data_frame[pivot] == val][[column]] if len(self._agg_df): data[val] = data[val].reindex(index=self._agg_df.index, method="nearest", limit=1) return pd.concat(data, axis=1).swaplevel(0, 1, axis=1) if len(self._agg_df): data_frame = data_frame.reindex(index=self._agg_df.index, method="nearest", limit=1) return pd.concat({ StatConf.GRAMMAR_DEFAULT_PIVOT: data_frame[[column]] }, axis=1).swaplevel(0, 1, axis=1) def _pre_process(self, tokens): """Pre-process accessors for super-indexing""" params = tokens[0] if params[1] in self._agg_df.columns: return self._agg_df[params[1]] cls = params[0] column = params[1] if cls in self._pvars: cls = self._pvars[cls] else: cls = str_to_attr(cls) data_frame = self._pivot(cls, column) self._agg_df = pd.concat([self._agg_df, data_frame], axis=1) return self._agg_df[params[1]] def _parse_var_id(self, tokens): """A function to parse a variable identifier """ params = tokens[0] try: return float(params) except (ValueError, TypeError): try: return self._pvars[params] except KeyError: return self._agg_df[params[1]] def _parse_func(self, tokens): """A function to parse a function string""" params = tokens[0] func_name = params[0] if func_name in self._pvars and isinstance(self._pvars[func_name], types.FunctionType): func = self._pvars[func_name] else: func = str_to_attr(params[0]) return func(*params[1]) def ref(self, mask): """Reference super indexed data with a boolean mask :param mask: A boolean :mod:`pandas.Series` that can be used to reference the aggregated data in the parser :type mask: :mod:`pandas.Series` :return: aggregated_data[mask] """ return self._agg_df[mask]