def render_output_analysis(self, renderer):
"""Render analysis of the expression's return type and its members."""
output_type = infer_type.infer_type(self.query, self)
renderer.section("Type Analysis", width=140)
renderer.table_header([
dict(name="Name", cname="name", type="TreeNode", max_depth=2,
width=60),
dict(name="Type", cname="type", width=40)
])
renderer.table_row(self.query.source,
repr(output_type),
depth=1)
try:
for member in structured.getmembers(output_type):
subq = "(%s)[%r]" % (self.query.source, member)
subtype = infer_type.infer_type(q.Query(subq), self)
if isinstance(subtype, type):
subtype = subtype.__name__
else:
subtype = repr(subtype)
renderer.table_row(subq, subtype, depth=2)
except (NotImplementedError, TypeError, AttributeError):
pass
def testResolve(self):
self.assertIsa(
infer_type.infer_type(
q.Query("Process.pid"),
mocks.MockRootType),
number.INumber)
self.assertIsa(
infer_type.infer_type(
q.Query("Process.parent.pid"),
mocks.MockRootType),
number.INumber)
def testSelect(self):
self.assertIsa(
infer_type.infer_type(
q.Query("Process['pid']"),
mocks.MockRootType),
number.INumber)
self.assertEqual(
infer_type.infer_type(
q.Query("Process[var_name]"),
mocks.MockRootType),
protocol.AnyType)
def testVariadicExpression(self):
self.assertIsa(
infer_type.infer_type(
q.Query("5 + 5"),
mocks.MockRootType),
number.INumber)
self.assertIsa(
infer_type.infer_type(
q.Query("10 * (1 - 4) / 5"),
mocks.MockRootType),
number.INumber)
def testMap(self):
self.assertIsa(
infer_type.infer_type(
q.Query("Process.parent.pid + 10"),
mocks.MockRootType),
number.INumber)
# Should be the same using shorthand syntax.
self.assertIsa(
infer_type.infer_type(
q.Query("Process.parent.pid - 1"),
mocks.MockRootType),
number.INumber)
def testCount(self):
"""Count is pretty simple."""
self.assertIsa(
infer_type.infer_type(
q.Query(("apply", ("var", "count"), ("repeat", 1, 2, 3))),
mocks.MockRootType),
number.INumber)
def validate(expr, scope):
lhs_type = infer_type.infer_type(expr.lhs, scope)
if not (lhs_type is protocol.AnyType
or protocol.isa(lhs_type, expr.type_signature[0])):
raise errors.EfilterTypeError(root=expr.lhs,
expected=expr.type_signature[0],
actual=lhs_type)
rhs_type = infer_type.infer_type(expr.rhs, scope)
if not (lhs_type is protocol.AnyType
or protocol.isa(rhs_type, expr.type_signature[1])):
raise errors.EfilterTypeError(root=expr.rhs,
expected=expr.type_signature[1],
actual=rhs_type)
return True
def render(self, renderer):
# Do we have a query?
if not self.query:
return self.render_error(renderer)
# Figure out what the header should look like.
# Can we infer the type?
try:
t = infer_type.infer_type(self.query, self)
except Exception:
t = None
# Get the data we're rendering.
try:
rows = self.collect() or []
except errors.EfilterError as error:
self.query_error = error
return self.render_error(renderer)
# If we know the header, great!
if isinstance(t, plugin.TypedProfileCommand):
renderer.table_header(t.table_header)
return self._render_plugin_output(renderer, t.table_header,
*rows)
# Maybe we cached the header when we ran the plugin?
if isinstance(t, plugin.Command):
wrapper = self._cached_plugin_wrappers.get(t.name)
# If we land here there's two options: either the query already
# forced the wrapper to apply (either by explicitly using it as a
# function, or by doing something non-lazy; or the wrapper hasn't
# been called yet. The latter can only be the case if the wrapper is
# being used as a variable, not a function, so we need to
# materialize it with no arguments. If it already has data for a
# call with no args then this won't do anything.
wrapper.materialize()
if wrapper:
renderer.table_header(wrapper.columns)
return self._render_plugin_output(renderer,
wrapper.columns,
*rows)
# Try to guess the header based on structure of the first row.
if not rows:
renderer.table_header([("No Results", "no_results", "20")])
return
rows = iter(rows)
first_row = next(rows)
if isinstance(first_row, dict):
renderer.table_header(
[dict(name=unicode(k), cname=unicode(k))
for k in first_row.iterkeys()])
return self._render_dicts(renderer, first_row, *rows)
# Sigh. Give up, and render whatever you got, I guess.
renderer.table_header([dict(name="Result", cname="result")])
return self._render_whatever_i_guess(renderer, first_row, *rows)
def _render_node(self, query, node, renderer, depth=1):
"""Render an AST node and recurse."""
t = infer_type.infer_type(node, self)
try:
name = "(%s) <%s>" % (t.__name__, type(node).__name__)
except AttributeError:
name = "(%r) <%s>" % (t, type(node).__name__)
renderer.table_row(
name,
utils.AttributedString(
str(query),
[dict(start=node.start, end=node.end, fg="RED", bold=True)]
),
depth=depth
)
for child in node.children:
if isinstance(child, ast.Expression):
self._render_node(node=child, renderer=renderer, query=query,
depth=depth + 1)
else:
renderer.table_row(
"(%s) <leaf: %r>" % (type(child).__name__, child),
None,
depth=depth + 1
)
def infer(query, replacements=None, root_type=None):
"""Determine the type of the query's output without actually running it.
Arguments:
query: A query object or string with the query.
replacements: Built-time parameters to the query, either as dict or as
an array (for positional interpolation).
root_type: The types of variables to be supplied to the query inference.
Returns:
The type of the query's output, if it can be determined. If undecidable,
returns efilter.protocol.AnyType.
NOTE: The inference returns the type of a row in the results, not of the
actual Python object returned by 'apply'. For example, if a query
returns multiple rows, each one of which is an integer, the type of the
output is considered to be int, not a collection of rows.
Examples:
infer("5 + 5") # -> INumber
infer("SELECT * FROM people WHERE age > 10") # -> AnyType
# If root_type implements the IStructured reflection API:
infer("SELECT * FROM people WHERE age > 10", root_type=...) # -> dict
"""
query = q.Query(query, params=replacements)
return infer_type.infer_type(query, root_type)
def validate(expr, scope):
t = infer_type.infer_type(expr.value, scope)
if not protocol.isa(t, boolean.IBoolean):
raise errors.EfilterTypeError(root=expr,
actual=t,
expected=boolean.IBoolean)
return True
def validate(expr, scope):
for subexpr in expr.children:
validate(subexpr, scope)
t = infer_type.infer_type(subexpr, scope)
if not (t is protocol.AnyType
or protocol.isa(t, expr.type_signature)):
raise errors.EfilterTypeError(root=subexpr,
expected=expr.type_signature,
actual=t)
return True
def infer(query, replacements=None, root_type=None,
libs=("stdcore", "stdmath")):
"""Determine the type of the query's output without actually running it.
Arguments:
query: A query object or string with the query.
replacements: Built-time parameters to the query, either as dict or as
an array (for positional interpolation).
root_type: The types of variables to be supplied to the query inference.
libs: What standard libraries should be taken into account for the
inference.
Returns:
The type of the query's output, if it can be determined. If undecidable,
returns efilter.protocol.AnyType.
NOTE: The inference returns the type of a row in the results, not of the
actual Python object returned by 'apply'. For example, if a query
returns multiple rows, each one of which is an integer, the type of the
output is considered to be int, not a collection of rows.
Examples:
infer("5 + 5") # -> INumber
infer("SELECT * FROM people WHERE age > 10") # -> AnyType
# If root_type implements the IStructured reflection API:
infer("SELECT * FROM people WHERE age > 10", root_type=...) # -> dict
"""
# Always make the scope stack start with stdcore.
if root_type:
type_scope = scope.ScopeStack(std_core.MODULE, root_type)
else:
type_scope = scope.ScopeStack(std_core.MODULE)
stdcore_included = False
for lib in libs:
if lib == "stdcore":
stdcore_included = True
continue
module = std_core.LibraryModule.ALL_MODULES.get(lib)
if not module:
raise TypeError("No standard library module %r." % lib)
type_scope = scope.ScopeStack(module, type_scope)
if not stdcore_included:
raise TypeError("'stdcore' must always be included.")
query = q.Query(query, params=replacements)
return infer_type.infer_type(query, type_scope)
def validate(expr, scope):
# Make sure there's an ELSE block.
if expr.default() is None:
raise errors.EfilterLogicError(
root=expr,
message="Else blocks are required in EFILTER.")
# Make sure conditions evaluate to IBoolean.
for condition, _ in expr.conditions():
t = infer_type.infer_type(condition, scope)
if not protocol.isa(t, boolean.IBoolean):
raise errors.EfilterTypeError(root=expr, actual=t,
expected=boolean.IBoolean)
def render(self, renderer):
# Do we have a query?
if not self.query:
return self.render_error(renderer)
# Figure out what the header should look like.
# Can we infer the type?
try:
t = infer_type.infer_type(self.query, self)
except Exception:
t = None
# Get the data we're rendering.
try:
rows = self.collect() or []
except errors.EfilterError as error:
self.query_error = error
return self.render_error(renderer)
# If we know the header, great!
if isinstance(t, plugin.TypedProfileCommand):
renderer.table_header(t.table_header)
return self._render_plugin_output(renderer, t.table_header,
*rows)
# Maybe we cached the header when we ran the plugin?
if isinstance(t, plugin.Command):
wrapper = self._cached_plugin_wrappers.get(t.name)
if wrapper:
renderer.table_header(wrapper.columns)
return self._render_plugin_output(renderer,
wrapper.columns,
*rows)
# Try to guess the header based on structure of the first row.
if not rows:
renderer.table_header([("No Results", "no_results", "20")])
return
rows = iter(rows)
first_row = next(rows)
if isinstance(first_row, dict):
renderer.table_header(
[dict(name=unicode(k), cname=unicode(k))
for k in first_row.iterkeys()])
return self._render_dicts(renderer, first_row, *rows)
# Sigh. Give up, and render whatever you got, I guess.
renderer.table_header([dict(name="Result", cname="result")])
return self._render_whatever_i_guess(renderer, first_row, *rows)
def render(self, renderer):
# Figure out what the header should look like.
# Can we infer the type?
try:
t = infer_type.infer_type(self.query, self)
except Exception:
t = None
rows = self.collect() or []
# If we know the header, great!
if isinstance(t, plugin.TypedProfileCommand):
renderer.table_header(t.table_header)
return self._render_plugin_output(renderer, t.table_header,
*rows)
# Maybe we cached the header when we ran the plugin?
if isinstance(t, plugin.Command):
header = self._cached_plugin_renderers.get(t.name)
if header:
renderer.table_header(header.columns)
return self._render_plugin_output(renderer,
header.columns, *rows)
# Try to guess the header based on structure of the first row.
if not rows:
renderer.table_header([("No Results", "no_results", "20")])
return
rows = iter(rows)
first_row = next(rows)
if isinstance(first_row, dict):
renderer.table_header(
[dict(name=k, cname=k) for k in first_row.iterkeys()])
return self._render_dicts(renderer, first_row, *rows)
# Sigh. Give up, and render whatever you got, I guess.
renderer.table_header([dict(name="Result", cname="result")])
return self._render_whatever_i_guess(renderer, first_row, *rows)
def testRepeat(self):
self.assertIsa(infer_type.infer_type(q.Query(("repeat", 1, 2, 3)), mocks.MockRootType), number.INumber)
def testApply(self):
self.assertIsa(infer_type.infer_type(q.Query("MockFunction(5, 10)"), mocks.MockRootType), number.INumber)
def testEach(self):
self.assertIsa(
infer_type.infer_type(q.Query("any Process.children where (name == 'init')"), mocks.MockRootType),
boolean.IBoolean,
)
def testAny(self):
self.assertIsa(
infer_type.infer_type(q.Query("any pslist where (parent.name == 'init')"), mocks.MockRootType),
boolean.IBoolean,
)
def testApply(self):
self.assertIsa(
infer_type.infer_type(q.Query("MockFunction(5, 10)"),
mocks.MockRootType), number.INumber)
def render(self, renderer):
# Do we have a query?
if not self.query:
return self.render_error(renderer)
# Figure out what the header should look like.
# Can we infer the type?
try:
t = infer_type.infer_type(self.query, self)
except Exception:
t = None
# Get the data we're rendering.
try:
rows = self.collect() or []
except errors.EfilterError as error:
self.query_error = error
return self.render_error(renderer)
# If the output type is a TypeProfileCommand subclass then we can just
# interrogate its header.
if getattr(t, "table_header", None):
renderer.table_header(t.table_header)
return self._render_plugin_output(renderer, t.table_header,
*rows)
# If the output type is a regular plugin then we must've run it at some
# point and should be able to retrieve a cached copy of the wrapper,
# which will have preserved the output columns.
if isinstance(t, plugin.Command):
cached_wrapper = self._cached_command_wrappers.get(t.name)
if not cached_wrapper:
raise RuntimeError("Command of type %r is the output of an "
"EFILTER query but no such command was "
"executed." % (t,))
if not cached_wrapper._applied_args:
cached_wrapper.apply((), {})
renderer.table_header(cached_wrapper.columns)
return self._render_plugin_output(renderer, cached_wrapper.columns,
*rows)
# If we got no rows in the output the just say so.
rows = iter(rows)
try:
first_row = next(rows)
except StopIteration:
renderer.table_header([("No Results", "no_results", "20")])
return
# As last ditch, try to guess the header based on the data in the
# first row.
if isinstance(first_row, dict):
# Maybe we have a plugin with matching columns in its output?
columns = self._find_matching_header(first_row.keys())
if columns:
renderer.table_header(columns)
return self._render_plugin_output(renderer,
columns,
first_row,
*rows)
renderer.table_header(
[dict(name=unicode(k), cname=unicode(k))
for k in first_row.iterkeys()])
return self._render_dicts(renderer, first_row, *rows)
# Sigh. Give up, and render whatever you got, I guess.
renderer.table_header([dict(name="Result", cname="result")])
return self._render_whatever_i_guess(renderer, first_row, *rows)
def testEquivalence(self):
self.assertIsa(infer_type.infer_type(q.Query("Process.name == 'init'"), mocks.MockRootType), boolean.IBoolean)
def testVar(self):
self.assertIsa(infer_type.infer_type(q.Query("foo"), mocks.MockRootType), protocol.AnyType)
def testLiteral(self):
self.assertIsa(infer_type.infer_type(q.Query("42"), mocks.MockRootType), number.INumber)
def testEach(self):
self.assertIsa(
infer_type.infer_type(
q.Query("any Process.children where (name == 'init')"),
mocks.MockRootType), boolean.IBoolean)
def testFilter(self):
self.assertIsa(
infer_type.infer_type(
q.Query("select * from pslist where (parent.pid == 10)"),
mocks.MockRootType), mocks.Process)
def testFilter(self):
self.assertIsa(
infer_type.infer_type(q.Query("select * from pslist where (parent.pid == 10)"), mocks.MockRootType),
mocks.Process,
)
def render(self, renderer):
# Do we have a query?
if not self.query:
return self.render_error(renderer)
# Figure out what the header should look like.
# Can we infer the type?
# For example for select statements the type will be
# associative.IAssociative because they return a dict like result.
try:
t = infer_type.infer_type(self.query, self)
except Exception:
t = None
if isinstance(t, CommandWrapper):
raise RuntimeError(
"%r is a plugin and must be called as a function. Try '%s()'"
" instead of '%s'"
% (t.plugin_cls, t.plugin_cls.name, t.plugin_cls.name))
# Get the data we're rendering.
try:
rows = self.collect() or []
except errors.EfilterError as error:
self.query_error = error
return self.render_error(renderer)
# If the query returns the output of a plugin then we have to render
# the same columns as the plugin. If the plugin declares its columns
# then that's easy. Otherwise we have to try and get the columns from
# cache.
# e.g. select * from pslist()
if isinstance(t, plugin.Command):
output_header = getattr(t, "table_header", None)
if output_header is None:
raise plugin.PluginError(
"Query is using plugin %s which is not typed." % t.name)
renderer.table_header(output_header)
return self._render_plugin_output(renderer, output_header, rows)
# For queries which name a list of columns we need to get the first row
# to know which columns will be output. Surely efilter can provide this
# from the AST? This seems like a hack because if the first row the
# plugin produces does not include all the columns we will miss them.
# If is also buggy because if the plugin does not produce any rows we
# can not know if the query is correct or not. For example "select XXXX
# from plugin()" can not raise an unknown column XXXX if the plugin does
# not produce at least one row.
remaining_rows = iter(rows)
try:
first_row = next(remaining_rows)
except StopIteration:
renderer.format("No results.")
return
all_rows = itertools.chain((first_row,), remaining_rows)
# If we have some output but don't know what it is we can try to use
# dict keys as columns.
if isinstance(first_row, row_tuple.RowTuple):
columns = [dict(name=x)
for x in structured.getmembers(first_row)]
renderer.table_header(columns, auto_widths=True)
return self._render_plugin_output(renderer, columns, all_rows)
# Sigh. Give up, and render whatever you got, I guess.
renderer.table_header([dict(name="Result", cname="result")])
return self._render_whatever_i_guess(renderer, all_rows)
def testComplement(self):
self.assertIsa(infer_type.infer_type(q.Query("not Process.name"), mocks.MockRootType), boolean.IBoolean)
def testRepeat(self):
self.assertIsa(
infer_type.infer_type(q.Query(("repeat", 1, 2, 3)),
mocks.MockRootType), number.INumber)
def testIsInstance(self):
self.assertIsa(infer_type.infer_type(q.Query("proc isa Process"), mocks.MockRootType), boolean.IBoolean)
def testVar(self):
self.assertIsa(infer_type.infer_type(q.Query("Process.pid"), mocks.MockRootType), int)
def testBinaryExpression(self):
self.assertIsa(infer_type.infer_type(q.Query("'foo' in ('bar', 'foo')"), mocks.MockRootType), boolean.IBoolean)
