def main():
long_description = """
Provide documentation for modules, actors, and components.
"""
default_format = 'compact' if sys.argv[0].endswith('csdocs') else 'detailed'
argparser = argparse.ArgumentParser(description=long_description)
group = argparser.add_mutually_exclusive_group()
group.add_argument('what', metavar='<actor or module>', type=str, nargs='?', default='',
help='What to look up documentation for, if empty show top level documentation')
group.add_argument('--all', action='store_const', const=True, default=False,
help='Generate complete actor documentation in Markdown format')
argparser.add_argument('--format', default=default_format, choices=['detailed', 'compact', 'raw'],
help='Options "detailed" and "compact" returns Markdown-formatted text,'
' while "raw" returns a JSON-formatted representation that can be'
' used to generated the documentation in other formats.')
args = argparser.parse_args()
store = DocumentationStore()
if args.all:
all_docs()
else:
if args.format == 'raw':
print json.dumps(store.help_raw(args.what))
else:
print store.help(args.what, args.format == 'compact')
def handle_get_actor_doc(self, handle, connection, match, data, hdr):
""" Query ActorStore for documentation
"""
path = match.group(1)
what = '.'.join(path.strip('/').split('/'))
ds = DocumentationStore()
data = ds.help_raw(what)
self.send_response(handle, connection, json.dumps(data))
def __init__(self, cs_info):
super(Checker, self).__init__()
self.ds = DocumentationStore()
self.cs_info = cs_info
self.local_actors = {}
self.errors = []
self.warnings = []
self.check()
def __init__(self, cs_info):
super(Checker, self).__init__()
self.ds = DocumentationStore()
self.cs_info = cs_info
self.constants = self.cs_info['constants']
self.comp_defs = self.cs_info['components']
self.errors = []
self.warnings = []
self.check()
def handle_get_actor_doc(self, handle, connection, match, data, hdr):
"""
GET /actor_doc {path}
Get documentation in 'raw' format for actor or module at {path}
Path is formatted as '/{module}/{submodule}/ ... /{actor}'.
If {path} is empty return top-level documentation.
See DocumentStore help_raw() for details on data format.
Response status code: OK
Response: dictionary with documentation
"""
path = match.group(1)
what = '.'.join(path.strip('/').split('/'))
ds = DocumentationStore()
what = None if not what else what
data = ds.help_raw(what)
self.send_response(handle, connection, data)
def _lookup(node, issue_tracker):
if _is_local_component(node.actor_type):
comps = query(_root(node), kind=ast.Component, attributes={'name':node.actor_type})
if not comps:
reason = "Missing local component definition: '{}'".format(node.actor_type)
issue_tracker.add_error(reason, node)
return {'is_known': False}
comp = comps[0]
metadata = {
'is_known': True,
'name': comp.name,
'type': 'component',
'inputs': comp.inports,
'outputs': comp.outports,
'args':{
'mandatory':comp.arg_names,
'optional':{}
},
'definition': comp.children[0]
}
else:
metadata = DocumentationStore().metadata(node.actor_type)
if not metadata['is_known']:
reason = "Not validating actor type: '{}'".format(node.actor_type)
issue_tracker.add_warning(reason, node)
return metadata
def handle_get_actor_doc(self, handle, connection, match, data, hdr):
"""
GET /actor_doc/{path}
Get documentation in 'raw' format for actor or module at {path}
Path is formatted as '/{module}/{submodule}/ ... /{actor}'.
If {path} is empty return top-level documentation.
See DocumentStore help_raw() for details on data format.
Response status code: OK
Response: dictionary with documentation
"""
path = match.group(1)
what = '.'.join(path.strip('/').split('/'))
ds = DocumentationStore()
what = None if not what else what
data = ds.help_raw(what)
self.send_response(handle, connection, data)
class ActorViz(Viz):
"""docstring for ActorViz"""
docstore = DocumentationStore()
def __init__(self, name, actor_type, args, **dummy):
super(ActorViz, self).__init__()
self.type_color = 'lightblue'
self.name = name
self.args = args
self.actor_type = actor_type
doc = self.docstore.help_raw(actor_type)
self.set_ports(doc)
def set_ports(self, doc):
inports = [p for p, _ in doc['inputs']]
outports = [p for p, _ in doc['outputs']]
inlen = len(inports)
outlen = len(outports)
self.portrows = max(inlen, outlen)
self.inports = inports + [''] * (self.portrows - inlen)
self.outports = outports + [''] * (self.portrows - outlen)
def __str__(self):
lines = []
lines.append('{0} [label=<'.format(_refname(self.name)))
lines.append(
'<TABLE BORDER="1" CELLBORDER="0" CELLSPACING="0" CELLPADDING="1">'
)
# Name
lines.append('<TR><TD bgcolor="{1}" COLSPAN="3">{0}</TD></TR>'.format(
self.name, self.type_color))
# Class
lines.append('<TR><TD COLSPAN="3">{0}</TD></TR>'.format(
self.actor_type))
is_first = True
for inport, outport in zip(self.inports, self.outports):
inref = ' bgcolor="lightgrey" PORT="{0}_in"'.format(
inport) if inport else ''
outref = ' bgcolor="lightgrey" PORT="{0}_out"'.format(
outport) if outport else ''
if is_first:
is_first = False
middle = '<TD ROWSPAN="{0}"> </TD>'.format(self.portrows)
else:
middle = ''
lines.append(
'<TR><TD{0} align="left">{1}</TD>{4}<TD{2} align="right">{3}</TD></TR>'
.format(inref, inport, outref, outport, middle))
lines.append('</TABLE>>];')
return '\n'.join(lines)
def all_docs(what=None):
ds = DocumentationStore()
raw = ds.help_raw(what)
print ds.help(what)
for actor in raw.get('actors', []):
print ds.help(what + '.' + actor)
for module in raw.get('modules', []):
all_docs(module)
def _init_metadata(self):
def gather_actors(module):
# Depth first
l = []
for m in d.modules(module):
l = l + gather_actors(m)
actors = d.actors(module)
if module:
# Add namespace
actors = ['.'.join([module, a]) for a in actors]
return l + actors
d = DocumentationStore()
metadata = {}
actors = gather_actors('')
for actor in actors:
parts = actor.split('.')
x = metadata
for p in parts[:-1]:
x = x.setdefault(p, {})
x[parts[-1]] = d.metadata(actor)
self.metadata = metadata
def _init_metadata(self):
def gather_actors(module):
# Depth first
l = []
for m in d.modules(module):
l = l + gather_actors(m)
actors = d.actors(module)
if module:
# Add namespace
actors = ['.'.join([module, a]) for a in actors]
return l + actors
d = DocumentationStore()
metadata = {}
actors = gather_actors('')
for actor in actors:
parts = actor.split('.')
x = metadata
for p in parts[:-1]:
x = x.setdefault(p, {})
x[parts[-1]] = d.metadata(actor)
self.metadata = metadata
import visitor
import astnode as ast
from codegen import calvin_astgen, query
from parser import calvin_parse
from calvin.actorstore.store import DocumentationStore
_docstore = DocumentationStore()
def _refname(name):
return name.replace(':', '_')
def _lookup_definition(actor_type, root):
if '.' in actor_type:
doc = _docstore.help_raw(actor_type)
if not doc['name']:
return ([], [], '')
t = doc['type']
inports = [p for p,_ in doc['inputs']]
outports = [p for p,_ in doc['outputs']]
else:
t = 'component'
comps = query(root, kind=ast.Component, attributes={'name':actor_type}, maxdepth=2)
if not comps:
return ([], [], '')
inports, outports = comps[0].inports, comps[0].outports
return (inports, outports, t)
#
class Checker(object):
# FIXME: Provide additional checks making use of
# - actor_def.inport_names and actor_def.outport_names
# - make use of arg_type (STRING, NUMBER, etc.)
# - analyze the actions wrt port use and token consumption:
# for f in actor_def.action_priority:
# print f.__name__, [x.cell_contents for x in f.__closure__]
#
def __init__(self, cs_info):
super(Checker, self).__init__()
self.ds = DocumentationStore()
self.cs_info = cs_info
self.local_actors = {}
self.errors = []
self.warnings = []
self.check()
def issue(self, fmt, **info):
return {'reason':fmt.format(**info), 'line':info.get('line', 0), 'col':info.get('col', 0)}
def append_error(self, fmt, **info):
issue = self.issue(fmt, **info)
self.errors.append(issue)
def append_warning(self, fmt, **info):
issue = self.issue(fmt, **info)
self.warnings.append(issue)
def check(self):
components = self.cs_info['components']
for comp in components:
self.local_actors[comp['name']] = comp
self.check_component(comp)
self.check_structure(self.cs_info['structure'])
def check_component(self, comp_def):
# print comp_name, json.dumps(comp_def, indent=4), json.dumps(self.get_definition(comp_name), indent=4)
# fake definition for component test inputs/outputs have reversed meaning
# defs = {'inputs':[(p, '') for p in comp_def['outports']], 'outputs':[(p, '') for p in comp_def['inports']]}
defs = self.get_definition(comp_def['name'])
self.check_connections(None, defs, comp_def['structure']['connections'])
self.check_structure(comp_def['structure'])
# Check for unused arguments and issue warning, not error
args = set(comp_def['arg_identifiers'])
used_args = set([])
for a in comp_def['structure']['actors'].values():
used_args.update(a['args'].keys())
unused_args = args - used_args
for u in unused_args:
fmt = "Unused argument: '{param}'"
self.append_warning(fmt, line=comp_def['dbg_line'], param=u)
def get_definition(self, actor_type):
if actor_type in self.local_actors:
return self.ds.component_docs("local."+actor_type, self.local_actors[actor_type])
return self.ds.actor_docs(actor_type)
def dbg_lines(self, s):
try:
return [x['dbg_line'] for x in s] + [0]
except:
return [0]
def check_component_connections(self, definition, connections):
inport_names = [p for p, _ in definition['inputs']]
outport_names = [p for p, _ in definition['outputs']]
# Verify names of ports
invalid_inports = [(c['src_port'], 'in', c['dbg_line']) for c in connections if not c['src'] and c['src_port'] not in inport_names]
invalid_outports = [(c['dst_port'], 'out', c['dbg_line']) for c in connections if not c['dst'] and c['dst_port'] not in outport_names]
invalid_ports = invalid_inports + invalid_outports
for port, port_dir, line in invalid_ports:
fmt = "Component {name} has no {port_dir}port '{port}'"
self.append_error(fmt, line=line, port=port, port_dir=port_dir, **definition)
# outports should have exactly one connection
for port in outport_names:
incoming = [c for c in connections if not c['dst'] and c['dst_port'] == port]
if len(incoming) == 0:
fmt = "Component {name} is missing connection to outport '{port}'"
self.append_error(fmt, line=max(self.dbg_lines(connections)), port=port, **definition)
if len(incoming) > 1:
fmt = "Component {name} has multiple connections to outport '{port}'"
for c in incoming:
self.append_error(fmt, line=c['dbg_line'], port=port, **definition)
# inports should have at least one connection
for port in inport_names:
outgoing = [c for c in connections if not c['src'] and c['src_port'] == port]
if len(outgoing) < 1:
fmt = "Component {name} is missing connection to inport '{port}'"
self.append_error(fmt, line=max(self.dbg_lines(connections)), port=port, **definition)
def check_actor_connections(self, actor, definition, connections):
inport_names = [p for p, _ in definition['inputs']]
outport_names = [p for p, _ in definition['outputs']]
# Verify names of ports
invalid_inports = [(c['dst_port'], 'in', c['dbg_line']) for c in connections if c['dst'] == actor and c['dst_port'] not in inport_names]
invalid_outports = [(c['src_port'], 'out', c['dbg_line']) for c in connections if c['src'] == actor and c['src_port'] not in outport_names]
invalid_ports = invalid_inports + invalid_outports
for port, port_dir, line in invalid_ports:
fmt = "Actor {actor} ({ns}.{name}) has no {port_dir}port '{port}'"
self.append_error(fmt, line=line, port=port, port_dir=port_dir, actor=actor, **definition)
# inports should have exactly one connection
for port in inport_names:
incoming = [c for c in connections if c['dst'] == actor and c['dst_port'] == port]
if len(incoming) == 0:
fmt = "Actor {actor} ({ns}.{name}) is missing connection to inport '{port}'"
self.append_error(fmt, line=max(self.dbg_lines(connections)), port=port, actor=actor, **definition)
if len(incoming) > 1:
fmt = "Actor {actor} ({ns}.{name}) has multiple connections to inport '{port}'"
for c in incoming:
self.append_error(fmt, line=c['dbg_line'], port=port, actor=actor, **definition)
# outports should have at least one connection
for port in outport_names:
outgoing = [c for c in connections if c['src'] == actor and c['src_port'] == port]
if not outgoing:
fmt = "Actor {actor} ({ns}.{name}) is missing connection to outport '{port}'"
self.append_error(fmt, line=max(self.dbg_lines(connections)), port=port, actor=actor, **definition)
def check_connections(self, actor, definition, connections):
if actor:
self.check_actor_connections(actor, definition, connections)
else:
self.check_component_connections(definition, connections)
def check_arguments(self, definition, declaration):
mandatory = set(definition['args']['mandatory'])
defined = set(declaration['args'].keys())
undefined = mandatory - defined
for u in undefined:
fmt = "Missing argument: '{param}'"
self.append_error(fmt, line=declaration['dbg_line'], param=u)
print self.errors[-1], definition
def check_structure(self, structure):
actors = structure['actors'].keys()
# Look for undefined actors
src_actors = {c['src'] for c in structure['connections'] if c['src']}
dst_actors = {c['dst'] for c in structure['connections'] if c['dst']}
all_actors = src_actors | dst_actors
undefined_actors = all_actors - set(actors)
for actor in undefined_actors:
fmt = "Undefined actor: '{actor}'"
lines = [c['dbg_line'] for c in structure['connections'] if c['src'] == actor or c['dst'] == actor]
for line in lines:
self.append_error(fmt, line=line, actor=actor)
# Note: Unused actors will be caught when checking connections
# Check if actor exists
for actor in actors:
actor_type = structure['actors'][actor]['actor_type']
definition = self.get_definition(actor_type)
if not definition:
fmt = "Unknown actor type: '{type}'"
self.append_error(fmt, type=actor_type, line=structure['actors'][actor]['dbg_line'])
continue
# We have actor definition, check that it is fully connected
self.check_connections(actor, definition, structure['connections'])
self.check_arguments(definition, structure['actors'][actor])
def setUp(self):
self.ds = DocumentationStore()
class Checker(object):
# FIXME: Provide additional checks making use of
# - actor_def.inport_names and actor_def.outport_names
# - make use of arg_type (STRING, NUMBER, etc.)
# - analyze the actions wrt port use and token consumption:
# for f in actor_def.action_priority:
# print f.__name__, [x.cell_contents for x in f.__closure__]
#
def __init__(self, cs_info, verify=True):
super(Checker, self).__init__()
self.ds = DocumentationStore()
self.cs_info = cs_info
self.constants = self.cs_info['constants']
self.comp_defs = self.cs_info['components']
self.errors = []
self.warnings = []
self.verify = verify
self.check()
def issue(self, fmt, **info):
return {
'reason': fmt.format(**info),
'line': info.get('line', 0),
'col': info.get('col', 0)
}
def append_error(self, fmt, **info):
issue = self.issue(fmt, **info)
self.errors.append(issue)
def append_warning(self, fmt, **info):
issue = self.issue(fmt, **info)
self.warnings.append(issue)
def check(self):
"""
Check constants, local components, and program, in that order.
Generate error and warning issues as they are encountered.
"""
self.check_constants()
for comp in self.comp_defs.values():
self.check_component(comp)
self.check_structure(self.cs_info['structure'])
def check_component(self, comp_def):
"""
Check connections, structure, and argument for local component
"""
defs = self.get_definition(comp_def['name'])
self.check_component_connections(defs, comp_def['structure']['connections'])
self.check_structure(comp_def['structure'], comp_def['arg_identifiers'])
implicits = self._find_implicits(comp_def['structure']['connections'])
self.check_component_arguments(comp_def, implicits)
def _find_implicits(self, connections):
implicit = [c['src_port'] for c in connections if not c['src']]
arg_names = [value for kind, value in implicit if kind == 'IDENTIFIER']
return arg_names
def get_definition(self, actor_type):
"""
Get the actor/component definition from the docstore.
For local components, let the docstore generate the definition.
"""
if actor_type in self.comp_defs:
return self.ds.component_docs("local." + actor_type, self.comp_defs[actor_type])
return self.ds.actor_docs(actor_type)
def dbg_lines(self, s):
"""Return the debug line numbers in a construct. Default to 0."""
try:
return [x['dbg_line'] for x in s] or [0]
except:
return [0]
def twiddle_portrefs(self, is_component, port_dir):
if port_dir not in ['in', 'out']:
raise Exception("Invalid port direction: {}".format(port_dir))
if is_component:
target, target_port = ('dst', 'dst_port') if port_dir == 'out' else ('src', 'src_port')
else:
target, target_port = ('dst', 'dst_port') if port_dir == 'in' else ('src', 'src_port')
return target, target_port
def generate_ports(self, definition, actor=None):
"""
This generator takes care of remapping src and dst with respect to programs and component definitions.
Given definition, connections, and optionally actor it will generate a list of tuples:
(port, target, target_port, port_dir, actor) according to the following scheme:
component inports : (port, 'src', 'src_port', 'in', '.')
component outports : (port, 'dst', 'dst_port', 'out', '.')
actor inports : (port, 'dst', 'dst_port', 'in', actor)
actor outports : (port, 'src', 'src_port', 'out', actor)
that help sorting out connections.
"""
is_component = actor is None
actor = '.' if is_component else actor
for port_dir in ['in', 'out']:
target, target_port = self.twiddle_portrefs(is_component, port_dir)
def_dir = 'inputs' if port_dir == 'in' else 'outputs'
for p, _ in definition[def_dir]:
yield((p, target, target_port, port_dir, actor))
def _verify_port_names(self, definition, connections, actor=None):
"""Look for misspelled port names."""
# A little transformation is required depending on actor vs. component and port direction
retval = []
is_component = actor is None
actor = '.' if is_component else actor
for port_dir in ['in', 'out']:
target, target_port = self.twiddle_portrefs(is_component, port_dir)
def_dir = 'inputs' if port_dir == 'in' else 'outputs'
ports = [p for p, _ in definition[def_dir]]
invalid_ports = [(c[target_port], port_dir, c['dbg_line']) for c in connections if c[target] == actor and c[target_port] not in ports]
retval.extend(invalid_ports)
return retval
def check_atleast_one_connection(self, definition, connections, actor=None):
"""Check that all ports have at least one connection"""
retval = []
for port, target, target_port, port_dir, actor in self.generate_ports(definition, actor):
pc = [c for c in connections if c[target] == actor and c[target_port] == port]
if len(pc) < 1:
retval.append((port, port_dir, max(self.dbg_lines(connections))))
return retval
def check_atmost_one_connection(self, definition, connections, actor=None):
"""Check that component destination port have at most one connection"""
retval = []
for port, target, target_port, port_dir, actor in self.generate_ports(definition, actor):
if target == 'src':
# Skip output (src) ports since they can have multiple connections
continue
pc = [c for c in connections if c[target] == actor and c[target_port] == port]
if len(pc) > 1:
retval.extend([(port, port_dir, c['dbg_line']) for c in pc])
return retval
def report_port_errors(self, fmt, portspecs, definition, actor=None):
for port, port_dir, line in portspecs:
self.append_error(fmt, line=line, port=port, port_dir=port_dir, actor=actor, **definition)
def check_component_connections(self, definition, connections):
# Check for bogus ports
invalid_ports = self._verify_port_names(definition, connections)
fmt = "Component {name} has no {port_dir}port '{port}'"
self.report_port_errors(fmt, invalid_ports, definition)
# All ports should have at least one connection...
bad_ports = self.check_atleast_one_connection(definition, connections)
fmt = "Component {name} is missing connection to {port_dir}port '{port}'"
self.report_port_errors(fmt, bad_ports, definition)
# ... but outports should have exactly one connection
bad_ports = self.check_atmost_one_connection(definition, connections)
fmt = "Component {name} has multiple connections to {port_dir}port '{port}'"
self.report_port_errors(fmt, bad_ports, definition)
# Check for illegal passthrough (.in > .out) connections
fmt = "Component {name} passes port '{src_port}' directly to port '{dst_port}'"
for pc in [c for c in connections if c['src'] == c['dst'] == '.']:
self.append_error(fmt, line=c['dbg_line'], src_port=c['src_port'], dst_port=c['dst_port'], **definition)
def check_actor_connections(self, actor, definition, connections):
invalid_ports = self._verify_port_names(definition, connections, actor)
fmt = "Actor {actor} ({ns}.{name}) has no {port_dir}port '{port}'"
self.report_port_errors(fmt, invalid_ports, definition, actor)
# All ports should have at least one connection...
bad_ports = self.check_atleast_one_connection(definition, connections, actor)
fmt = "Actor {actor} ({ns}.{name}) is missing connection to {port_dir}port '{port}'"
self.report_port_errors(fmt, bad_ports, definition, actor)
def check_constants(self):
"""Verify that all constant definitions evaluate to a value."""
for constant in self.constants:
try:
self.lookup_constant(constant)
except KeyError as e:
fmt = "Constant '{name}' is undefined"
self.append_error(fmt, name=e.args[0])
except:
fmt = "Constant '{name}' has a circular reference"
self.append_error(fmt, name=constant)
def lookup_constant(self, identifier, seen=None):
"""
Return value for constant 'identifier' by recursively looking for a value.
Raise an exception if not found
"""
seen = seen or []
kind, value = self.constants[identifier]
if kind != "IDENTIFIER":
return value
if value in seen:
raise Exception("Circular reference in constant definition")
seen.append(value)
return self.lookup_constant(value, seen)
def check_component_arguments(self, comp_def, implicits):
"""
Warn if component declares parameters that are not used by the actors in the component.
"""
declared_args = set(comp_def['arg_identifiers'])
used_args = set(implicits)
for actor_def in comp_def['structure']['actors'].values():
used_args.update({value for kind, value in actor_def['args'].values() if kind == 'IDENTIFIER'})
unused_args = declared_args - used_args
for u in unused_args:
fmt = "Unused argument: '{param}'"
self.append_error(fmt, line=comp_def['dbg_line'], param=u)
def check_arguments(self, definition, declaration, arguments):
"""
Verify that all arguments are present and valid when instantiating actors.
'arguments' is a list of the arguments whose value is supplied by a component
to it's constituent actors.
"""
mandatory = set(definition['args']['mandatory'])
optional = set(definition['args']['optional'])
defined = set(declaration['args'].keys())
# Case 1: Missing arguments
missing = mandatory - defined
for m in missing:
fmt = "Missing argument: '{param}'"
self.append_error(fmt, line=declaration['dbg_line'], param=m)
# Case 2: Extra (unused) arguments
unused = defined - (mandatory | optional)
for m in unused:
fmt = "Unused argument: '{param}'"
self.append_error(fmt, line=declaration['dbg_line'], param=m)
# Case 3: value for arg is IDENTIFIER rather than VALUE, and IDENTIFIER is not in constants
for param, (kind, value) in declaration['args'].iteritems():
# If kind is not IDENTIFIER we have an actual value => continue to next argument
if kind != 'IDENTIFIER':
continue
# First check if the identifier (value) is provided by a wrapping component...
if value in arguments:
continue
# ... and if it is not it have to be a constant or there was an error in the script.
if value not in self.constants:
fmt = "Undefined identifier: '{param}'"
self.append_error(fmt, line=declaration['dbg_line'], param=value)
def undeclared_actors(self, connections, declared_actors):
"""
Scan connections for actors and compare to the set of declared actors in order
to find actors that are referenced in the script but not declared.
"""
# Look for undeclared actors
src_actors = {c['src'] for c in connections if c['src'] != "."}
dst_actors = {c['dst'] for c in connections if c['dst'] != "."}
# Implicit src actors are defined by a constant on the inport
implicit_src_actors = {c['src'] for c in connections if c['src'] is None}
all_actors = src_actors | dst_actors
undefined_actors = all_actors - (set(declared_actors) | implicit_src_actors)
return undefined_actors
def unknown_actors(self, actor_decls):
"""
Find unknown actors, i.e. actors that are declared in the script,
but whose definition is missing from the actorstore.
"""
unknown_actors = [a for a, decl in actor_decls.iteritems() if not self.get_definition(decl['actor_type'])]
return unknown_actors
def check_structure(self, structure, arguments=None):
"""
Check structure of program or component definition.
'arguments' is a list of the parameters provided by the component to its constituent actors,
and is only present if this method is called from 'check_component'.
"""
connections = structure['connections']
actor_declarations = structure['actors']
declared_actors = actor_declarations.keys()
arguments = arguments or []
# Look for missing actors
for actor in self.undeclared_actors(connections, declared_actors):
fmt = "Undefined actor: '{actor}'"
lines = [c['dbg_line'] for c in connections if c['src'] == actor or c['dst'] == actor]
for line in lines:
self.append_error(fmt, line=line, actor=actor)
# FIXME: Add check for actors declared more than once
# Note: Unused actors will be caught when checking connections
# Check if actor definition exists
unknown_actors = self.unknown_actors(actor_declarations)
if self.verify:
for actor in unknown_actors:
fmt = "Unknown actor type: '{type}'"
self.append_error(fmt, type=actor_declarations[actor]['actor_type'], line=actor_declarations[actor]['dbg_line'])
# Check the validity of the known actors
known_actors = set(declared_actors) - set(unknown_actors)
for actor in known_actors:
definition = self.get_definition(actor_declarations[actor]['actor_type'])
self.check_actor_connections(actor, definition, connections)
self.check_arguments(definition, actor_declarations[actor], arguments)
def document(what):
store = DocumentationStore()
print store.help(what or None, compact=False, formatting='md', links=False)
class Checker(object):
# FIXME: Provide additional checks making use of
# - actor_def.inport_names and actor_def.outport_names
# - make use of arg_type (STRING, NUMBER, etc.)
# - analyze the actions wrt port use and token consumption:
# for f in actor_def.action_priority:
# print f.__name__, [x.cell_contents for x in f.__closure__]
#
def __init__(self, cs_info, verify=True):
super(Checker, self).__init__()
self.ds = DocumentationStore()
self.cs_info = cs_info
self.constants = self.cs_info['constants']
self.comp_defs = self.cs_info['components']
self.errors = []
self.warnings = []
self.verify = verify
self.check()
def issue(self, fmt, **info):
return {
'reason': fmt.format(**info),
'line': info.get('line', 0),
'col': info.get('col', 0)
}
def append_error(self, fmt, **info):
issue = self.issue(fmt, **info)
self.errors.append(issue)
def append_warning(self, fmt, **info):
issue = self.issue(fmt, **info)
self.warnings.append(issue)
def check(self):
"""
Check constants, local components, and program, in that order.
Generate error and warning issues as they are encountered.
"""
self.check_constants()
for comp in self.comp_defs.values():
self.check_component(comp)
self.check_structure(self.cs_info['structure'])
def check_component(self, comp_def):
"""
Check connections, structure, and argument for local component
"""
defs = self.get_definition(comp_def['name'])
self.check_component_connections(defs,
comp_def['structure']['connections'])
self.check_structure(comp_def['structure'],
comp_def['arg_identifiers'])
implicits = self._find_implicits(comp_def['structure']['connections'])
self.check_component_arguments(comp_def, implicits)
def _find_implicits(self, connections):
implicit = [c['src_port'] for c in connections if not c['src']]
arg_names = [value for kind, value in implicit if kind == 'IDENTIFIER']
return arg_names
def get_definition(self, actor_type):
"""
Get the actor/component definition from the docstore.
For local components, let the docstore generate the definition.
"""
if actor_type in self.comp_defs:
return self.ds.component_docs("local." + actor_type,
self.comp_defs[actor_type])
return self.ds.actor_docs(actor_type)
def dbg_lines(self, s):
"""Return the debug line numbers in a construct. Default to 0."""
try:
return [x['dbg_line'] for x in s] or [0]
except:
return [0]
def twiddle_portrefs(self, is_component, port_dir):
if port_dir not in ['in', 'out']:
raise Exception("Invalid port direction: {}".format(port_dir))
if is_component:
target, target_port = (
'dst', 'dst_port') if port_dir == 'out' else ('src',
'src_port')
else:
target, target_port = (
'dst', 'dst_port') if port_dir == 'in' else ('src', 'src_port')
return target, target_port
def generate_ports(self, definition, actor=None):
"""
This generator takes care of remapping src and dst with respect to programs and component definitions.
Given definition, connections, and optionally actor it will generate a list of tuples:
(port, target, target_port, port_dir, actor) according to the following scheme:
component inports : (port, 'src', 'src_port', 'in', '.')
component outports : (port, 'dst', 'dst_port', 'out', '.')
actor inports : (port, 'dst', 'dst_port', 'in', actor)
actor outports : (port, 'src', 'src_port', 'out', actor)
that help sorting out connections.
"""
is_component = actor is None
actor = '.' if is_component else actor
for port_dir in ['in', 'out']:
target, target_port = self.twiddle_portrefs(is_component, port_dir)
def_dir = 'inputs' if port_dir == 'in' else 'outputs'
for p, _ in definition[def_dir]:
yield ((p, target, target_port, port_dir, actor))
def _verify_port_names(self, definition, connections, actor=None):
"""Look for misspelled port names."""
# A little transformation is required depending on actor vs. component and port direction
retval = []
is_component = actor is None
actor = '.' if is_component else actor
for port_dir in ['in', 'out']:
target, target_port = self.twiddle_portrefs(is_component, port_dir)
def_dir = 'inputs' if port_dir == 'in' else 'outputs'
ports = [p for p, _ in definition[def_dir]]
invalid_ports = [
(c[target_port], port_dir, c['dbg_line']) for c in connections
if c[target] == actor and c[target_port] not in ports
]
retval.extend(invalid_ports)
return retval
def check_atleast_one_connection(self,
definition,
connections,
actor=None):
"""Check that all ports have at least one connection"""
retval = []
for port, target, target_port, port_dir, actor in self.generate_ports(
definition, actor):
pc = [
c for c in connections
if c[target] == actor and c[target_port] == port
]
if len(pc) < 1:
retval.append(
(port, port_dir, max(self.dbg_lines(connections))))
return retval
def check_atmost_one_connection(self, definition, connections, actor=None):
"""Check that input ports have at most one connection"""
retval = []
for port, target, target_port, port_dir, actor in self.generate_ports(
definition, actor):
if target == 'src':
# Skip output (src) ports since they can have multiple connections
continue
pc = [
c for c in connections
if c[target] == actor and c[target_port] == port
]
if len(pc) > 1:
retval.extend([(port, port_dir, c['dbg_line']) for c in pc])
return retval
def report_port_errors(self, fmt, portspecs, definition, actor=None):
for port, port_dir, line in portspecs:
self.append_error(fmt,
line=line,
port=port,
port_dir=port_dir,
actor=actor,
**definition)
def check_component_connections(self, definition, connections):
# Check for bogus ports
invalid_ports = self._verify_port_names(definition, connections)
fmt = "Component {name} has no {port_dir}port '{port}'"
self.report_port_errors(fmt, invalid_ports, definition)
# All ports should have at least one connection...
bad_ports = self.check_atleast_one_connection(definition, connections)
fmt = "Component {name} is missing connection to {port_dir}port '{port}'"
self.report_port_errors(fmt, bad_ports, definition)
# ... but outports should have exactly one connection
bad_ports = self.check_atmost_one_connection(definition, connections)
fmt = "Component {name} has multiple connections to {port_dir}port '{port}'"
self.report_port_errors(fmt, bad_ports, definition)
# Check for illegal passthrough (.in > .out) connections
fmt = "Component {name} passes port '{src_port}' directly to port '{dst_port}'"
for pc in [c for c in connections if c['src'] == c['dst'] == '.']:
self.append_error(fmt,
line=c['dbg_line'],
src_port=c['src_port'],
dst_port=c['dst_port'],
**definition)
def check_actor_connections(self, actor, definition, connections):
invalid_ports = self._verify_port_names(definition, connections, actor)
fmt = "Actor {actor} ({ns}.{name}) has no {port_dir}port '{port}'"
self.report_port_errors(fmt, invalid_ports, definition, actor)
# All ports should have at least one connection...
bad_ports = self.check_atleast_one_connection(definition, connections,
actor)
fmt = "Actor {actor} ({ns}.{name}) is missing connection to {port_dir}port '{port}'"
self.report_port_errors(fmt, bad_ports, definition, actor)
# ... but inports should have exactly one connection
bad_ports = self.check_atmost_one_connection(definition, connections,
actor)
fmt = "Actor {actor} ({ns}.{name}) has multiple connections to {port_dir}port '{port}'"
self.report_port_errors(fmt, bad_ports, definition, actor)
def check_constants(self):
"""Verify that all constant definitions evaluate to a value."""
for constant in self.constants:
try:
self.lookup_constant(constant)
except KeyError as e:
fmt = "Constant '{name}' is undefined"
self.append_error(fmt, name=e.args[0])
except:
fmt = "Constant '{name}' has a circular reference"
self.append_error(fmt, name=constant)
def lookup_constant(self, identifier, seen=None):
"""
Return value for constant 'identifier' by recursively looking for a value.
Raise an exception if not found
"""
seen = seen or []
kind, value = self.constants[identifier]
if kind != "IDENTIFIER":
return value
if value in seen:
raise Exception("Circular reference in constant definition")
seen.append(value)
return self.lookup_constant(value, seen)
def check_component_arguments(self, comp_def, implicits):
"""
Warn if component declares parameters that are not used by the actors in the component.
"""
declared_args = set(comp_def['arg_identifiers'])
used_args = set(implicits)
for actor_def in comp_def['structure']['actors'].values():
used_args.update({
value
for kind, value in actor_def['args'].values()
if kind == 'IDENTIFIER'
})
unused_args = declared_args - used_args
for u in unused_args:
fmt = "Unused argument: '{param}'"
self.append_error(fmt, line=comp_def['dbg_line'], param=u)
def check_arguments(self, definition, declaration, arguments):
"""
Verify that all arguments are present and valid when instantiating actors.
'arguments' is a list of the arguments whose value is supplied by a component
to it's constituent actors.
"""
mandatory = set(definition['args']['mandatory'])
optional = set(definition['args']['optional'])
defined = set(declaration['args'].keys())
# Case 1: Missing arguments
missing = mandatory - defined
for m in missing:
fmt = "Missing argument: '{param}'"
self.append_error(fmt, line=declaration['dbg_line'], param=m)
# Case 2: Extra (unused) arguments
unused = defined - (mandatory | optional)
for m in unused:
fmt = "Unused argument: '{param}'"
self.append_error(fmt, line=declaration['dbg_line'], param=m)
# Case 3: value for arg is IDENTIFIER rather than VALUE, and IDENTIFIER is not in constants
for param, (kind, value) in declaration['args'].iteritems():
# If kind is not IDENTIFIER we have an actual value => continue to next argument
if kind != 'IDENTIFIER':
continue
# First check if the identifier (value) is provided by a wrapping component...
if value in arguments:
continue
# ... and if it is not it have to be a constant or there was an error in the script.
if value not in self.constants:
fmt = "Undefined identifier: '{param}'"
self.append_error(fmt,
line=declaration['dbg_line'],
param=value)
def undeclared_actors(self, connections, declared_actors):
"""
Scan connections for actors and compare to the set of declared actors in order
to find actors that are referenced in the script but not declared.
"""
# Look for undeclared actors
src_actors = {c['src'] for c in connections if c['src'] != "."}
dst_actors = {c['dst'] for c in connections if c['dst'] != "."}
# Implicit src actors are defined by a constant on the inport
implicit_src_actors = {
c['src']
for c in connections if c['src'] is None
}
all_actors = src_actors | dst_actors
undefined_actors = all_actors - (set(declared_actors)
| implicit_src_actors)
return undefined_actors
def unknown_actors(self, actor_decls):
"""
Find unknown actors, i.e. actors that are declared in the script,
but whose definition is missing from the actorstore.
"""
unknown_actors = [
a for a, decl in actor_decls.iteritems()
if not self.get_definition(decl['actor_type'])
]
return unknown_actors
def check_structure(self, structure, arguments=None):
"""
Check structure of program or component definition.
'arguments' is a list of the parameters provided by the component to its constituent actors,
and is only present if this method is called from 'check_component'.
"""
connections = structure['connections']
actor_declarations = structure['actors']
declared_actors = actor_declarations.keys()
arguments = arguments or []
# Look for missing actors
for actor in self.undeclared_actors(connections, declared_actors):
fmt = "Undefined actor: '{actor}'"
lines = [
c['dbg_line'] for c in connections
if c['src'] == actor or c['dst'] == actor
]
for line in lines:
self.append_error(fmt, line=line, actor=actor)
# FIXME: Add check for actors declared more than once
# Note: Unused actors will be caught when checking connections
# Check if actor definition exists
unknown_actors = self.unknown_actors(actor_declarations)
if self.verify:
for actor in unknown_actors:
fmt = "Unknown actor type: '{type}'"
self.append_error(fmt,
type=actor_declarations[actor]['actor_type'],
line=actor_declarations[actor]['dbg_line'])
# Check the validity of the known actors
known_actors = set(declared_actors) - set(unknown_actors)
for actor in known_actors:
definition = self.get_definition(
actor_declarations[actor]['actor_type'])
self.check_actor_connections(actor, definition, connections)
self.check_arguments(definition, actor_declarations[actor],
arguments)
