def extract_abstract_code_functions(code):
'''
Returns a set of abstract code functions from function definitions.
Returns all functions defined at the top level and ignores any other
code in the string.
Parameters
----------
code : str
The code string defining some functions.
Returns
-------
funcs : dict
A mapping ``(name, func)`` for ``func`` an `AbstractCodeFunction`.
'''
code = deindent(code)
nodes = ast.parse(code, mode='exec').body
funcs = {}
for node in nodes:
if node.__class__ is ast.FunctionDef:
func = abstract_code_from_function(node)
funcs[func.name] = func
return funcs
def __init__(self):
self.prefs = {}
self.backup_prefs = {}
self.prefs_unvalidated = {}
self.pref_register = {}
self.eval_namespace = {}
exec(
deindent('''
from numpy import *
from angela2.units import *
from angela2.units.stdunits import *
'''), self.eval_namespace)
def _as_pref_file(self, valuefunc):
'''
Helper function used to generate the preference file for the default or current preference values.
'''
s = ''
for basename, (prefdefs, basedoc) in self.pref_register.items():
s += '#' + '-' * 79 + '\n'
s += '\n'.join([
'# ' + line for line in deindent(
basedoc, docstring=True).strip().split('\n')
]) + '\n'
s += '#' + '-' * 79 + '\n\n'
s += '[' + basename + ']\n\n'
for name in sorted(prefdefs.keys()):
pref = prefdefs[name]
s += '\n'.join([
'# ' + line for line in deindent(
pref.docs, docstring=True).strip().split('\n')
]) + '\n\n'
s += name + ' = ' + pref.representor(
valuefunc(pref, basename + '.' + name)) + '\n\n'
return s
def abstract_code_from_function(func):
'''
Converts the body of the function to abstract code
Parameters
----------
func : function, str or ast.FunctionDef
The function object to convert. Note that the arguments to the
function are ignored.
Returns
-------
func : AbstractCodeFunction
The corresponding abstract code function
Raises
------
SyntaxError
If unsupported features are used such as if statements or indexing.
'''
if callable(func):
code = deindent(inspect.getsource(func))
funcnode = ast.parse(code, mode='exec').body[0]
elif isinstance(func, str):
funcnode = ast.parse(func, mode='exec').body[0]
elif func.__class__ is ast.FunctionDef:
funcnode = func
else:
raise TypeError("Unsupported function type")
if funcnode.args.vararg is not None:
raise SyntaxError("No support for variable number of arguments")
if funcnode.args.kwarg is not None:
raise SyntaxError("No support for arbitrary keyword arguments")
if len(funcnode.args.defaults):
raise SyntaxError("No support for default values in functions")
nodes = funcnode.body
nr = NodeRenderer()
lines = []
return_expr = None
for node in nodes:
if node.__class__ is ast.Return:
return_expr = nr.render_node(node.value)
break
else:
lines.append(nr.render_node(node))
abstract_code = '\n'.join(lines)
args = [arg.arg for arg in funcnode.args.args]
name = funcnode.name
return AbstractCodeFunction(name, args, abstract_code, return_expr)
def _get_one_documentation(self, basename, link_targets):
'''
Document a single category of preferences.
'''
s = ''
if not basename in self.pref_register:
raise ValueError(
'No preferences under the name "%s" are registered' % basename)
prefdefs, basedoc = self.pref_register[basename]
s += deindent(basedoc, docstring=True).strip() + '\n\n'
for name in sorted(prefdefs.keys()):
pref = prefdefs[name]
name = basename + '.' + name
linkname = name.replace('_', '-').replace('.', '-')
if link_targets:
# Make a link target
s += '.. _angela-pref-{name}:\n\n'.format(name=linkname)
s += '``{name}`` = ``{default}``\n'.format(
name=name, default=pref.representor(pref.default))
s += indent(deindent(pref.docs, docstring=True))
s += '\n\n'
return s
def _add_user_function(self, varname, var, added):
user_functions = []
load_namespace = []
support_code = []
impl = var.implementations[self.codeobj_class]
if (impl.name, var) in added:
return # nothing to do
else:
added.add((impl.name, var))
func_code = impl.get_code(self.owner)
# Implementation can be None if the function is already
# available in Cython (possibly under a different name)
if func_code is not None:
if isinstance(func_code, str):
# Function is provided as Cython code
# To make namespace variables available to functions, we
# create global variables and assign to them in the main
# code
user_functions.append((varname, var))
func_namespace = impl.get_namespace(self.owner) or {}
for ns_key, ns_value in func_namespace.items():
load_namespace.append(
'# namespace for function %s' % varname)
if hasattr(ns_value, 'dtype'):
if ns_value.shape == ():
raise NotImplementedError((
'Directly replace scalar values in the function '
'instead of providing them via the namespace'))
newlines = [
"global _namespace{var_name}",
"global _namespace_num{var_name}",
"cdef _numpy.ndarray[{cpp_dtype}, ndim=1, mode='c'] _buf_{var_name} = _namespace['{var_name}']",
"_namespace{var_name} = <{cpp_dtype} *> _buf_{var_name}.data",
"_namespace_num{var_name} = len(_namespace['{var_name}'])"
]
support_code.append(
"cdef {cpp_dtype} *_namespace{var_name}".format(
cpp_dtype=get_cpp_dtype(ns_value.dtype),
var_name=ns_key))
else: # e.g. a function
newlines = [
"_namespace{var_name} = namespace['{var_name}']"
]
for line in newlines:
load_namespace.append(
line.format(cpp_dtype=get_cpp_dtype(ns_value.dtype),
numpy_dtype=get_numpy_dtype(
ns_value.dtype),
var_name=ns_key))
# Rename references to any dependencies if necessary
for dep_name, dep in impl.dependencies.items():
dep_impl = dep.implementations[self.codeobj_class]
dep_impl_name = dep_impl.name
if dep_impl_name is None:
dep_impl_name = dep.pyfunc.__name__
if dep_name != dep_impl_name:
func_code = word_substitute(func_code,
{dep_name: dep_impl_name})
support_code.append(deindent(func_code))
elif callable(func_code):
self.variables[varname] = func_code
line = '{0} = _namespace["{1}"]'.format(varname, varname)
load_namespace.append(line)
else:
raise TypeError(('Provided function implementation '
'for function %s is neither a string '
'nor callable (is type %s instead)') % (
varname,
type(func_code)))
dep_support_code = []
dep_load_namespace = []
dep_user_functions = []
if impl.dependencies is not None:
for dep_name, dep in impl.dependencies.items():
if dep_name not in self.variables:
self.variables[dep_name] = dep
user_func = self._add_user_function(dep_name, dep, added)
if user_func is not None:
sc, ln, uf = user_func
dep_support_code.extend(sc)
dep_load_namespace.extend(ln)
dep_user_functions.extend(uf)
return (support_code + dep_support_code,
dep_load_namespace + load_namespace,
dep_user_functions + user_functions)
def create_extension(
self,
code,
force=False,
name=None,
define_macros=None,
include_dirs=None,
library_dirs=None,
runtime_library_dirs=None,
extra_compile_args=None,
extra_link_args=None,
libraries=None,
compiler=None,
sources=None,
owner_name='',
):
if sources is None:
sources = []
self._simplify_paths()
if Cython is None:
raise ImportError('Cython is not available')
code = deindent(code)
lib_dir = get_cython_cache_dir()
if '~' in lib_dir:
lib_dir = os.path.expanduser(lib_dir)
try:
os.makedirs(lib_dir)
except OSError:
if not os.path.exists(lib_dir):
raise IOError(
"Couldn't create Cython cache directory '%s', try setting the "
"cache directly with prefs.codegen.runtime.cython.cache_dir."
% lib_dir)
numpy_version = '.'.join(
numpy.__version__.split('.')[:2]) # Only use major.minor version
key = code, sys.version_info, sys.executable, Cython.__version__, numpy_version
if force:
# Force a new module name by adding the current time to the
# key which is hashed to determine the module name.
key += time.time(),
if key in self._code_cache:
return self._code_cache[key]
if name is not None:
module_name = name #py3compat.unicode_to_str(args.name)
else:
module_name = "_cython_magic_" + hashlib.md5(
str(key).encode('utf-8')).hexdigest()
if owner_name:
logger.diagnostic(
'"{owner_name}" using Cython module "{module_name}"'.format(
owner_name=owner_name, module_name=module_name))
module_path = os.path.join(lib_dir, module_name + self.so_ext)
if prefs['codegen.runtime.cython.multiprocess_safe']:
lock = FileLock(os.path.join(lib_dir, module_name + '.lock'))
with lock:
module = self._load_module(
module_path,
define_macros=define_macros,
include_dirs=include_dirs,
library_dirs=library_dirs,
extra_compile_args=extra_compile_args,
extra_link_args=extra_link_args,
libraries=libraries,
code=code,
lib_dir=lib_dir,
module_name=module_name,
runtime_library_dirs=runtime_library_dirs,
compiler=compiler,
key=key,
sources=sources)
return module
else:
return self._load_module(module_path,
define_macros=define_macros,
include_dirs=include_dirs,
library_dirs=library_dirs,
extra_compile_args=extra_compile_args,
extra_link_args=extra_link_args,
libraries=libraries,
code=code,
lib_dir=lib_dir,
module_name=module_name,
runtime_library_dirs=runtime_library_dirs,
compiler=compiler,
key=key,
sources=sources)
def abstract_code_dependencies(code, known_vars=None, known_funcs=None):
'''
Analyses identifiers used in abstract code blocks
Parameters
----------
code : str
The abstract code block.
known_vars : set
The set of known variable names.
known_funcs : set
The set of known function names.
Returns
-------
results : namedtuple with the following fields
``all``
The set of all identifiers that appear in this code block,
including functions.
``read``
The set of values that are read, excluding functions.
``write``
The set of all values that are written to.
``funcs``
The set of all function names.
``known_all``
The set of all identifiers that appear in this code block and
are known.
``known_read``
The set of known values that are read, excluding functions.
``known_write``
The set of known values that are written to.
``known_funcs``
The set of known functions that are used.
``unknown_read``
The set of all unknown variables whose values are read. Equal
to ``read-known_vars``.
``unknown_write``
The set of all unknown variables written to. Equal to
``write-known_vars``.
``unknown_funcs``
The set of all unknown function names, equal to
``funcs-known_funcs``.
``undefined_read``
The set of all unknown variables whose values are read before they
are written to. If this set is nonempty it usually indicates an
error, since a variable that is read should either have been
defined in the code block (in which case it will appear in
``newly_defined``) or already be known.
``newly_defined``
The set of all variable names which are newly defined in this
abstract code block.
'''
if known_vars is None:
known_vars = set([])
if known_funcs is None:
known_funcs = set([])
if not isinstance(known_vars, set):
known_vars = set(known_vars)
if not isinstance(known_funcs, set):
known_funcs = set(known_funcs)
code = deindent(code, docstring=True)
parsed_code = ast.parse(code, mode='exec')
# Get the list of all variables that are read from and written to,
# ignoring the order
allids, read, write, funcs = get_read_write_funcs(parsed_code)
# Now check if there are any values that are unknown and read before
# they are written to
defined = known_vars.copy()
newly_defined = set([])
undefined_read = set([])
for line in parsed_code.body:
_, cur_read, cur_write, _ = get_read_write_funcs(line)
undef = cur_read-defined
undefined_read |= undef
newly_defined |= (cur_write-defined)-undefined_read
defined |= cur_write
# Return the results as a named tuple
results = dict(
all=allids,
read=read,
write=write,
funcs=funcs,
known_all=allids.intersection(known_vars.union(known_funcs)),
known_read=read.intersection(known_vars),
known_write=write.intersection(known_vars),
known_funcs=funcs.intersection(known_funcs),
unknown_read=read-known_vars,
unknown_write=write-known_vars,
unknown_funcs=funcs-known_funcs,
undefined_read=undefined_read,
newly_defined=newly_defined,
)
return namedtuple('AbstractCodeDependencies', list(results.keys()))(**results)
def substitute_abstract_code_functions(code, funcs):
'''
Performs inline substitution of all the functions in the code
Parameters
----------
code : str
The abstract code to make inline substitutions into.
funcs : list, dict or set of AbstractCodeFunction
The function substitutions to use, note in the case of a dict, the
keys are ignored and the function name is used.
Returns
-------
code : str
The code with inline substitutions performed.
'''
if isinstance(funcs, (list, set)):
newfuncs = dict()
for f in funcs:
newfuncs[f.name] = f
funcs = newfuncs
code = deindent(code)
lines = ast.parse(code, mode='exec').body
# This is a slightly nasty hack, but basically we just check by looking at
# the existing identifiers how many inline operations have already been
# performed by previous calls to this function
ids = get_identifiers(code)
funcstarts = {}
for func in funcs.values():
subids = {
id
for id in ids if id.startswith('_inline_' + func.name + '_')
}
subids = {
id.replace('_inline_' + func.name + '_', '')
for id in subids
}
alli = []
for subid in subids:
p = subid.find('_')
if p > 0:
subid = subid[:p]
i = int(subid)
alli.append(i)
if len(alli) == 0:
i = 0
else:
i = max(alli) + 1
funcstarts[func.name] = i
# Now we rewrite all the lines, replacing each line with a sequence of
# lines performing the inlining
newlines = []
for line in lines:
for func in funcs.values():
rw = FunctionRewriter(func, funcstarts[func.name])
line = rw.visit(line)
newlines.extend(rw.pre)
funcstarts[func.name] = rw.numcalls
newlines.append(line)
# Now we render to a code string
nr = NodeRenderer()
newcode = '\n'.join(nr.render_node(line) for line in newlines)
# We recurse until no changes in the code to ensure that all functions
# are expanded if one function refers to another, etc.
if newcode == code:
return newcode
else:
return substitute_abstract_code_functions(newcode, funcs)
print('Saving notebook and converting to RST')
exporter = NotebookExporter()
output, _ = exporter.from_notebook_node(notebook)
with codecs.open(output_ipynb_fname, 'w', encoding='utf-8') as f:
f.write(output)
# Insert a note about ipython notebooks with a download link
note = deindent('''
.. only:: html
.. |launchbinder| image:: http://mybinder.org/badge.svg
.. _launchbinder: https://mybinder.org/v2/gh/angela-team/angela2-binder/master?filepath=tutorials/{tutorial}.ipynb
.. note::
This tutorial is a static non-editable version. You can launch an
interactive, editable version without installing any local files
using the Binder service (although note that at some times this
may be slow or fail to open): |launchbinder|_
Alternatively, you can download a copy of the notebook file
to use locally: :download:`{tutorial}.ipynb`
See the :doc:`tutorial overview page <index>` for more details.
'''.format(tutorial=basename))
notebook.cells.insert(
1, NotebookNode(cell_type='raw', metadata={}, source=note))
exporter = RSTExporter()
output, resources = exporter.from_notebook_node(
notebook, resources={'unique_key': basename + '_image'})
with codecs.open(output_rst_fname, 'w', encoding='utf-8') as f:
f.write(output)
def make_statements(code, variables, dtype, optimise=True, blockname=''):
'''
make_statements(code, variables, dtype, optimise=True, blockname='')
Turn a series of abstract code statements into Statement objects, inferring
whether each line is a set/declare operation, whether the variables are
constant or not, and handling the cacheing of subexpressions.
Parameters
----------
code : str
A (multi-line) string of statements.
variables : dict-like
A dictionary of with `Variable` and `Function` objects for every
identifier used in the `code`.
dtype : `dtype`
The data type to use for temporary variables
optimise : bool, optional
Whether to optimise expressions, including
pulling out loop invariant expressions and putting them in new
scalar constants. Defaults to ``False``, since this function is also
used just to in contexts where we are not interested by this kind of
optimisation. For the main code generation stage, its value is set by
the `codegen.loop_invariant_optimisations` preference.
blockname : str, optional
A name for the block (used to name intermediate variables to avoid
name clashes when multiple blocks are used together)
Returns
-------
scalar_statements, vector_statements : (list of `Statement`, list of `Statement`)
Lists with statements that are to be executed once and statements that
are to be executed once for every neuron/synapse/... (or in a vectorised
way)
Notes
-----
If ``optimise`` is ``True``, then the
``scalar_statements`` may include newly introduced scalar constants that
have been identified as loop-invariant and have therefore been pulled out
of the vector statements. The resulting statements will also use augmented
assignments where possible, i.e. a statement such as ``w = w + 1`` will be
replaced by ``w += 1``. Also, statements involving booleans will have
additional information added to them (see `Statement` for details)
describing how the statement can be reformulated as a sequence of if/then
statements. Calls `~angela2.codegen.optimisation.optimise_statements`.
'''
code = strip_empty_lines(deindent(code))
lines = re.split(r'[;\n]', code)
lines = [LineInfo(code=line) for line in lines if len(line)]
# Do a copy so we can add stuff without altering the original dict
variables = dict(variables)
# we will do inference to work out which lines are := and which are =
defined = set(k for k, v in variables.items()
if not isinstance(v, AuxiliaryVariable))
for line in lines:
statement = None
# parse statement into "var op expr"
var, op, expr, comment = parse_statement(line.code)
if var in variables and isinstance(variables[var], Subexpression):
raise SyntaxError("Illegal line '{line}' in abstract code. "
"Cannot write to subexpression "
"'{var}'.".format(line=line.code, var=var))
if op == '=':
if var not in defined:
op = ':='
defined.add(var)
if var not in variables:
annotated_ast = angela_ast(expr, variables)
is_scalar = annotated_ast.scalar
if annotated_ast.dtype == 'boolean':
use_dtype = bool
elif annotated_ast.dtype == 'integer':
use_dtype = int
else:
use_dtype = dtype
new_var = AuxiliaryVariable(var,
dtype=use_dtype,
scalar=is_scalar)
variables[var] = new_var
elif not variables[var].is_boolean:
sympy_expr = str_to_sympy(expr, variables)
if variables[var].is_integer:
sympy_var = sympy.Symbol(var, integer=True)
else:
sympy_var = sympy.Symbol(var, real=True)
try:
collected = sympy.collect(sympy_expr,
sympy_var,
exact=True,
evaluate=False)
except AttributeError:
# If something goes wrong during collection, e.g. collect
# does not work for logical expressions
collected = {1: sympy_expr}
if (len(collected) == 2
and set(collected.keys()) == {1, sympy_var}
and collected[sympy_var] == 1):
# We can replace this statement by a += assignment
statement = Statement(var,
'+=',
sympy_to_str(collected[1]),
comment,
dtype=variables[var].dtype,
scalar=variables[var].scalar)
elif len(collected) == 1 and sympy_var in collected:
# We can replace this statement by a *= assignment
statement = Statement(var,
'*=',
sympy_to_str(collected[sympy_var]),
comment,
dtype=variables[var].dtype,
scalar=variables[var].scalar)
if statement is None:
statement = Statement(var,
op,
expr,
comment,
dtype=variables[var].dtype,
scalar=variables[var].scalar)
line.statement = statement
# for each line will give the variable being written to
line.write = var
# each line will give a set of variables which are read
line.read = get_identifiers_recursively([expr], variables)
# All writes to scalar variables must happen before writes to vector
# variables
scalar_write_done = False
for line in lines:
stmt = line.statement
if stmt.op != ':=' and variables[
stmt.var].scalar and scalar_write_done:
raise SyntaxError(
('All writes to scalar variables in a code block '
'have to be made before writes to vector '
'variables. Illegal write to %s.') % line.write)
elif not variables[stmt.var].scalar:
scalar_write_done = True
# all variables which are written to at some point in the code block
# used to determine whether they should be const or not
all_write = set(line.write for line in lines)
# backwards compute whether or not variables will be read again
# note that will_read for a line gives the set of variables it will read
# on the current line or subsequent ones. will_write gives the set of
# variables that will be written after the current line
will_read = set()
will_write = set()
for line in lines[::-1]:
will_read = will_read.union(line.read)
line.will_read = will_read.copy()
line.will_write = will_write.copy()
will_write.add(line.write)
subexpressions = dict((name, val) for name, val in variables.items()
if isinstance(val, Subexpression))
# Check that no scalar subexpression refers to a vectorised function
# (e.g. rand()) -- otherwise it would be differently interpreted depending
# on whether it is used in a scalar or a vector context (i.e., even though
# the subexpression is supposed to be scalar, it would be vectorised when
# used as part of non-scalar expressions)
for name, subexpr in subexpressions.items():
if subexpr.scalar:
identifiers = get_identifiers(subexpr.expr)
for identifier in identifiers:
if (identifier in variables and getattr(
variables[identifier], 'auto_vectorise', False)):
raise SyntaxError(('The scalar subexpression {} refers to '
'the implicitly vectorised function {} '
'-- this is not allowed since it leads '
'to different interpretations of this '
'subexpression depending on whether it '
'is used in a scalar or vector '
'context.').format(name, identifier))
# sort subexpressions into an order so that subexpressions that don't depend
# on other subexpressions are first
subexpr_deps = dict(
(name, [dep for dep in subexpr.identifiers if dep in subexpressions])
for name, subexpr in subexpressions.items())
sorted_subexpr_vars = topsort(subexpr_deps)
statements = []
# none are yet defined (or declared)
subdefined = dict((name, None) for name in subexpressions)
for line in lines:
stmt = line.statement
read = line.read
write = line.write
will_read = line.will_read
will_write = line.will_write
# update/define all subexpressions needed by this statement
for var in sorted_subexpr_vars:
if var not in read:
continue
subexpression = subexpressions[var]
# if already defined/declared
if subdefined[var] == 'constant':
continue
elif subdefined[var] == 'variable':
op = '='
constant = False
else:
op = ':='
# check if the referred variables ever change
ids = subexpression.identifiers
constant = all(v not in will_write for v in ids)
subdefined[var] = 'constant' if constant else 'variable'
statement = Statement(var,
op,
subexpression.expr,
comment='',
dtype=variables[var].dtype,
constant=constant,
subexpression=True,
scalar=variables[var].scalar)
statements.append(statement)
var, op, expr, comment = stmt.var, stmt.op, stmt.expr, stmt.comment
# constant only if we are declaring a new variable and we will not
# write to it again
constant = op == ':=' and var not in will_write
statement = Statement(var,
op,
expr,
comment,
dtype=variables[var].dtype,
constant=constant,
scalar=variables[var].scalar)
statements.append(statement)
scalar_statements = [s for s in statements if s.scalar]
vector_statements = [s for s in statements if not s.scalar]
if optimise and prefs.codegen.loop_invariant_optimisations:
scalar_statements, vector_statements = optimise_statements(
scalar_statements,
vector_statements,
variables,
blockname=blockname)
return scalar_statements, vector_statements
def _add_user_function(self, varname, variable, added):
impl = variable.implementations[self.codeobj_class]
if (impl.name, variable) in added:
return # nothing to do
else:
added.add((impl.name, variable))
support_code = []
hash_defines = []
pointers = []
user_functions = [(varname, variable)]
funccode = impl.get_code(self.owner)
if isinstance(funccode, str):
# Rename references to any dependencies if necessary
for dep_name, dep in impl.dependencies.items():
dep_impl = dep.implementations[self.codeobj_class]
dep_impl_name = dep_impl.name
if dep_impl_name is None:
dep_impl_name = dep.pyfunc.__name__
if dep_name != dep_impl_name:
funccode = word_substitute(funccode,
{dep_name: dep_impl_name})
funccode = {'support_code': funccode}
if funccode is not None:
# To make namespace variables available to functions, we
# create global variables and assign to them in the main
# code
func_namespace = impl.get_namespace(self.owner) or {}
for ns_key, ns_value in func_namespace.items():
if hasattr(ns_value, 'dtype'):
if ns_value.shape == ():
raise NotImplementedError(
('Directly replace scalar values in the function '
'instead of providing them via the namespace'))
type_str = self.c_data_type(ns_value.dtype) + '*'
else: # e.g. a function
type_str = 'py::object'
support_code.append('static {0} _namespace{1};'.format(
type_str, ns_key))
pointers.append('_namespace{0} = {1};'.format(ns_key, ns_key))
support_code.append(deindent(funccode.get('support_code', '')))
hash_defines.append(deindent(funccode.get('hashdefine_code', '')))
dep_hash_defines = []
dep_pointers = []
dep_support_code = []
if impl.dependencies is not None:
for dep_name, dep in impl.dependencies.items():
if dep_name not in self.variables:
self.variables[dep_name] = dep
dep_impl = dep.implementations[self.codeobj_class]
if dep_name != dep_impl.name:
self.func_name_replacements[dep_name] = dep_impl.name
user_function = self._add_user_function(
dep_name, dep, added)
if user_function is not None:
hd, ps, sc, uf = user_function
dep_hash_defines.extend(hd)
dep_pointers.extend(ps)
dep_support_code.extend(sc)
user_functions.extend(uf)
return (dep_hash_defines + hash_defines, dep_pointers + pointers,
dep_support_code + support_code, user_functions)