def final_spec(args, obj_space, cspaces, addr_spaces, targets, architecture):
"""
Generates a final CapDL spec file that can be given to a capdl loader application
"""
arch = lookup_architecture(architecture)
for (e, key) in targets:
name = os.path.basename(e)
elf = ELF(e, name, architecture)
cspace = cspaces[key]
# Avoid inferring a TCB as we've already created our own.
elf_spec = elf.get_spec(infer_tcb=False,
infer_asid=False,
pd=addr_spaces[key].vspace_root,
addr_space=addr_spaces[key])
obj_space.merge(elf_spec, label=key)
for (slot, tcb) in [(k, v.referent)
for (k, v) in cspace.cnode.slots.items()
if v is not None and isinstance(v.referent, TCB)]:
if tcb['cspace'] and tcb['cspace'].referent is not cspace.cnode:
# We exclude TCBs that refer to a different CSpace
continue
funcs = {"get_vaddr": lambda x: elf.get_symbol_vaddr(x)}
s = EvalWithCompoundTypes(functions=funcs)
tcb.ip = s.eval(str(tcb.ip))
tcb.sp = s.eval(str(tcb.sp))
tcb.addr = s.eval(str(tcb.addr))
tcb.init = s.eval(str(tcb.init))
if not args.fprovide_tcb_caps:
del cspace.cnode[slot]
cspace.cnode.finalise_size(arch=arch)
return obj_space
def expression_validator(validator: Validator, value: Any,
form_data: Dict[str, Any]):
"""Validate if expression associated with value is true.
:param validator:
The validator instance for the current question.
:param value:
The value to be validated.
:param form_data:
The dictionary containing from data entered for current form.
:Returns:
If validation passes, :data:`True`, else :data:`False`.
"""
expression = validator.expression
result = True
if expression:
expression = expression.replace("{", "").replace("}", "")
expression = expression.replace(" empty", " in [[], {}]")
expression = expression.replace(" notempty", " not in [[], {}]")
expression = expression.replace(" anyof ", " in ")
try:
evaluator = Evaluator(names=form_data)
result = evaluator.eval(expression) is True
except (ValueError, TypeError, SyntaxError, KeyError,
InvalidExpression):
result = False
return result
def __call__(self):
s = EvalWithCompoundTypes(names=self.vars, functions=self.functions)
recipients = s.eval(self.settings['recipients'])
for recipient in recipients:
recipient_data = self._get_recipient_data(recipient)
subject = self.settings['subject_tpl'].format(
recipient=recipient_data,
**self.vars,
)
body = self.settings['body_tpl'].format(
recipient=recipient_data,
**self.vars,
)
to_email = getattr(recipient_data, 'email', recipient)
if to_email:
send_mail(
subject,
body,
self.settings['from_email'],
[
to_email,
],
fail_silently=True,
)
else:
logger.error('No destination e-mail could be found for %s',
recipient)
def parse_expr(expr: str, names: dict, fns: dict, errors: list = None):
errors = errors if type(errors) == list else []
names2 = Munch2.fromDict({**names, **{
}})
fns = {'F': update_recursive(fns, {
'list': {
'next': fn_list_next,
'join': fn_list_join
},
'str': fn_str,
'float': fn_float,
'int': fn_int,
'has': fn_has,
'next': fn_list_next,
'join': fn_list_join,
'trim': fn_str_trim,
'extend': fn_extend,
'update': fn_update,
'encrypt': fn_encrypt_names(names2),
'decrypt': fn_decrypt_names(names2),
'inc': fn_inc_names(names2),
'linspace': fn_linspace,
'arange': fn_arange
})}
s = EvalWithCompoundTypes(names=names2, functions=fns)
try:
v = s.eval(expr=expr)
except Exception as ex:
v = expr
errors.append([expr, ex])
v = filter_value(v)
return v
class TestComprehensions(DRYTest):
""" Test the comprehensions support of the compound-types edition of the class. """
def setUp(self):
self.s = EvalWithCompoundTypes()
def test_basic(self):
self.t('[a + 1 for a in [1,2,3]]', [2, 3, 4])
def test_with_self_reference(self):
self.t('[a + a for a in [1,2,3]]', [2, 4, 6])
def test_with_if(self):
self.t('[a for a in [1,2,3,4,5] if a <= 3]', [1, 2, 3])
def test_with_multiple_if(self):
self.t('[a for a in [1,2,3,4,5] if a <= 3 and a > 1 ]', [2, 3])
def test_attr_access_fails(self):
with self.assertRaises(FeatureNotAvailable):
self.t('[a.__class__ for a in [1,2,3]]', None)
def test_unpack(self):
self.t('[a+b for a,b in ((1,2),(3,4))]', [3, 7])
def test_nested_unpack(self):
self.t('[a+b+c for a, (b, c) in ((1,(1,1)),(3,(2,2)))]', [3, 7])
def test_other_places(self):
self.s.functions = {'sum': sum}
self.t('sum([a+1 for a in [1,2,3,4,5]])', 20)
self.t('sum(a+1 for a in [1,2,3,4,5])', 20)
def test_external_names_work(self):
self.s.names = {'x': [22, 102, 12.3]}
self.t('[a/2 for a in x]', [11.0, 51.0, 6.15])
self.s.names = lambda x: ord(x.id)
self.t('[a + a for a in [b, c, d]]', [ord(x) * 2 for x in 'bcd'])
def test_multiple_generators(self):
self.s.functions = {'range': range}
s = '[j for i in range(100) if i > 10 for j in range(i) if j < 20]'
self.t(s, eval(s))
def test_triple_generators(self):
self.s.functions = {'range': range}
s = '[(a,b,c) for a in range(4) for b in range(a) for c in range(b)]'
self.t(s, eval(s))
def test_too_long_generator(self):
self.s.functions = {'range': range}
s = '[j for i in range(1000) if i > 10 for j in range(i) if j < 20]'
with self.assertRaises(simpleeval.IterableTooLong):
self.s.eval(s)
def test_too_long_generator_2(self):
self.s.functions = {'range': range}
s = '[j for i in range(100) if i > 1 for j in range(i+10) if j < 100 for k in range(i*j)]'
with self.assertRaises(simpleeval.IterableTooLong):
self.s.eval(s)
def test_nesting_generators_to_cheat(self):
self.s.functions = {'range': range}
s = '[[[c for c in range(a)] for a in range(b)] for b in range(200)]'
with self.assertRaises(simpleeval.IterableTooLong):
self.s.eval(s)
class TestCompoundTypes(DRYTest):
""" Test the compound-types edition of the library """
def setUp(self):
self.s = EvalWithCompoundTypes()
def test_dict(self):
self.t('{}', {})
self.t('{"foo": "bar"}', {'foo': 'bar'})
self.t('{"foo": "bar"}["foo"]', 'bar')
self.t('dict()', {})
self.t('dict(a=1)', {'a': 1})
def test_dict_contains(self):
self.t('{"a":22}["a"]', 22)
with self.assertRaises(KeyError):
self.t('{"a":22}["b"]', 22)
self.t('{"a": 24}.get("b", 11)', 11)
self.t('"a" in {"a": 24}', True)
def test_tuple(self):
self.t('()', ())
self.t('(1,)', (1, ))
self.t('(1, 2, 3, 4, 5, 6)', (1, 2, 3, 4, 5, 6))
self.t('(1, 2) + (3, 4)', (1, 2, 3, 4))
self.t('(1, 2, 3)[1]', 2)
self.t('tuple()', ())
self.t('tuple("foo")', ('f', 'o', 'o'))
def test_tuple_contains(self):
self.t('("a","b")[1]', 'b')
with self.assertRaises(IndexError):
self.t('("a","b")[5]', 'b')
self.t('"a" in ("b","c","a")', True)
def test_list(self):
self.t('[]', [])
self.t('[1]', [1])
self.t('[1, 2, 3, 4, 5]', [1, 2, 3, 4, 5])
self.t('[1, 2, 3][1]', 2)
self.t('list()', [])
self.t('list("foo")', ['f', 'o', 'o'])
def test_list_contains(self):
self.t('["a","b"][1]', 'b')
with self.assertRaises(IndexError):
self.t('("a","b")[5]', 'b')
self.t('"b" in ["a","b"]', True)
def test_set(self):
self.t('{1}', {1})
self.t('{1, 2, 1, 2, 1, 2, 1}', {1, 2})
self.t('set()', set())
self.t('set("foo")', {'f', 'o'})
self.t('2 in {1,2,3,4}', True)
self.t('22 not in {1,2,3,4}', True)
def test_not(self):
self.t('not []', True)
self.t('not [0]', False)
self.t('not {}', True)
self.t('not {0: 1}', False)
self.t('not {0}', False)
def test_use_func(self):
self.s = EvalWithCompoundTypes(functions={"map": map, "str": str})
self.t('list(map(str, [-1, 0, 1]))', ['-1', '0', '1'])
with self.assertRaises(NameNotDefined):
self.s.eval('list(map(bad, [-1, 0, 1]))')
with self.assertRaises(FunctionNotDefined):
self.s.eval('dir(str)')
with self.assertRaises(FeatureNotAvailable):
self.s.eval('str.__dict__')
self.s = EvalWithCompoundTypes(functions={"dir": dir, "str": str})
self.t('dir(str)', dir(str))
class TestComprehensions(DRYTest):
""" Test the comprehensions support of the compound-types edition of the class. """
def setUp(self):
self.s = EvalWithCompoundTypes()
def test_basic(self):
self.t('[a + 1 for a in [1,2,3]]', [2,3,4])
def test_with_self_reference(self):
self.t('[a + a for a in [1,2,3]]', [2,4,6])
def test_with_if(self):
self.t('[a for a in [1,2,3,4,5] if a <= 3]', [1,2,3])
def test_with_multiple_if(self):
self.t('[a for a in [1,2,3,4,5] if a <= 3 and a > 1 ]', [2,3])
def test_attr_access_fails(self):
with self.assertRaises(FeatureNotAvailable):
self.t('[a.__class__ for a in [1,2,3]]', None)
def test_unpack(self):
self.t('[a+b for a,b in ((1,2),(3,4))]', [3, 7])
def test_nested_unpack(self):
self.t('[a+b+c for a, (b, c) in ((1,(1,1)),(3,(2,2)))]', [3, 7])
def test_other_places(self):
self.s.functions = {'sum': sum}
self.t('sum([a+1 for a in [1,2,3,4,5]])', 20)
self.t('sum(a+1 for a in [1,2,3,4,5])', 20)
def test_external_names_work(self):
self.s.names = {'x': [22, 102, 12.3]}
self.t('[a/2 for a in x]', [11.0, 51.0, 6.15])
self.s.names = lambda x: ord(x.id)
self.t('[a + a for a in [b, c, d]]', [ord(x) * 2 for x in 'bcd'])
def test_multiple_generators(self):
self.s.functions = {'range': range}
s = '[j for i in range(100) if i > 10 for j in range(i) if j < 20]'
self.t(s, eval(s))
def test_triple_generators(self):
self.s.functions = {'range': range}
s = '[(a,b,c) for a in range(4) for b in range(a) for c in range(b)]'
self.t(s, eval(s))
def test_too_long_generator(self):
self.s.functions = {'range': range}
s = '[j for i in range(1000) if i > 10 for j in range(i) if j < 20]'
with self.assertRaises(simpleeval.IterableTooLong):
self.s.eval(s)
def test_too_long_generator_2(self):
self.s.functions = {'range': range}
s = '[j for i in range(100) if i > 1 for j in range(i+10) if j < 100 for k in range(i*j)]'
with self.assertRaises(simpleeval.IterableTooLong):
self.s.eval(s)
def test_nesting_generators_to_cheat(self):
self.s.functions = {'range': range}
s = '[[[c for c in range(a)] for a in range(b)] for b in range(200)]'
with self.assertRaises(simpleeval.IterableTooLong):
self.s.eval(s)
def test_no_leaking_names(self):
# see issue #52, failing list comprehensions could leak locals
with self.assertRaises(simpleeval.NameNotDefined):
self.s.eval('[x if x == "2" else y for x in "123"]')
with self.assertRaises(simpleeval.NameNotDefined):
self.s.eval('x')
class TestCompoundTypes(DRYTest):
""" Test the compound-types edition of the library """
def setUp(self):
self.s = EvalWithCompoundTypes()
def test_dict(self):
self.t('{}', {})
self.t('{"foo": "bar"}', {'foo': 'bar'})
self.t('{"foo": "bar"}["foo"]', 'bar')
self.t('dict()', {})
self.t('dict(a=1)', {'a': 1})
def test_dict_contains(self):
self.t('{"a":22}["a"]', 22)
with self.assertRaises(KeyError):
self.t('{"a":22}["b"]', 22)
self.t('{"a": 24}.get("b", 11)', 11)
self.t('"a" in {"a": 24}', True)
def test_tuple(self):
self.t('()', ())
self.t('(1,)', (1,))
self.t('(1, 2, 3, 4, 5, 6)', (1, 2, 3, 4, 5, 6))
self.t('(1, 2) + (3, 4)', (1, 2, 3, 4))
self.t('(1, 2, 3)[1]', 2)
self.t('tuple()', ())
self.t('tuple("foo")', ('f', 'o', 'o'))
def test_tuple_contains(self):
self.t('("a","b")[1]', 'b')
with self.assertRaises(IndexError):
self.t('("a","b")[5]', 'b')
self.t('"a" in ("b","c","a")', True)
def test_list(self):
self.t('[]', [])
self.t('[1]', [1])
self.t('[1, 2, 3, 4, 5]', [1, 2, 3, 4, 5])
self.t('[1, 2, 3][1]', 2)
self.t('list()', [])
self.t('list("foo")', ['f', 'o', 'o'])
def test_list_contains(self):
self.t('["a","b"][1]', 'b')
with self.assertRaises(IndexError):
self.t('("a","b")[5]', 'b')
self.t('"b" in ["a","b"]', True)
def test_set(self):
self.t('{1}', {1})
self.t('{1, 2, 1, 2, 1, 2, 1}', {1, 2})
self.t('set()', set())
self.t('set("foo")', {'f', 'o'})
self.t('2 in {1,2,3,4}', True)
self.t('22 not in {1,2,3,4}', True)
def test_not(self):
self.t('not []', True)
self.t('not [0]', False)
self.t('not {}', True)
self.t('not {0: 1}', False)
self.t('not {0}', False)
def test_use_func(self):
self.s = EvalWithCompoundTypes(functions={"map": map, "str": str})
self.t('list(map(str, [-1, 0, 1]))', ['-1', '0', '1'])
with self.assertRaises(NameNotDefined):
self.s.eval('list(map(bad, [-1, 0, 1]))')
with self.assertRaises(FunctionNotDefined):
self.s.eval('dir(str)')
with self.assertRaises(FeatureNotAvailable):
self.s.eval('str.__dict__')
self.s = EvalWithCompoundTypes(functions={"dir": dir, "str": str})
self.t('dir(str)', dir(str))
print(e) # Sorry, List is not available in this evaluator
try:
my_eval = SimpleEval()
print(my_eval.eval('[1, 2, 3, 4]'))
except Exception as e:
print(e) # Sorry, List is not available in this evaluator
print()
# Compound Types
my_compound_types_eval = EvalWithCompoundTypes()
my_compound_types_eval.functions['len'] = len
# list
print(my_compound_types_eval.eval('[1, 2, 3, 4]')) # [1, 2, 3, 4]
print(my_compound_types_eval.eval('[1, 2] + [3, 4]')) # [1, 2, 3, 4]
print(my_compound_types_eval.eval('len([1, 2, 3, 4])')) # 4
print(my_compound_types_eval.eval('[1, 2, 1, 3, 4].count(1)')) # 2
print(my_compound_types_eval.eval('list("1234")')) # ['1', '2', '3', '4']
print()
# dict
print(my_compound_types_eval.eval('{"a": 1, "b": 999}')) # {'a': 1, 'b': 999}
print(my_compound_types_eval.eval('{"a": 1, "b": 999}["b"]')) # 999
print(my_compound_types_eval.eval(
'{"a": 1, "b": 999}.items()')) # dict_items([('a', 1), ('b', 999)])
print(my_compound_types_eval.eval('len({"a": 1, "b": 999})')) # 2
print(my_compound_types_eval.eval(
'dict([("a", 1), ("b", 999)])')) # {'a': 1, 'b': 999}
print()
class Parser:
opts = {
'max_power': simpleeval.MAX_POWER,
'max_string_length': simpleeval.MAX_STRING_LENGTH,
'disallow_prefixes': simpleeval.DISALLOW_PREFIXES
}
def __init__(self,
params=None,
names=None,
fns=None,
operators=None,
opts=None):
"""
:param params: A dictionary containing some parameters that will modify
how the builtin functions run. For example, the type of encryption to
use and the encrpyption key to use
"""
self._params = params if params is not None else {}
self._params.update({'etype': 'fernet'})
self._operators = operators if operators is not None else simpleeval.DEFAULT_OPERATORS
opts = opts if opts is not None else {}
self.opts.update(opts)
self._names = Munch2(names) if names is not None else {}
fns = fns if fns is not None else {}
self._fns = {
'F':
update_recursive(
fns,
{fn[3:]: getattr(self, fn)
for fn in dir(self) if 'fn_' in fn})
}
# Update simpleeval safety options
for k, v in opts.items():
setattr(simpleeval, k.upper(), v)
self._eval = EvalWithCompoundTypes()
# self._evals_functions should mirror self._fns
# TODO: Make a test to ensure proper mirroring
self._eval.functions = self._fns
self._eval.names = self._names
if operators:
self._operators = operators
self._eval.operators = self._operators
self.errors = []
def parse_file(self,
fs_path: str,
fs_root: str = '',
include_dirs: list = [],
syntax='yaml'):
"""
Parse configuration file on disk.
:param fs_path: The path to the file on disk. If fs_root is specified,
this will be interpreted as a path relative to fs_root
:type fs_path: str
:param fs_root: Root directory to use when parsing. Defaults to the
directory of the input file.
:type fs_root: str
:param include_dirs: A list of additional directories in which to
search for included files. Always contains the directory of the input
file, and will also contain fs_root if specified.
:type include_dirs: list
:param syntax: The syntax of the file on disk. Defaults to YAML
:type syntax: str
"""
fs_file_path = os.path.join(fs_root, fs_path)
fs_root = fs_root if fs_root is None else os.path.dirname(fs_file_path)
self._params.update({'fs_path': fs_path, 'fs_root': fs_root})
with open(fs_file_path) as stream:
# load_yaml initial structure
data = yaml_safe_load(stream, ordered=True)
names = {'R': data}
self._names.update(names)
data = self._parse(data)
# Delete anything specific to this file so we can reuse the parser
for k in ('fs_path', 'fs_root', 'R'):
if k in self._params:
del self._params[k]
return data
def parse_dict(self, in_data):
"""
Parse a dictionary containing conff syntax
"""
names = {'R': in_data}
self._names.update(names)
data = self._parse(in_data)
return data
def parse_expr(self, expr: str):
"""
Parse a string
"""
try:
v = self._eval.eval(expr=expr)
except SyntaxError as ex:
v = expr
# print("Raised simpleeval exception {} for expression {}".format(type(ex), v))
self.errors.append([expr, ex])
except simpleeval.InvalidExpression as ex:
v = expr
# print("Raised simpleeval exception {} for expression {}".format(type(ex), v))
# print("Raised simpleeval exception {} for expression {}".format(type(ex), v))
# print("Message: {}".format(ex))
self.errors.append(ex)
except Exception as ex:
print('Exception on expression: {}'.format(expr))
raise
v = filter_value(v)
return v
def _parse(self, root):
"""
The main parsing function
"""
root_type = type(root)
if root_type == dict or root_type == odict:
root_keys = list(root.keys())
for k, v in root.items():
root[k] = self._parse(v)
if 'F.extend' in root_keys:
root = self.fn_extend(root['F.extend'], root)
if isinstance(root, dict):
del root['F.extend']
if 'F.update' in root_keys:
self.fn_update(root['F.update'], root)
del root['F.update']
if 'F.foreach' in root_keys:
for k in ('values', 'template'):
if k not in root['F.foreach']:
raise ValueError('F.foreach missing key: {}'.format(k))
self.fn_foreach(root['F.foreach'], root)
del root['F.foreach']
elif root_type == list:
for i, v in enumerate(root):
root[i] = self._parse(root=v)
elif root_type == str:
value = root
if type(value) == str:
value = self.parse_expr(root)
return value
return root
def add_functions(self, funcs: dict):
"""
Add functions to the list of available parsing function. Funcs should
be a dict whose keys are the name you would like the function to have,
and whose value is a callable that maps to that name. The functions
will be callable via F.name_of_func(args_go_here)
"""
def add_names(self, names: dict):
"""
Add names to the dictionary of names available when parsing. These
names are accessible via the syntax R.path.to.name
"""
def generate_crypto_key(self):
"""
Generate a cryptographic key for encrypting data. Stores the key in
self._params['ekey'] so it is accessible to encrypt parsing functions.
Also returns the key
"""
try:
etype = self._params['etype']
except KeyError:
etype = 'fernet'
if etype == 'fernet':
key = Fernet.generate_key()
else:
key = None
self._params['ekey'] = key
return key
def fn_str(self, val):
return str(val)
def fn_float(self, val):
return float(val)
def fn_int(self, val):
return int(val)
def fn_has(self, val, name):
if isinstance(val, collections.Mapping):
return val.get(name, False) is not False
else:
return name in val
def fn_next(self, vals, default=None):
vals = [vals] if type(vals) != list else vals
val = next(iter(vals), default)
return val
def fn_join(self, vals, sep=' '):
vals = [val for val in vals if val]
return sep.join(vals)
def fn_trim(self, val: str, cs: list = None):
cs = cs if cs else ['/', ' ']
for c in cs:
val = val.strip(c)
return val
def fn_linspace(self, start, end, steps):
delta = (end - start) / (steps - 1)
return [start + delta * i for i in range(steps)]
def fn_arange(self, start, end, delta):
vals = [start]
while vals[-1] + delta <= end:
vals.append(vals[-1] + delta)
return vals
def fn_extend(self, val, val2):
val = copy.deepcopy(val)
type_val = type(val)
type_val2 = type(val2)
if type_val == list and type_val2 == list:
val.extend(val2)
elif type_val in [dict, odict, Munch2
] and type_val2 in [dict, odict, Munch2]:
for k, v in val2.items():
val[k] = v
return val
def fn_update(self, update, parent):
def walk(u, p):
tu, tp = type(u), type(p)
if tu in [dict, odict, Munch2] and tp in [dict, odict, Munch2]:
for k, v in u.items():
p[k] = walk(v, p.get(k, v))
return p
else:
return u
walk(update, parent)
def fn_encrypt(self, data):
etype = self._params.get('etype', None)
ekey = self._params.get('ekey', None)
token = None
if etype == 'fernet':
f = Fernet(ekey)
token = f.encrypt(data=str(data).encode()).decode()
return token
def fn_decrypt(self, data):
etype = self._params.get('etype', None)
ekey = self._params.get('ekey', None)
message = None
if etype == 'fernet':
f = Fernet(ekey)
message = f.decrypt(token=str(data).encode()).decode()
return message
def fn_inc(self, fs_path, syntax: str = 'yaml', fs_root: str = None):
fs_root = fs_root if fs_root else self._params['fs_root']
# Make sure to pass on any modified options to the sub parser
sub_parser = Parser(opts=self.opts)
data = sub_parser.parse_file(fs_path=fs_path,
fs_root=fs_root,
syntax=syntax)
return data
def fn_foreach(self, foreach, parent):
template = foreach['template']
if not isinstance(template, dict):
raise ValueError('template item of F.foreach must be a dict')
for i, v in enumerate(foreach['values']):
self._names.update({
'loop': {
'index': i,
'value': v,
'length': len(foreach['values'])
}
})
result = {}
for key, val in template.items():
pkey = self.parse_expr(key)
pval = self._parse(copy.copy(val))
result[pkey] = pval
parent.update(result)
# remove this specific foreach loop info from names dict so we don't
# break any subsequent foreach loops
del self._names['loop']
