def __init__(self,
deployable,
node,
name=None,
deploy_info=None,
security=None,
verify=True,
cb=None):
super(Deployer, self).__init__()
self.deployable = deployable
self.deploy_info = deploy_info
self.sec = security
self.actorstore = ActorStore(security=self.sec)
self.actor_map = {}
self.actor_connections = {}
self.node = node
#self.verify = verify # FIXME: what is this used for?
self.cb = cb
self._verified_actors = {}
self._deploy_counter = 0
self._instantiate_counter = 0
if name:
self.name = name
self.app_id = self.node.app_manager.new(self.name)
self.ns = os.path.splitext(os.path.basename(self.name))[0]
elif "name" in self.deployable:
self.name = self.deployable["name"]
self.app_id = self.node.app_manager.new(self.name)
self.ns = os.path.splitext(os.path.basename(self.name))[0]
else:
self.app_id = self.node.app_manager.new(None)
self.name = self.app_id
self.ns = ""
self.group_components()
_log.analyze(self.node.id, "+ SECURITY", {'sec': str(self.sec)})
class TestActorStore(object):
def setup_class(self):
self.ms = ActorStore()
pass
def teardown_class(self):
pass
def test_find_modules(self):
# Valid
module = self.ms.lookup("std.Sum")
assert len(module) is 3
assert module[0]
# Fail
module = self.ms.lookup("non.ExistantActor")
assert not module[0]
# Sys module
module = self.ms.lookup("os")
assert not module[0]
def test_load_modules(self):
pass
@pytest.mark.xfail # May or may not pass. Not that important
def test_perf(self):
time = timeit.timeit(lambda: self.ms.lookup("std.Sum"), number=1000)
assert time < .2
def __init__(self):
self.store = ActorStore()
self.actors = {}
self.illegal_actors = {}
self.components = {}
self.id = "ActorTester"
self.metering = metering.set_metering(metering.Metering(self))
class TestActorStore(object):
def setup_class(self):
self.ms = ActorStore()
pass
def teardown_class(self):
pass
def test_find_modules(self):
# Valid
module = self.ms.lookup("std.Sum")
assert len(module) is 4
assert module[0]
# Fail
module = self.ms.lookup("non.ExistantActor")
assert not module[0]
# Sys module
module = self.ms.lookup("os")
assert not module[0]
def test_load_modules(self):
pass
@pytest.mark.xfail # May or may not pass. Not that important
def test_perf(self):
time = timeit.timeit(lambda: self.ms.lookup("std.Sum"), number=1000)
assert time < .2
def __init__(self):
self.store = ActorStore()
self.actors = {}
self.illegal_actors = {}
self.components = {}
self.id = "ActorTester"
setup_calvinlib()
self.test_sys = setup_calvinsys()
def __init__(self, deployable, node, name=None, deploy_info=None, security=None, verify=True, cb=None):
super(Deployer, self).__init__()
self.deployable = deployable
self.deploy_info = deploy_info
self.sec = security
self.actorstore = ActorStore(security=self.sec)
self.actor_map = {}
self.actor_connections = {}
self.node = node
#self.verify = verify # FIXME: what is this used for?
self.cb = cb
self._verified_actors = {}
self._deploy_counter = 0
self._instantiate_counter = 0
if name:
self.name = name
self.app_id = self.node.app_manager.new(self.name)
self.ns = os.path.splitext(os.path.basename(self.name))[0]
elif "name" in self.deployable:
self.name = self.deployable["name"]
self.app_id = self.node.app_manager.new(self.name)
self.ns = os.path.splitext(os.path.basename(self.name))[0]
else:
self.app_id = self.node.app_manager.new(None)
self.name = self.app_id
self.ns = ""
self.group_components()
_log.analyze(self.node.id, "+ SECURITY", {'sec': str(self.sec)})
def __init__(self):
self.store = ActorStore()
self.actors = {}
self.illegal_actors = {}
self.components = {}
self.id = "ActorTester"
self.metering = metering.set_metering(metering.Metering(self))
def _new_actor(self, actor_type, actor_id=None):
"""Return a 'bare' actor of actor_type, raises an exception on failure."""
(found, is_primitive, class_) = ActorStore().lookup(actor_type)
if not found:
# Here assume a primtive actor, now become shadow actor
_log.analyze(self.node.id, "+ NOT FOUND CREATE SHADOW ACTOR",
{'class': class_})
found = True
is_primitive = True
class_ = ShadowActor
if not found or not is_primitive:
_log.error("Requested actor %s is not available" % (actor_type))
raise Exception("ERROR_NOT_FOUND")
try:
# Create a 'bare' instance of the actor
a = class_(actor_type, actor_id)
except Exception as e:
_log.exception("")
_log.error("The actor %s(%s) can't be instantiated." %
(actor_type, class_.__init__))
raise (e)
try:
a._calvinsys = self.node.calvinsys()
a.check_requirements()
except Exception as e:
_log.analyze(self.node.id, "+ FAILED REQS CREATE SHADOW ACTOR",
{'class': class_})
a = ShadowActor(actor_type, actor_id)
a._calvinsys = self.node.calvinsys()
return a
def __init__(self, deployable, node, name=None, deploy_info=None, credentials=None, verify=True, cb=None):
super(Deployer, self).__init__()
self.deployable = deployable
self.deploy_info = deploy_info
self.credentials = credentials
self.sec = Security(node)
self.sec.set_subject(self.credentials)
self.actorstore = ActorStore(security=self.sec)
self.actor_map = {}
self.actor_connections = {}
self.node = node
self.verify = verify
self.cb = cb
self._deploy_cont_done = False
if name:
self.name = name
self.app_id = self.node.app_manager.new(self.name)
self.ns = os.path.splitext(os.path.basename(self.name))[0]
elif "name" in self.deployable:
self.name = self.deployable["name"]
self.app_id = self.node.app_manager.new(self.name)
self.ns = os.path.splitext(os.path.basename(self.name))[0]
else:
self.app_id = self.node.app_manager.new(None)
self.name = self.app_id
self.ns = ""
self.group_components()
_log.analyze(self.node.id, "+ SECURITY", {'sec': str(self.sec)})
def lookup_and_verify(self, actor_type, security=None):
"""Lookup and verify actor in actor store."""
found, is_primitive, actor_def, signer = ActorStore(
security=security).lookup(actor_type)
if not found or not is_primitive:
raise Exception("Not known actor type: %s" % actor_type)
return (actor_def, signer)
def manage_cs_sign(args):
if not args.name:
raise Exception("No code signer name supplied")
if not args.file:
raise Exception("supply path to a file(s) to sign")
cs = code_signer.CS(organization=args.name, commonName=args.name+"CS", security_dir=args.dir, force=args.force)
# Collect files to sign
files = []
if args.file:
for f in args.file:
exist = os.path.isfile(f)
if not exist:
raise Exception("The file path supplied is not an existing file")
files.extend(glob.glob(f))
if args.nsfile:
store = ActorStore()
for m in args.nsfile:
files.extend(store.actor_paths(m))
# Filter out any files not *.calvin, *.py
files = [f for f in files if f.endswith(('.calvin', '.py')) and not f.endswith('__init__.py')]
if not files:
raise Exception("No (*.calvin, *.py) files supplied")
exceptions = []
for f in files:
try:
cs.sign_file(f)
except Exception as e:
exceptions.append(e)
for e in exceptions:
print "Error {}".format(e)
def __init__(self):
self.store = ActorStore()
self.actors = {}
self.illegal_actors = {}
self.components = {}
self.id = "ActorTester"
setup_calvinlib()
self.test_sys = setup_calvinsys()
def __init__(self, cs_info, verify=True):
super(Analyzer, self).__init__()
self.cs_info = cs_info
self.local_components = cs_info['components'] if 'components' in cs_info else {}
self.constants = {}
self.app_info = {}
self.connections = {}
self.actors = {}
self.verify = verify
self.actorstore = ActorStore()
self.analyze()
def _new_actor(self, actor_type, actor_id=None):
"""Return a 'bare' actor of actor_type, raises an exception on failure."""
(found, is_primitive, class_) = ActorStore().lookup(actor_type)
if not found or not is_primitive:
_log.error("Requested actor %s is not available" % (actor_type))
raise Exception("ERROR_NOT_FOUND")
try:
# Create a 'bare' instance of the actor
a = class_(actor_type, actor_id)
# Hand over a CalvinSys factory method to the actor.
a.attach_API("calvinsys", self.node.calvinsys)
except Exception as e:
_log.exception("")
_log.error("The actor %s(%s) can't be instantiated." % (actor_type, class_.__init__))
raise(e)
return a
def lookup(self, actor_type):
"""
Search for the definition of 'actor_type'.
Returns a tuple (found, is_primitive, info) where info is either a
class (primitive) or a dictionary with component definition
Search order:
1 - components defined in the current script: self.local_components
2 - primitive actors in the order defined by actor store
3 - components in the order defined by actor store
Steps 2 and 3 are handled by generic lookup in actor store
"""
if actor_type in self.local_components:
compdef = self.local_components[actor_type]
return (True, False, compdef)
return ActorStore().lookup(actor_type)
def _new_actor(self, actor_type):
"""Return a 'bare' actor of actor_type, raises an exception on failure."""
(found, is_primitive, class_) = ActorStore().lookup(actor_type)
if not found or not is_primitive:
_log.error("Requested actor %s is not available" % (actor_type))
raise Exception("ERROR_NOT_FOUND")
try:
# Create a 'bare' instance of the actor
a = class_(actor_type)
# FIXME: Resolve the required (calvin) APIs and attach them to the actor
# (if it has the required access rights)
a.attach_API("calvinsys", CalvinSys(self.node))
except Exception as e:
_log.exception("")
_log.error("The actor %s(%s) can't be instantiated." %
(actor_type, class_.__init__))
raise (e)
return a
def instantiate(self, actor_name, actor_type, argd):
"""
Instantiate an actor.
- 'actor_name' is <namespace>:<identifier>, e.g. app:src, or app:component:src
- 'actor_type' is the actor class to instatiate
- 'argd' is a dictionary with <actor_name>:<argdict> pairs
"""
found, is_primitive, actor_def = ActorStore().lookup(actor_type)
if not found:
raise Exception("Unknown actor type: %s" % actor_type)
if not is_primitive:
raise Exception("Non-primitive type: %s" % actor_type)
instance_id = self.instantiate_primitive(actor_name, actor_type, argd)
if not instance_id:
raise Exception("Could not instantiate actor of type: %s" %
actor_type)
self.actor_map[actor_name] = instance_id
class ActorTester(object):
def __init__(self):
self.store = ActorStore()
self.actors = {}
self.illegal_actors = {}
self.components = {}
self.id = "ActorTester"
setup_calvinlib()
self.test_sys = setup_calvinsys()
def collect_actors(self, actor):
actors = [m + '.' + a for m in self.store.modules() for a in self.store.actors(m)]
if actor:
actors = [a for a in actors if actor in a]
self.actor_names = actors
def instantiate_actor(self, actorclass, actorname):
try:
actor = actorclass(actorname, disable_state_checks=True)
if not hasattr(actor, 'test_set'):
self.actors[actorname] = 'no_test'
_log.warning("%s not tested, no test_set defined." % actorname)
return
actor.init(*actorclass.test_args, **actorclass.test_kwargs)
actor.setup_complete()
except AssertionError as e:
raise e
except Exception as e:
self.illegal_actors[actorname] = "Failed to instantiate"
sys.stderr.write("Actor %s: %s" % (actorname, e))
sys.stderr.write(''.join(traceback.format_exc()))
raise e
for inport in actor.inports.values():
inport.set_queue(queue.fanout_fifo.FanoutFIFO({'queue_length': 100, 'direction': "in"}, {}))
inport.endpoint = DummyInEndpoint(inport)
inport.queue.add_reader(inport.id, {})
for outport in actor.outports.values():
outport.set_queue(queue.fanout_fifo.FanoutFIFO({'queue_length': 100, 'direction': "out"}, {}))
outport.queue.add_reader(actor.id, {})
outport.endpoints.append(DummyOutEndpoint(outport))
self.actors[actorname] = actor
def instantiate_actors(self):
test_fail = {}
for a in self.actor_names:
found, primitive, actorclass, signer = self.store.lookup(a)
if found and primitive:
try:
self.instantiate_actor(actorclass, a)
except AssertionError as e:
test_fail[a] = e.message
except Exception as e:
raise e
elif found and not primitive:
self.components[a] = "TODO: Cannot test components (%s)" % (a,)
else:
self.illegal_actors[a] = "Unknown actor - probably parsing issues"
return test_fail
def load_actor(self, path):
test_fail = {}
actorclass, _ = self.store.load_from_path(path)
if actorclass:
try:
self.instantiate_actor(actorclass, path)
except AssertionError as e:
test_fail[path] = e.message
except Exception as e:
raise e
else:
self.illegal_actors[path] = "Unknown actor - probably parsing issues"
return test_fail
def test_actor(self, actor, aut):
for idx in range(len(aut.test_set)):
test_index = idx + 1
test = aut.test_set[idx]
setups = test.get('setup', [])
inputs = test.get('inports', {})
outputs = test.get('outports', {})
postconds = test.get('postcond', [])
for f in setups:
try:
f(aut)
except Exception as e:
print "Actor %s failed during setup of test %d: %s" % (actor, test_index, e.message)
raise Exception("Failed during setup of test %d" % (test_index, ))
for port, values in inputs.iteritems():
pwrite(aut, port, values)
self.test_sys.verify_read_write_during_init(aut, actor)
self.test_sys.init_done(actor)
sched = scheduler.BaseScheduler(None, None)
sched._fire_actor(aut)
for port, values in outputs.iteritems():
try:
vals = pread(aut, port, len(values))
if type(values) is set:
# disregard order
assert set(vals) == values, "Expected output set '%s' does not match '%s'" % (set(vals), values)
else:
assert vals == values, "Expected output '%s' does not match '%s'" % (vals, values)
except AssertionError as e:
print "Error:", str(e)
raise AssertionError("Failed test %d" % (test_index,))
if not all(f(aut) for f in postconds):
raise AssertionError("Failed post condition of test %d" % (test_index, ))
return True
def test_actors(self):
test_pass = []
test_fail = {}
no_test = []
for actor in self.actors:
aut = self.actors[actor]
if aut == "no_test":
no_test.append(actor)
continue
try:
self.test_actor(actor, aut)
test_pass.append(actor)
except AssertionError as e:
test_fail[actor] = e.message
except Exception as e:
self.illegal_actors[actor] = str(e) + '\n' + ''.join(
traceback.format_exception(sys.exc_type, sys.exc_value, sys.exc_traceback))
return {'pass': test_pass, 'fail': test_fail, 'skipped': no_test,
'errors': self.illegal_actors, 'components': self.components}
class Deployer(object):
"""
Process an app_info dictionary (output from calvin parser) to
produce a running calvin application.
"""
def __init__(self, deployable, node, name=None, deploy_info=None, credentials=None, verify=True, cb=None):
super(Deployer, self).__init__()
self.deployable = deployable
self.deploy_info = deploy_info
self.credentials = credentials
self.sec = Security(node)
self.sec.set_subject(self.credentials)
self.actorstore = ActorStore(security=self.sec)
self.actor_map = {}
self.actor_connections = {}
self.node = node
self.verify = verify
self.cb = cb
self._deploy_cont_done = False
if name:
self.name = name
self.app_id = self.node.app_manager.new(self.name)
self.ns = os.path.splitext(os.path.basename(self.name))[0]
elif "name" in self.deployable:
self.name = self.deployable["name"]
self.app_id = self.node.app_manager.new(self.name)
self.ns = os.path.splitext(os.path.basename(self.name))[0]
else:
self.app_id = self.node.app_manager.new(None)
self.name = self.app_id
self.ns = ""
self.group_components()
_log.analyze(self.node.id, "+ SECURITY", {'sec': str(self.sec)})
# TODO Make deployer use the Application class group_components, component_name and get_req
def group_components(self):
self.components = {}
l = (len(self.ns)+1) if self.ns else 0
for name in self.deployable['actors']:
if name.find(':',l)> -1:
# This is part of a component
# component name including optional namespace
component = ':'.join(name.split(':')[0:(2 if self.ns else 1)])
if component in self.components:
self.components[component].append(name)
else:
self.components[component] = [name]
def component_name(self, name):
l = (len(self.ns)+1) if self.ns else 0
if name.find(':',l)> -1:
return ':'.join(name.split(':')[0:(2 if self.ns else 1)])
else:
return None
def get_req(self, actor_name):
name = self.component_name(actor_name) or actor_name
name = name.split(':', 1)[1] if self.ns else name
return self.deploy_info['requirements'][name] if (self.deploy_info and 'requirements' in self.deploy_info
and name in self.deploy_info['requirements']) else []
def instantiate(self, actor_name, actor_type, argd, signature=None):
"""
Instantiate an actor.
- 'actor_name' is <namespace>:<identifier>, e.g. app:src, or app:component:src
- 'actor_type' is the actor class to instatiate
- 'argd' is a dictionary with <actor_name>:<argdict> pairs
- 'signature' is the GlobalStore actor-signature to lookup the actor
"""
req = self.get_req(actor_name)
_log.analyze(self.node.id, "+ SECURITY", {'sec': str(self.sec)})
found, is_primitive, actor_def = self.actorstore.lookup(actor_type)
if not found or not is_primitive:
raise Exception("Not known actor type: %s" % actor_type)
actor_id = self.instantiate_primitive(actor_name, actor_type, argd, req, signature)
if not actor_id:
raise Exception(
"Could not instantiate actor of type: %s" % actor_type)
self.actor_map[actor_name] = actor_id
self.node.app_manager.add(self.app_id, actor_id)
def instantiate_primitive(self, actor_name, actor_type, args, req=None, signature=None):
# name is <namespace>:<identifier>, e.g. app:src, or app:component:src
# args is a **dictionary** of key-value arguments for this instance
# signature is the GlobalStore actor-signature to lookup the actor
args['name'] = actor_name
actor_id = self.node.am.new(actor_type=actor_type, args=args, signature=signature, credentials=self.credentials)
if req:
self.node.am.actors[actor_id].requirements_add(req, extend=False)
return actor_id
def connectid(self, connection):
src_actor, src_port, dst_actor, dst_port = connection
# connect from dst to src
# use node info if exists, otherwise assume local node
dst_actor_id = self.actor_map[dst_actor]
src_actor_id = self.actor_map[src_actor]
src_node = self.node.id
result = self.node.connect(
actor_id=dst_actor_id,
port_name=dst_port,
port_dir='in',
peer_node_id=src_node,
peer_actor_id=src_actor_id,
peer_port_name=src_port,
peer_port_dir='out')
return result
def set_port_property(self, actor, port_type, port_name, port_property, value):
self.node.am.set_port_property(self.actor_map[actor], port_type, port_name, port_property, value)
def select_actor(self, out_iter, kwargs, final, comp_name_desc):
_log.analyze(self.node.id, "+", {'comp_name_desc': comp_name_desc}, tb=True)
if final[0] and not kwargs['done']:
kwargs['done'] = True
for name, desc_list in kwargs['priority'].iteritems():
if desc_list:
out_iter.append(desc_list[0])
out_iter.final()
return
desc = comp_name_desc[1]
try:
# List of (found, is_primitive, info)
actor_types = [self.actorstore.lookup(actor['actor_type'])
for actor in desc['component']['structure']['actors'].values()]
except KeyError:
actor_types = []
# Not a component, shadow actor candidate, likely
kwargs['priority'][comp_name_desc[0]].insert(0, comp_name_desc)
comp_name_desc[1]['shadow_actor'] = True
return
except Exception as e:
# FIXME Handled when security verification failed
_log.exception("select_actor desc: %s" % desc)
raise e
if all([a[0] and a[1] for a in actor_types]):
# All found and primitive (quite unlikely), insert after any primitive shadow actors in priority
index = len([1 for a in kwargs['priority'][comp_name_desc[0]] if 'shadow_actor' in a[1]])
kwargs['priority'][comp_name_desc[0]].insert(index, comp_name_desc)
comp_name_desc[1]['shadow_component'] = actor_types
return
# A component containing shadow actors
# TODO Dig deeper to priorities between shadow components, now just insert in order
kwargs['priority'][comp_name_desc[0]].append(comp_name_desc)
comp_name_desc[1]['shadow_component'] = actor_types
def resolve_remote(self, deployables):
all_desc_iters = dynops.List()
store = GlobalStore(node=self.node)
for actor_name, info in deployables.iteritems():
desc_iter = store.global_lookup_iter(info['signature'], info['args'].keys())
all_desc_iters.append((actor_name, desc_iter), trigger_iter=desc_iter)
all_desc_iters.final()
collect_desc_iter = dynops.Collect(all_desc_iters).set_name("collected_desc")
select_iter = dynops.Map(self.select_actor, collect_desc_iter, done=False,
priority={k:[] for k in self.deployable['actors'].keys()},
eager=True).set_name("select_actor")
select_iter.set_cb(self.deploy_unhandled_actors, select_iter)
self.deploy_unhandled_actors(select_iter)
def deploy_unhandled_actors(self, comp_name_desc):
while True:
try:
name, desc = comp_name_desc.next()
_log.analyze(self.node.id, "+", {'name': name, 'desc': desc}, tb=True)
except StopIteration:
# Done
if self._deploy_cont_done:
return
self._deploy_cont_done = True
self.group_components()
_log.analyze(self.node.id, "+ DONE", {'deployable': self.deployable, 'components': self.components})
self._deploy_cont()
return
except dynops.PauseIteration:
return
if 'shadow_actor' in desc:
_log.analyze(self.node.id, "+ SHADOW ACTOR", {'name': name})
# It was a normal primitive shadow actor, just instanciate
req = self.get_req(name)
info = self.deployable['actors'][name]
actor_id = self.instantiate_primitive(name, info['actor_type'], info['args'], req, info['signature'])
if not actor_id:
_log.error("Second phase, could not make shadow actor %s!" % info['actor_type'])
self.actor_map[name] = actor_id
self.node.app_manager.add(self.app_id, actor_id)
elif 'shadow_component' in desc:
_log.analyze(self.node.id, "+ SHADOW COMPONENT", {'name': name})
# A component that needs to be broken up into individual primitive actors
# First get the info and remove the component
req = self.get_req(name)
info = self.deployable['actors'][name]
self.deployable['actors'].pop(name)
# Then add the new primitive actors
for actor_name, actor_desc in desc['component']['structure']['actors'].iteritems():
args = {k: v[1] if v[0] == 'VALUE' else info['args'][v[1]] for k, v in actor_desc['args'].iteritems()}
inports = [c['dst_port'] for c in desc['component']['structure']['connections'] if c['dst'] == actor_name]
outports = [c['src_port'] for c in desc['component']['structure']['connections'] if c['src'] == actor_name]
sign_desc = {'is_primitive': True,
'actor_type': actor_desc['actor_type'],
'inports': inports[:],
'outports': outports[:]}
sign = GlobalStore.actor_signature(sign_desc)
self.deployable['actors'][name + ":" + actor_name] = {'args': args,
'actor_type': actor_desc['actor_type'],
'signature_desc': sign_desc,
'signature': sign}
# Replace component connections with actor connection
# outports
comp_outports = [(c['dst_port'], c['src_port']) for c in desc['component']['structure']['connections']
if c['src'] == actor_name and c['dst'] == "."]
for c_port, a_port in comp_outports:
if (name + "." + c_port) in self.deployable['connections']:
self.deployable['connections'][name + ":" + actor_name + "." + a_port] = \
self.deployable['connections'].pop(name + "." + c_port)
# inports
comp_inports = [(c['src_port'], c['dst_port']) for c in desc['component']['structure']['connections']
if c['dst'] == actor_name and c['src'] == "."]
for outport, ports in self.deployable['connections'].iteritems():
for c_inport, a_inport in comp_inports:
if (name + "." + c_inport) in ports:
ports.remove(name + "." + c_inport)
ports.append(name + ":" + actor_name + "." + a_inport)
_log.analyze(self.node.id, "+ REPLACED PORTS", {'comp_outports': comp_outports,
'comp_inports': comp_inports,
'actor_name': actor_name,
'connections': self.deployable['connections']})
# Add any new component internal connections (enough with outports)
for connection in desc['component']['structure']['connections']:
if connection['src'] == actor_name and connection['src_port'] in outports and connection['dst'] != ".":
self.deployable['connections'].setdefault(
name + ":" + actor_name + "." + connection['src_port'], []).append(
name + ":" + connection['dst'] + "." + connection['dst_port'])
_log.analyze(self.node.id, "+ ADDED PORTS", {'connections': self.deployable['connections']})
# Instanciate it
actor_id = self.instantiate_primitive(name + ":" + actor_name, actor_desc['actor_type'], args, req, sign)
if not actor_id:
_log.error("Third phase, could not make shadow actor %s!" % info['actor_type'])
self.actor_map[name + ":" + actor_name] = actor_id
self.node.app_manager.add(self.app_id, actor_id)
def deploy(self):
"""
Instantiate actors and link them together.
"""
if not self.deployable['valid']:
raise Exception("Deploy information is not valid")
unhandled = {}
for actor_name, info in self.deployable['actors'].iteritems():
try:
self.instantiate(actor_name, info['actor_type'], info['args'], signature=info['signature'])
except:
unhandled[actor_name] = info
if unhandled:
_log.analyze(self.node.id, "+ UNHANDLED", {'unhandled': unhandled})
self.resolve_remote(unhandled)
return
self._deploy_cont()
def _deploy_cont(self):
for component_name, actor_names in self.components.iteritems():
actor_ids = [self.actor_map[n] for n in actor_names]
for actor_id in actor_ids:
self.node.am.actors[actor_id].component_add(actor_ids)
for src, dst_list in self.deployable['connections'].iteritems():
if len(dst_list) > 1:
src_name, src_port = src.split('.')
self.set_port_property(src_name, 'out', src_port, 'fanout', len(dst_list))
for src, dst_list in self.deployable['connections'].iteritems():
src_actor, src_port = src.split('.')
for dst in dst_list:
dst_actor, dst_port = dst.split('.')
c = (src_actor, src_port, dst_actor, dst_port)
self.connectid(c)
self.node.app_manager.finalize(self.app_id, migrate=True if self.deploy_info else False,
cb=CalvinCB(self.cb, deployer=self))
class ActorTester(object):
def __init__(self):
self.store = ActorStore()
self.actors = {}
self.illegal_actors = {}
self.components = {}
self.id = "ActorTester"
setup_calvinlib()
self.test_sys = setup_calvinsys()
def collect_actors(self, actor):
actors = [
m + '.' + a for m in self.store.modules()
for a in self.store.actors(m)
]
if actor:
actors = [a for a in actors if actor in a]
self.actor_names = actors
def instantiate_actor(self, actorclass, actorname):
try:
actor = actorclass(actorname, disable_state_checks=True)
if not hasattr(actor, 'test_set'):
self.actors[actorname] = 'no_test'
_log.warning("%s not tested, no test_set defined." % actorname)
return
actor.init(*actorclass.test_args, **actorclass.test_kwargs)
actor.setup_complete()
except AssertionError as e:
raise e
except Exception as e:
self.illegal_actors[actorname] = "Failed to instantiate"
sys.stderr.write("Actor %s: %s" % (actorname, e))
sys.stderr.write(''.join(traceback.format_exc()))
raise e
for inport in actor.inports.values():
inport.set_queue(
queue.fanout_fifo.FanoutFIFO(
{
'queue_length': 100,
'direction': "in"
}, {}))
inport.endpoint = DummyInEndpoint(inport)
inport.queue.add_reader(inport.id, {})
for outport in actor.outports.values():
outport.set_queue(
queue.fanout_fifo.FanoutFIFO(
{
'queue_length': 100,
'direction': "out"
}, {}))
outport.queue.add_reader(actor.id, {})
outport.endpoints.append(DummyOutEndpoint(outport))
self.actors[actorname] = actor
def instantiate_actors(self):
test_fail = {}
for a in self.actor_names:
found, primitive, actorclass, signer = self.store.lookup(a)
if found and primitive:
try:
self.instantiate_actor(actorclass, a)
except AssertionError as e:
test_fail[a] = e.message
except Exception as e:
raise e
elif found and not primitive:
self.components[a] = "TODO: Cannot test components (%s)" % (
a, )
else:
self.illegal_actors[
a] = "Unknown actor - probably parsing issues"
return test_fail
def load_actor(self, path):
test_fail = {}
actorclass, _ = self.store.load_from_path(path)
if actorclass:
try:
self.instantiate_actor(actorclass, path)
except AssertionError as e:
test_fail[path] = e.message
except Exception as e:
raise e
else:
self.illegal_actors[
path] = "Unknown actor - probably parsing issues"
return test_fail
def test_actor(self, actor, aut):
for idx in range(len(aut.test_set)):
test_index = idx + 1
test = aut.test_set[idx]
setups = test.get('setup', [])
inputs = test.get('inports', {})
outputs = test.get('outports', {})
postconds = test.get('postcond', [])
for f in setups:
try:
f(aut)
except Exception as e:
print "Actor %s failed during setup of test %d: %s" % (
actor, test_index, e.message)
raise Exception("Failed during setup of test %d" %
(test_index, ))
for port, values in inputs.iteritems():
pwrite(aut, port, values)
self.test_sys.verify_read_write_during_init(aut, actor)
self.test_sys.init_done(actor)
sched = scheduler.BaseScheduler(None, None)
sched._fire_actor(aut)
for port, values in outputs.iteritems():
try:
vals = pread(aut, port, len(values))
if type(values) is set:
# disregard order
assert set(
vals
) == values, "Expected output set '%s' does not match '%s'" % (
set(vals), values)
else:
assert vals == values, "Expected output '%s' does not match '%s'" % (
vals, values)
except AssertionError as e:
print "Error:", str(e)
raise AssertionError("Failed test %d" % (test_index, ))
if not all(f(aut) for f in postconds):
raise AssertionError("Failed post condition of test %d" %
(test_index, ))
return True
def test_actors(self):
test_pass = []
test_fail = {}
no_test = []
for actor in self.actors:
aut = self.actors[actor]
if aut == "no_test":
no_test.append(actor)
continue
try:
self.test_actor(actor, aut)
test_pass.append(actor)
except AssertionError as e:
test_fail[actor] = e.message
except Exception as e:
self.illegal_actors[actor] = str(e) + '\n' + ''.join(
traceback.format_exception(sys.exc_type, sys.exc_value,
sys.exc_traceback))
return {
'pass': test_pass,
'fail': test_fail,
'skipped': no_test,
'errors': self.illegal_actors,
'components': self.components
}
class ActorTester(object):
def __init__(self):
self.store = ActorStore()
self.actors = {}
self.illegal_actors = {}
self.components = {}
self.id = "ActorTester"
self.metering = metering.set_metering(metering.Metering(self))
def collect_actors(self, actor):
actors = [m + '.' + a for m in self.store.modules() for a in self.store.actors(m)]
if actor:
actors = [a for a in actors if actor in a]
self.actor_names = actors
def instantiate_actor(self, actorclass, actorname):
try:
actor = actorclass(actorname, disable_state_checks=True)
if not hasattr(actor, 'test_set'):
self.actors[actorname] = 'no_test'
return
actor._calvinsys = CalvinSysMock()
actor._calvinsys['file'] = CalvinSysFileMock()
actor._calvinsys['timer'] = CalvinSysTimerMock()
actor.init(*actorclass.test_args, **actorclass.test_kwargs)
actor.setup_complete()
except Exception as e:
self.illegal_actors[actorname] = "Failed to instantiate"
sys.stderr.write("Actor %s: %s" % (actorname, e))
import traceback
sys.stderr.write(''.join(traceback.format_exc()))
raise e
for inport in actor.inports.values():
inport.endpoint = DummyInEndpoint(inport)
for outport in actor.outports.values():
outport.fifo.add_reader(actor.id)
self.actors[actorname] = actor
def instantiate_actors(self):
for a in self.actor_names:
found, primitive, actorclass = self.store.lookup(a)
if found and primitive:
self.instantiate_actor(actorclass, a)
elif found and not primitive:
self.components[a] = "TODO: Cannot test components (%s)" % (a,)
else:
self.illegal_actors[a] = "Unknown actor - probably parsing issues"
def load_actor(self, path):
actorclass = self.store.load_from_path(path)
if actorclass:
self.instantiate_actor(actorclass, path)
else:
self.illegal_actors[path] = "Unknown actor - probably parsing issues"
def test_actor(self, actor, aut):
for idx in range(len(aut.test_set)):
test_index = idx + 1
test = aut.test_set[idx]
setups = test.get('setup', [])
inputs = test.get('in', {})
outputs = test.get('out', {})
postconds = test.get('postcond', [])
for f in setups:
try:
f(aut)
except Exception as e:
print "Actor %s failed during setup of test %d: %s" % (actor, test_index, e.message)
raise Exception("Failed during setup of test %d" % (test_index, ))
for port, values in inputs.iteritems():
pwrite(aut, port, values)
aut.fire()
for port, values in outputs.iteritems():
try:
vals = pread(aut, port, len(values))
assert vals == values, "Expected output '%s' does not match '%s'" % (vals, values)
except AssertionError as e:
print "Error:", str(e)
raise AssertionError("Failed test %d" % (test_index,))
if not all(f(aut) for f in postconds):
raise AssertionError("Failed post condition of test %d" % (test_index, ))
return True
def test_actors(self):
test_pass = []
test_fail = {}
no_test = []
for actor in self.actors:
aut = self.actors[actor]
if aut == "no_test":
no_test.append(actor)
continue
try:
self.test_actor(actor, aut)
test_pass.append(actor)
except AssertionError as e:
test_fail[actor] = e.message
# raise e
except Exception as e:
self.illegal_actors[actor] = str(e)
return {'pass': test_pass, 'fail': test_fail, 'skipped': no_test,
'errors': self.illegal_actors, 'components': self.components}
class Deployer(object):
"""
Process an app_info dictionary (output from calvin parser) to
produce a running calvin application.
"""
def __init__(self,
deployable,
node,
name=None,
deploy_info=None,
security=None,
verify=True,
cb=None):
super(Deployer, self).__init__()
self.deployable = deployable
self.deploy_info = deploy_info
self.sec = security
self.actorstore = ActorStore(security=self.sec)
self.actor_map = {}
self.actor_connections = {}
self.node = node
#self.verify = verify # FIXME: what is this used for?
self.cb = cb
self._verified_actors = {}
self._deploy_counter = 0
self._instantiate_counter = 0
if name:
self.name = name
self.app_id = self.node.app_manager.new(self.name)
self.ns = os.path.splitext(os.path.basename(self.name))[0]
elif "name" in self.deployable:
self.name = self.deployable["name"]
self.app_id = self.node.app_manager.new(self.name)
self.ns = os.path.splitext(os.path.basename(self.name))[0]
else:
self.app_id = self.node.app_manager.new(None)
self.name = self.app_id
self.ns = ""
self.group_components()
_log.analyze(self.node.id, "+ SECURITY", {'sec': str(self.sec)})
# TODO Make deployer use the Application class group_components, component_name and get_req
def group_components(self):
self.components = {}
l = (len(self.ns) + 1) if self.ns else 0
for name in self.deployable['actors']:
if name.find(':', l) > -1:
# This is part of a component
# component name including optional namespace
component = ':'.join(name.split(':')[0:(2 if self.ns else 1)])
if component in self.components:
self.components[component].append(name)
else:
self.components[component] = [name]
def component_name(self, name):
l = (len(self.ns) + 1) if self.ns else 0
if name.find(':', l) > -1:
return ':'.join(name.split(':')[0:(2 if self.ns else 1)])
else:
return None
def get_req(self, actor_name):
name = self.component_name(actor_name) or actor_name
name = name.split(':', 1)[1] if self.ns else name
return self.deploy_info['requirements'][name] if (
self.deploy_info and 'requirements' in self.deploy_info
and name in self.deploy_info['requirements']) else []
def lookup_and_verify(self, actor_name, info, cb=None):
"""
Lookup and verify actor in actor store.
- 'actor_name' is <namespace>:<identifier>, e.g. app:src, or app:component:src
- 'info' is information about the actor
"""
actor_type = info['actor_type']
try:
actor_def, signer = self.node.am.lookup_and_verify(
actor_type, self.sec)
except Exception:
self.resolve_remote(actor_name, info, cb)
return
info['signer'] = signer
info['requires'] = actor_def.requires if hasattr(
actor_def, "requires") else []
self._verified_actors[actor_name] = (info, actor_def)
if cb:
cb()
def check_requirements_and_sec_policy(self,
actor_name,
info,
actor_def=None,
cb=None):
"""
Check requirements and security policy for actor.
- 'actor_name' is <namespace>:<identifier>, e.g. app:src, or app:component:src
- 'info' is information about the actor
- 'actor_def' is the actor definition returned from the actor store
"""
try:
if not 'shadow_actor' in info:
self.node.am.check_requirements_and_sec_policy(
info['requires'],
security=self.sec,
signer=info['signer'],
callback=CalvinCB(self.instantiate,
actor_name,
info,
actor_def,
cb=cb))
return
self.instantiate(actor_name, info, cb=cb)
except Exception:
# Still want to create shadow actor.
info['shadow_actor'] = True
self.instantiate(actor_name, info, cb=cb)
def instantiate(self,
actor_name,
info,
actor_def=None,
access_decision=None,
cb=None):
"""
Instantiate an actor.
- 'actor_name' is <namespace>:<identifier>, e.g. app:src, or app:component:src
- 'info' is information about the actor
info['args'] is a dictionary of key-value arguments for this instance
info['signature'] is the GlobalStore actor-signature to lookup the actor
- 'access_decision' is a boolean indicating if access is permitted
"""
try:
info['args']['name'] = actor_name
actor_id = self.node.am.new(actor_type=info['actor_type'],
args=info['args'],
signature=info['signature'],
actor_def=actor_def,
security=self.sec,
access_decision=access_decision,
shadow_actor='shadow_actor' in info)
if not actor_id:
raise Exception("Could not instantiate actor %s" % actor_name)
deploy_req = self.get_req(actor_name)
if deploy_req:
self.node.am.actors[actor_id].requirements_add(deploy_req,
extend=False)
self.actor_map[actor_name] = actor_id
self.node.app_manager.add(self.app_id, actor_id)
except Exception as e:
# FIXME: what should happen here?
raise e
finally:
if cb:
cb()
def connectid(self, connection):
src_actor, src_port, dst_actor, dst_port = connection
# connect from dst to src
# use node info if exists, otherwise assume local node
dst_actor_id = self.actor_map[dst_actor]
src_actor_id = self.actor_map[src_actor]
src_node = self.node.id
result = self.node.connect(actor_id=dst_actor_id,
port_name=dst_port,
port_dir='in',
peer_node_id=src_node,
peer_actor_id=src_actor_id,
peer_port_name=src_port,
peer_port_dir='out')
return result
def select_actor(self, out_iter, kwargs, final, comp_name_desc):
_log.analyze(self.node.id,
"+", {'comp_name_desc': comp_name_desc},
tb=True)
if final[0] and not kwargs['done']:
kwargs['done'] = True
for name, desc_list in kwargs['priority'].iteritems():
if desc_list:
out_iter.append(desc_list[0])
out_iter.final()
return
desc = comp_name_desc[1]
try:
# List of (found, is_primitive, info, signer)
# TODO: call lookup_and_verify() instead
actor_types = [
self.actorstore.lookup(actor['actor_type'])
for actor in desc['component']['structure']['actors'].values()
]
except KeyError:
actor_types = []
# Not a component, shadow actor candidate, likely
kwargs['priority'][comp_name_desc[0]].insert(0, comp_name_desc)
comp_name_desc[1]['shadow_actor'] = True
return
except Exception as e:
_log.exception("select_actor desc: %s" % desc)
raise e
if all([a[0] and a[1] for a in actor_types]):
# All found and primitive (quite unlikely), insert after any primitive shadow actors in priority
index = len([
1 for a in kwargs['priority'][comp_name_desc[0]]
if 'shadow_actor' in a[1]
])
kwargs['priority'][comp_name_desc[0]].insert(index, comp_name_desc)
comp_name_desc[1]['shadow_component'] = actor_types
return
# A component containing shadow actors
# TODO Dig deeper to priorities between shadow components, now just insert in order
kwargs['priority'][comp_name_desc[0]].append(comp_name_desc)
comp_name_desc[1]['shadow_component'] = actor_types
def resolve_remote(self, actor_name, info, cb=None):
# FIXME: no list needed since it is only done for one actor
all_desc_iters = dynops.List()
store = GlobalStore(node=self.node)
desc_iter = store.global_lookup_iter(info['signature'],
info['args'].keys())
all_desc_iters.append((actor_name, desc_iter), trigger_iter=desc_iter)
all_desc_iters.final()
collect_desc_iter = dynops.Collect(all_desc_iters).set_name(
"collected_desc")
select_iter = dynops.Map(
self.select_actor,
collect_desc_iter,
done=False,
priority={k: []
for k in self.deployable['actors'].keys()},
eager=True).set_name("select_actor")
select_iter.set_cb(self.deploy_unhandled_actor, select_iter, cb)
self.deploy_unhandled_actor(select_iter, cb)
def deploy_unhandled_actor(self, comp_name_desc, cb=None):
while True:
try:
name, desc = comp_name_desc.next()
_log.analyze(self.node.id,
"+", {
'name': name,
'desc': desc
},
tb=True)
except StopIteration:
return
except dynops.PauseIteration:
return
if 'shadow_actor' in desc:
_log.analyze(self.node.id, "+ SHADOW ACTOR", {'name': name})
# It was a normal primitive shadow actor, just add to list of verified actors.
info = self.deployable['actors'][name]
info['shadow_actor'] = True
self._verified_actors[name] = (info, None)
elif 'shadow_component' in desc:
_log.analyze(self.node.id, "+ SHADOW COMPONENT",
{'name': name})
# A component that needs to be broken up into individual primitive actors
# First, get the component requirements and info
req = self.get_req(name)
info = self.deployable['actors'][name]
# Then add the new primitive actors
for actor_name, actor_desc in desc['component']['structure'][
'actors'].iteritems():
args = {
k: v[1] if v[0] == 'VALUE' else info['args'][v[1]]
for k, v in actor_desc['args'].iteritems()
}
inports = [
c['dst_port']
for c in desc['component']['structure']['connections']
if c['dst'] == actor_name
]
outports = [
c['src_port']
for c in desc['component']['structure']['connections']
if c['src'] == actor_name
]
sign_desc = {
'is_primitive': True,
'actor_type': actor_desc['actor_type'],
'inports': inports[:],
'outports': outports[:]
}
sign = GlobalStore.actor_signature(sign_desc)
self._verified_actors[name + ":" + actor_name] = ({
'args':
args,
'actor_type':
actor_desc['actor_type'],
'signature_desc':
sign_desc,
'signature':
sign
}, None)
# Replace component connections with actor connection
# outports
comp_outports = [
(c['dst_port'], c['src_port'])
for c in desc['component']['structure']['connections']
if c['src'] == actor_name and c['dst'] == "."
]
for c_port, a_port in comp_outports:
if (name + "." +
c_port) in self.deployable['connections']:
self.deployable['connections'][name + ":" + actor_name + "." + a_port] = \
self.deployable['connections'].pop(name + "." + c_port)
# inports
comp_inports = [
(c['src_port'], c['dst_port'])
for c in desc['component']['structure']['connections']
if c['dst'] == actor_name and c['src'] == "."
]
for outport, ports in self.deployable[
'connections'].iteritems():
for c_inport, a_inport in comp_inports:
if (name + "." + c_inport) in ports:
ports.remove(name + "." + c_inport)
ports.append(name + ":" + actor_name + "." +
a_inport)
_log.analyze(
self.node.id, "+ REPLACED PORTS", {
'comp_outports': comp_outports,
'comp_inports': comp_inports,
'actor_name': actor_name,
'connections': self.deployable['connections']
})
# Add any new component internal connections (enough with outports)
for connection in desc['component']['structure'][
'connections']:
if connection['src'] == actor_name and connection[
'src_port'] in outports and connection[
'dst'] != ".":
self.deployable['connections'].setdefault(
name + ":" + actor_name + "." +
connection['src_port'],
[]).append(name + ":" + connection['dst'] +
"." + connection['dst_port'])
_log.analyze(
self.node.id, "+ ADDED PORTS",
{'connections': self.deployable['connections']})
self.group_components()
if cb:
cb()
def deploy(self):
"""Verify actors, instantiate and link them together."""
if not self.deployable['valid']:
raise Exception("Deploy information is not valid")
for actor_name, info in self.deployable['actors'].iteritems():
self.lookup_and_verify(actor_name,
info,
cb=CalvinCB(self._deploy_instantiate))
def _deploy_instantiate(self):
self._deploy_counter += 1
if self._deploy_counter < len(self.deployable['actors']):
return
for actor_name, info in self._verified_actors.iteritems():
self.check_requirements_and_sec_policy(actor_name,
info[0],
info[1],
cb=CalvinCB(
self._deploy_finalize))
def _deploy_finalize(self):
self._instantiate_counter += 1
if self._instantiate_counter < len(self._verified_actors):
return
for component_name, actor_names in self.components.iteritems():
actor_ids = [self.actor_map[n] for n in actor_names]
for actor_id in actor_ids:
self.node.am.actors[actor_id].component_add(actor_ids)
for src, dst_list in self.deployable['connections'].iteritems():
if len(dst_list) > 1:
src_name, src_port = src.split('.')
# TODO get routing method from actor or calvinscript, now set only existing option
self.node.pm.set_port_properties(
actor_id=self.actor_map[src_name],
port_dir='out',
port_name=src_port,
routing='fanout',
nbr_peers=len(dst_list))
for src, dst_list in self.deployable['connections'].iteritems():
src_actor, src_port = src.split('.')
for dst in dst_list:
dst_actor, dst_port = dst.split('.')
c = (src_actor, src_port, dst_actor, dst_port)
self.connectid(c)
self.node.app_manager.finalize(
self.app_id,
migrate=True if self.deploy_info else False,
cb=CalvinCB(self.cb, deployer=self))
class ActorTester(object):
def __init__(self):
self.store = ActorStore()
self.actors = {}
self.illegal_actors = {}
self.components = {}
def collect_actors(self, actor):
actors = [
m + '.' + a for m in self.store.modules()
for a in self.store.actors(m)
]
if actor:
actors = [a for a in actors if actor in a]
self.actor_names = actors
def instantiate_actor(self, actorclass, actorname):
try:
actor = actorclass(actorname, disable_state_checks=True)
if not hasattr(actor, 'test_set'):
self.actors[actorname] = 'no_test'
return
actor._calvinsys = CalvinSysMock()
actor._calvinsys['file'] = CalvinSysFileMock()
actor._calvinsys['timer'] = CalvinSysTimerMock()
actor.init(*actorclass.test_args, **actorclass.test_kwargs)
actor.setup_complete()
except Exception as e:
self.illegal_actors[actorname] = "Failed to instantiate"
sys.stderr.write("Actor %s: %s" % (actorname, e))
import traceback
sys.stderr.write(''.join(traceback.format_exc()))
raise e
for inport in actor.inports.values():
inport.endpoint = DummyInEndpoint(inport)
for outport in actor.outports.values():
outport.fifo.add_reader(actor.id)
self.actors[actorname] = actor
def instantiate_actors(self):
for a in self.actor_names:
found, primitive, actorclass = self.store.lookup(a)
if found and primitive:
self.instantiate_actor(actorclass, a)
elif found and not primitive:
self.components[a] = "TODO: Cannot test components (%s)" % (
a, )
else:
self.illegal_actors[
a] = "Unknown actor - probably parsing issues"
def load_actor(self, path):
actorclass = self.store.load_from_path(path)
if actorclass:
self.instantiate_actor(actorclass, path)
else:
self.illegal_actors[
path] = "Unknown actor - probably parsing issues"
def test_actor(self, actor, aut):
for idx in range(len(aut.test_set)):
test_index = idx + 1
test = aut.test_set[idx]
setups = test.get('setup', [])
inputs = test.get('in', {})
outputs = test.get('out', {})
postconds = test.get('postcond', [])
for f in setups:
try:
f(aut)
except Exception as e:
print "Actor %s failed during setup of test %d: %s" % (
actor, test_index, e.message)
raise Exception("Failed during setup of test %d" %
(test_index, ))
for port, values in inputs.iteritems():
pwrite(aut, port, values)
aut.fire()
for port, values in outputs.iteritems():
try:
vals = pread(aut, port, len(values))
assert vals == values, "Expected output '%s' does not match '%s'" % (
vals, values)
except AssertionError as e:
print "Error:", str(e)
raise AssertionError("Failed test %d" % (test_index, ))
if not all(f(aut) for f in postconds):
raise AssertionError("Failed post condition of test %d" %
(test_index, ))
return True
def test_actors(self):
test_pass = []
test_fail = {}
no_test = []
for actor in self.actors:
aut = self.actors[actor]
if aut == "no_test":
no_test.append(actor)
continue
try:
self.test_actor(actor, aut)
test_pass.append(actor)
except AssertionError as e:
test_fail[actor] = e.message
# raise e
except Exception as e:
self.illegal_actors[actor] = str(e)
return {
'pass': test_pass,
'fail': test_fail,
'skipped': no_test,
'errors': self.illegal_actors,
'components': self.components
}
def __init__(self):
self.store = ActorStore()
self.actors = {}
self.illegal_actors = {}
self.components = {}
class Deployer(object):
"""
Process an app_info dictionary (output from calvin parser) to
produce a running calvin application.
"""
def __init__(self, deployable, node, name=None, deploy_info=None, security=None, verify=True, cb=None):
super(Deployer, self).__init__()
self.deployable = deployable
self.deploy_info = deploy_info
self.sec = security
self.actorstore = ActorStore(security=self.sec)
self.actor_map = {}
self.actor_connections = {}
self.node = node
#self.verify = verify # FIXME: what is this used for?
self.cb = cb
self._verified_actors = {}
self._deploy_counter = 0
self._instantiate_counter = 0
if name:
self.name = name
self.app_id = self.node.app_manager.new(self.name)
self.ns = os.path.splitext(os.path.basename(self.name))[0]
elif "name" in self.deployable:
self.name = self.deployable["name"]
self.app_id = self.node.app_manager.new(self.name)
self.ns = os.path.splitext(os.path.basename(self.name))[0]
else:
self.app_id = self.node.app_manager.new(None)
self.name = self.app_id
self.ns = ""
self.group_components()
_log.analyze(self.node.id, "+ SECURITY", {'sec': str(self.sec)})
# TODO Make deployer use the Application class group_components, component_name and get_req
def group_components(self):
self.components = {}
l = (len(self.ns)+1) if self.ns else 0
for name in self.deployable['actors']:
if name.find(':',l)> -1:
# This is part of a component
# component name including optional namespace
component = ':'.join(name.split(':')[0:(2 if self.ns else 1)])
if component in self.components:
self.components[component].append(name)
else:
self.components[component] = [name]
def component_name(self, name):
l = (len(self.ns)+1) if self.ns else 0
if name.find(':',l)> -1:
return ':'.join(name.split(':')[0:(2 if self.ns else 1)])
else:
return None
def get_req(self, actor_name):
name = self.component_name(actor_name) or actor_name
name = name.split(':', 1)[1] if self.ns else name
return self.deploy_info['requirements'][name] if (self.deploy_info and 'requirements' in self.deploy_info
and name in self.deploy_info['requirements']) else []
def lookup_and_verify(self, actor_name, info, cb=None):
"""
Lookup and verify actor in actor store.
- 'actor_name' is <namespace>:<identifier>, e.g. app:src, or app:component:src
- 'info' is information about the actor
"""
actor_type = info['actor_type']
try:
actor_def, signer = self.node.am.lookup_and_verify(actor_type, self.sec)
except Exception:
self.resolve_remote(actor_name, info, cb)
return
info['signer'] = signer
info['requires'] = actor_def.requires if hasattr(actor_def, "requires") else []
self._verified_actors[actor_name] = (info, actor_def)
if cb:
cb()
def check_requirements_and_sec_policy(self, actor_name, info, actor_def=None, cb=None):
"""
Check requirements and security policy for actor.
- 'actor_name' is <namespace>:<identifier>, e.g. app:src, or app:component:src
- 'info' is information about the actor
- 'actor_def' is the actor definition returned from the actor store
"""
try:
if not 'shadow_actor' in info:
self.node.am.check_requirements_and_sec_policy(info['requires'],
security=self.sec,
signer=info['signer'],
callback=CalvinCB(self.instantiate,
actor_name, info,
actor_def, cb=cb))
return
self.instantiate(actor_name, info, cb=cb)
except Exception:
# Still want to create shadow actor.
info['shadow_actor'] = True
self.instantiate(actor_name, info, cb=cb)
def instantiate(self, actor_name, info, actor_def=None, access_decision=None, cb=None):
"""
Instantiate an actor.
- 'actor_name' is <namespace>:<identifier>, e.g. app:src, or app:component:src
- 'info' is information about the actor
info['args'] is a dictionary of key-value arguments for this instance
info['signature'] is the GlobalStore actor-signature to lookup the actor
- 'access_decision' is a boolean indicating if access is permitted
"""
try:
info['args']['name'] = actor_name
actor_id = self.node.am.new(actor_type=info['actor_type'], args=info['args'], signature=info['signature'],
actor_def=actor_def, security=self.sec, access_decision=access_decision,
shadow_actor='shadow_actor' in info)
if not actor_id:
raise Exception("Could not instantiate actor %s" % actor_name)
deploy_req = self.get_req(actor_name)
if deploy_req:
self.node.am.actors[actor_id].requirements_add(deploy_req, extend=False)
self.actor_map[actor_name] = actor_id
self.node.app_manager.add(self.app_id, actor_id)
except Exception as e:
# FIXME: what should happen here?
raise e
finally:
if cb:
cb()
def connectid(self, connection):
src_actor, src_port, dst_actor, dst_port = connection
# connect from dst to src
# use node info if exists, otherwise assume local node
dst_actor_id = self.actor_map[dst_actor]
src_actor_id = self.actor_map[src_actor]
src_node = self.node.id
result = self.node.connect(
actor_id=dst_actor_id,
port_name=dst_port,
port_dir='in',
peer_node_id=src_node,
peer_actor_id=src_actor_id,
peer_port_name=src_port,
peer_port_dir='out')
return result
def select_actor(self, out_iter, kwargs, final, comp_name_desc):
_log.analyze(self.node.id, "+", {'comp_name_desc': comp_name_desc}, tb=True)
if final[0] and not kwargs['done']:
kwargs['done'] = True
for name, desc_list in kwargs['priority'].iteritems():
if desc_list:
out_iter.append(desc_list[0])
out_iter.final()
return
desc = comp_name_desc[1]
try:
# List of (found, is_primitive, info, signer)
# TODO: call lookup_and_verify() instead
actor_types = [self.actorstore.lookup(actor['actor_type'])
for actor in desc['component']['structure']['actors'].values()]
except KeyError:
actor_types = []
# Not a component, shadow actor candidate, likely
kwargs['priority'][comp_name_desc[0]].insert(0, comp_name_desc)
comp_name_desc[1]['shadow_actor'] = True
return
except Exception as e:
_log.exception("select_actor desc: %s" % desc)
raise e
if all([a[0] and a[1] for a in actor_types]):
# All found and primitive (quite unlikely), insert after any primitive shadow actors in priority
index = len([1 for a in kwargs['priority'][comp_name_desc[0]] if 'shadow_actor' in a[1]])
kwargs['priority'][comp_name_desc[0]].insert(index, comp_name_desc)
comp_name_desc[1]['shadow_component'] = actor_types
return
# A component containing shadow actors
# TODO Dig deeper to priorities between shadow components, now just insert in order
kwargs['priority'][comp_name_desc[0]].append(comp_name_desc)
comp_name_desc[1]['shadow_component'] = actor_types
def resolve_remote(self, actor_name, info, cb=None):
# FIXME: no list needed since it is only done for one actor
all_desc_iters = dynops.List()
store = GlobalStore(node=self.node)
desc_iter = store.global_lookup_iter(info['signature'], info['args'].keys())
all_desc_iters.append((actor_name, desc_iter), trigger_iter=desc_iter)
all_desc_iters.final()
collect_desc_iter = dynops.Collect(all_desc_iters).set_name("collected_desc")
select_iter = dynops.Map(self.select_actor, collect_desc_iter, done=False,
priority={k:[] for k in self.deployable['actors'].keys()},
eager=True).set_name("select_actor")
select_iter.set_cb(self.deploy_unhandled_actor, select_iter, cb)
self.deploy_unhandled_actor(select_iter, cb)
def deploy_unhandled_actor(self, comp_name_desc, cb=None):
while True:
try:
name, desc = comp_name_desc.next()
_log.analyze(self.node.id, "+", {'name': name, 'desc': desc}, tb=True)
except StopIteration:
return
except dynops.PauseIteration:
return
if 'shadow_actor' in desc:
_log.analyze(self.node.id, "+ SHADOW ACTOR", {'name': name})
# It was a normal primitive shadow actor, just add to list of verified actors.
info = self.deployable['actors'][name]
info['shadow_actor'] = True
self._verified_actors[name] = (info, None)
elif 'shadow_component' in desc:
_log.analyze(self.node.id, "+ SHADOW COMPONENT", {'name': name})
# A component that needs to be broken up into individual primitive actors
# First, get the component requirements and info
req = self.get_req(name)
info = self.deployable['actors'][name]
# Then add the new primitive actors
for actor_name, actor_desc in desc['component']['structure']['actors'].iteritems():
args = {k: v[1] if v[0] == 'VALUE' else info['args'][v[1]] for k, v in actor_desc['args'].iteritems()}
inports = [c['dst_port'] for c in desc['component']['structure']['connections'] if c['dst'] == actor_name]
outports = [c['src_port'] for c in desc['component']['structure']['connections'] if c['src'] == actor_name]
sign_desc = {'is_primitive': True,
'actor_type': actor_desc['actor_type'],
'inports': inports[:],
'outports': outports[:]}
sign = GlobalStore.actor_signature(sign_desc)
self._verified_actors[name + ":" + actor_name] = ({'args': args,
'actor_type': actor_desc['actor_type'],
'signature_desc': sign_desc,
'signature': sign}, None)
# Replace component connections with actor connection
# outports
comp_outports = [(c['dst_port'], c['src_port']) for c in desc['component']['structure']['connections']
if c['src'] == actor_name and c['dst'] == "."]
for c_port, a_port in comp_outports:
if (name + "." + c_port) in self.deployable['connections']:
self.deployable['connections'][name + ":" + actor_name + "." + a_port] = \
self.deployable['connections'].pop(name + "." + c_port)
# inports
comp_inports = [(c['src_port'], c['dst_port']) for c in desc['component']['structure']['connections']
if c['dst'] == actor_name and c['src'] == "."]
for outport, ports in self.deployable['connections'].iteritems():
for c_inport, a_inport in comp_inports:
if (name + "." + c_inport) in ports:
ports.remove(name + "." + c_inport)
ports.append(name + ":" + actor_name + "." + a_inport)
_log.analyze(self.node.id, "+ REPLACED PORTS", {'comp_outports': comp_outports,
'comp_inports': comp_inports,
'actor_name': actor_name,
'connections': self.deployable['connections']})
# Add any new component internal connections (enough with outports)
for connection in desc['component']['structure']['connections']:
if connection['src'] == actor_name and connection['src_port'] in outports and connection['dst'] != ".":
self.deployable['connections'].setdefault(
name + ":" + actor_name + "." + connection['src_port'], []).append(
name + ":" + connection['dst'] + "." + connection['dst_port'])
_log.analyze(self.node.id, "+ ADDED PORTS", {'connections': self.deployable['connections']})
self.group_components()
if cb:
cb()
def deploy(self):
"""Verify actors, instantiate and link them together."""
if not self.deployable['valid']:
raise Exception("Deploy information is not valid")
for actor_name, info in self.deployable['actors'].iteritems():
self.lookup_and_verify(actor_name, info, cb=CalvinCB(self._deploy_instantiate))
def _deploy_instantiate(self):
self._deploy_counter += 1
if self._deploy_counter < len(self.deployable['actors']):
return
for actor_name, info in self._verified_actors.iteritems():
self.check_requirements_and_sec_policy(actor_name, info[0], info[1], cb=CalvinCB(self._deploy_finalize))
def _deploy_finalize(self):
self._instantiate_counter += 1
if self._instantiate_counter < len(self._verified_actors):
return
for component_name, actor_names in self.components.iteritems():
actor_ids = [self.actor_map[n] for n in actor_names]
for actor_id in actor_ids:
self.node.am.actors[actor_id].component_add(actor_ids)
for src, dst_list in self.deployable['connections'].iteritems():
if len(dst_list) > 1:
src_name, src_port = src.split('.')
# TODO get routing method from actor or calvinscript, now set only existing option
self.node.pm.set_port_properties(actor_id=self.actor_map[src_name], port_dir='out', port_name=src_port,
routing='fanout', nbr_peers=len(dst_list))
for src, dst_list in self.deployable['connections'].iteritems():
src_actor, src_port = src.split('.')
for dst in dst_list:
dst_actor, dst_port = dst.split('.')
c = (src_actor, src_port, dst_actor, dst_port)
self.connectid(c)
self.node.app_manager.finalize(self.app_id, migrate=True if self.deploy_info else False,
cb=CalvinCB(self.cb, deployer=self))
class Analyzer(object):
"""
Process an cs_info dictionary (output from calvin parser) to
produce a running calvin application.
"""
# FIXME: Provide additional checks making use of
# - actor_def.inport_names and actor_def.outport_names
# - make use of arg_type (STRING, NUMBER, etc.)
# - analyze the actions wrt port use and token consumption:
# for f in actor_def.action_priority:
# print f.__name__, [x.cell_contents for x in f.__closure__]
#
def __init__(self, cs_info, verify=True):
super(Analyzer, self).__init__()
self.cs_info = cs_info
self.local_components = cs_info['components'] if 'components' in cs_info else {}
self.constants = {}
self.app_info = {}
self.connections = {}
self.actors = {}
self.verify = verify
self.actorstore = ActorStore()
self.analyze()
def analyze(self):
"""
Analyze a CalvinScript in canonical format (i.e. as given by the CS parser)
and produce an app_info structure, sutitable for deploying an application.
The app_info contains a dict of actors and their type, arguments, etc., and
a dict of connections between ports where output ports are keys and the
corresponding value is a list of input ports to connect to.
The value for the 'valid' key is True of the script is syntactically correct.
A CalvinScript consists of component definitions, and a _structure_ defining
the actual script. Each component definition consists of statements declaring
ports and arguments, and a _structure_ part, defining what the component does.
"""
self.script_name = os.path.basename(self.cs_info['sourcefile'])
s = self.cs_info['structure']
root_namespace, _ = os.path.splitext(self.script_name)
self.constants = self.cs_info['constants']
argd = {}
valid = True
try:
self.analyze_structure(s, root_namespace, argd)
except Exception as e:
_log.exception(e)
valid = False
self.app_info = {'valid': valid, 'actors': self.actors, 'connections': self.connections}
if self.script_name:
self.app_info['name'] = self.script_name
def debug_info(self, d):
file = 'File "%s"' % self.script_name
line = ', line %d' % d['dbg_line'] if 'dbg_line' in d else ''
return file + line
def lookup_constant(self, identifier):
"""
Return value for constant 'identifier'
Raise an exception if not found
"""
kind, value = self.constants[identifier]
if kind != "IDENTIFIER":
return (kind, value)
return self.lookup_constant(value)
def lookup(self, actor_type):
"""
Search for the definition of 'actor_type'.
Returns a tuple (found, is_primitive, info) where info is either a
class (primitive) or a dictionary with component definition
Search order:
1 - components defined in the current script: self.local_components
2 - primitive actors in the order defined by actor store
3 - components in the order defined by actor store
Steps 2 and 3 are handled by generic lookup in actor store
"""
if actor_type in self.local_components:
compdef = self.local_components[actor_type]
return compdef, False
found, is_actor, info = self.actorstore.lookup(actor_type)
if self.verify and not found:
msg = 'Actor "{}" not found.'.format(actor_type)
raise Exception(msg)
return info, is_actor or not found
def add_connection(self, src_actor_port, dst_actor_port):
if type(dst_actor_port) is list:
self.connections.setdefault(src_actor_port, []).extend(dst_actor_port)
else:
self.connections.setdefault(src_actor_port, []).append(dst_actor_port)
def expand_literals(self, structure, argd):
# Check for literals on inports...
const_count = 1
implicit = [c for c in structure['connections'] if not c['src']]
for c in implicit:
kind, value = c['src_port']
if kind == "IDENTIFIER":
if value in argd:
kind, value = ('VALUE', argd[value])
else:
kind, value = self.lookup_constant(value)
# Replace constant with std.Constant(data=value, n=-1) its outport
name = '_literal_const_' + str(const_count)
const_count += 1
# Replace implicit actor with actual actor ...
structure['actors'][name] = {
'actor_type': 'std.Constant',
'args': {
'data': (kind, value),
'n': ('NUMBER', -1)
},
'dbg_line': c['dbg_line']}
# ... and create a connection from it
c['src'] = name
c['src_port'] = 'token'
def resolve_arguments(self, arguments, argd):
args = {}
for arg_name, (arg_type, arg_value) in arguments.iteritems():
if arg_type == 'IDENTIFIER':
# We are looking for the value of a variable whose name is in value
variable_name = arg_value
if variable_name in argd:
arg_value = argd[variable_name]
else:
_, arg_value = self.lookup_constant(variable_name)
args[arg_name] = arg_value
return args
def create_connection(self, c, namespace, in_mappings, out_mappings):
# export_mapping = {'in':{}, 'out':{}}
# Get the full port name.
# If src/dst is ".", the full port name is component port name at caller level
src_actor_port = full_port_name(namespace, c['src'], c['src_port'])
dst_actor_port = full_port_name(namespace, c['dst'], c['dst_port'])
# resolve any references to components first
# N.B. if there is a match for src_actor_port the result is a list:
dst_actor_port = in_mappings.get(dst_actor_port, dst_actor_port)
src_actor_port = out_mappings.get(src_actor_port, src_actor_port)
# Add connections if possible, or export a port mapping for calling level
if c['src'] != '.' and c['dst'] != '.':
self.add_connection(src_actor_port, dst_actor_port)
export_mapping = {}, {}
elif c['dst'] != '.':
# Add mapping from component inport to internal actors/components
if type(dst_actor_port) is not list:
dst_actor_port = [dst_actor_port]
export_mapping = {src_actor_port: dst_actor_port}, {}
else:
# Add mapping from internal actor/component to component outport
export_mapping = {}, {dst_actor_port: src_actor_port}
return export_mapping
def analyze_structure(self, structure, namespace, argd):
"""
Analyze a (sub) structure and resolve actor names, arguments, and connections.
Parameter argd is a dict with arguments for the structure
Returns a dict with port mappings corresponding to the externally visible ports
of the structure, i.e. the ports of a component.
"""
# Check for literals on inports...
self.expand_literals(structure, argd)
in_mappings = {}
out_mappings = {}
for actor_name, actor_def in structure['actors'].iteritems():
# Look up actor
info, is_actor = self.lookup(actor_def['actor_type'])
# Resolve arguments
args = self.resolve_arguments(actor_def['args'], argd)
qualified_name = namespace + ':' + actor_name
if is_actor:
# Create the actor signature to be able to look it up in the GlobalStore if neccessary
signature_desc = {'is_primitive': True,
'actor_type': actor_def['actor_type'],
'inports': [],
'outports': []}
for c in structure['connections']:
if actor_name == c['src'] and c['src_port'] not in signature_desc['outports']:
signature_desc['outports'].append(c['src_port'])
elif actor_name == c['dst'] and c['dst_port'] not in signature_desc['inports']:
signature_desc['inports'].append(c['dst_port'])
signature = GlobalStore.actor_signature(signature_desc)
# Add actor and its arguments to the list of actor instances
self.actors[qualified_name] = {'actor_type': actor_def['actor_type'], 'args': args,
'signature': signature, 'signature_desc': signature_desc}
else:
# Recurse into components
# qualified_name constitutes a namespace here
comp_in_mapping, comp_out_mapping = self.analyze_structure(info['structure'], qualified_name, args)
in_mappings.update(comp_in_mapping)
out_mappings.update(comp_out_mapping)
export_in_mappings = {}
export_out_mappings = {}
for c in structure['connections']:
in_mapping, out_mapping = self.create_connection(c, namespace, in_mappings, out_mappings)
for p in in_mapping:
export_in_mappings.setdefault(p, []).extend(in_mapping[p])
export_out_mappings.update(out_mapping)
return export_in_mappings, export_out_mappings
def setup_class(self):
self.ms = ActorStore()
pass
def setup_class(self):
self.ms = ActorStore()
pass
def __init__(self):
self.store = ActorStore()
self.actors = {}
self.illegal_actors = {}
self.components = {}