def _new_actor(self, actor_type, actor_id=None, credentials=None): """Return a 'bare' actor of actor_type, raises an exception on failure.""" if credentials is not None: sec = Security(self.node) sec.set_subject(credentials) sec.authenticate_subject() else: sec = None (found, is_primitive, class_) = ActorStore(security=sec).lookup(actor_type) if not found: # Here assume a primitive actor, now become shadow actor _log.analyze(self.node.id, "+ NOT FOUND CREATE SHADOW ACTOR", {'class': class_}) found = True is_primitive = True class_ = ShadowActor if not found or not is_primitive: _log.error("Requested actor %s is not available" % (actor_type)) raise Exception("ERROR_NOT_FOUND") try: # Create a 'bare' instance of the actor a = class_(actor_type, actor_id=actor_id) except Exception as e: _log.exception("") _log.error("The actor %s(%s) can't be instantiated." % (actor_type, class_.__init__)) raise(e) try: a.set_credentials(credentials, security=sec) a._calvinsys = self.node.calvinsys() a.check_requirements() except Exception as e: _log.exception("Catched new from state") _log.analyze(self.node.id, "+ FAILED REQS CREATE SHADOW ACTOR", {'class': class_}) a = ShadowActor(actor_type, actor_id=actor_id) a.set_credentials(credentials, security=sec) a._calvinsys = self.node.calvinsys() return a
def compile(source_text, filename='', content=None, credentials=None, verify=True, node=None): # Steps taken: # 1) Verify signature when credentials supplied # 2) parser .calvin file -> IR. May produce syntax errors/warnings # 3) checker IR -> IR. May produce syntax errors/warnings # 4) analyzer IR -> app. Should not fail. Sets 'valid' property of IR to True/False deployable = {'valid': False, 'actors': {}, 'connections': {}} errors = [] #TODO: fill in something meaningful warnings = [] if credentials: _log.debug("Check credentials...") sec = Security(node) sec.set_subject(credentials) if not sec.authenticate_subject(): _log.error("Check credentials...failed authentication") # This error reason is detected in calvin control and gives proper REST response errors.append({'reason': "401: UNAUTHORIZED", 'line': 0, 'col': 0}) return deployable, errors, warnings if (not sec.verify_signature_content(content, "application") or not sec.check_security_policy()): # Verification not OK if sign or cert not OK or if the signer is denied by security policies print "\n IN DEPLOYER\n " _log.error("Check credentials...failed application verification") # This error reason is detected in calvin control and gives proper REST response errors.append({'reason': "401: UNAUTHORIZED", 'line': None, 'col': None}) return deployable, errors, warnings _log.debug("Parsing...") ir, errors, warnings = calvin_parser(source_text, filename) _log.debug("Parsed %s, %s, %s" % (ir, errors, warnings)) # If there were errors during parsing no IR will be generated if not errors: c_errors, c_warnings = check(ir, verify=verify) errors.extend(c_errors) warnings.extend(c_warnings) deployable = generate_app_info(ir, verify=verify) if errors: deployable['valid'] = False _log.debug("Compiled %s, %s, %s" % (deployable, errors, warnings)) return deployable, errors, warnings
def compile_script_check_security(source_text, filename, cb, credentials=None, verify=True, node=None): """ Compile a script and return a tuple (deployable, errors, warnings). 'credentials' are optional security credentials(?) 'verify' is deprecated and will be removed 'node' is the runtime performing security check(?) 'cb' is a CalvinCB callback N.B. If callback 'cb' is given, this method calls cb(deployable, errors, warnings) and returns None N.B. If callback 'cb' is given, and method runs to completion, cb is called with additional parameter 'security' (?) """ def _exit_with_error(callback): """Helper method to generate a proper error""" it = IssueTracker() it.add_error("UNAUTHORIZED", info={'status': 401}) callback({}, it) def _handle_authentication_decision(source_text, appname, verify, authentication_decision, security, org_cb, content=None): if not authentication_decision: _log.error("Authentication failed") # This error reason is detected in calvin control and gives proper REST response _exit_with_error(org_cb) verified, signer = security.verify_signature_content( content, "application") if not verified: # Verification not OK if sign or cert not OK. _log.error("Failed application verification") # This error reason is detected in calvin control and gives proper REST response _exit_with_error(org_cb) security.check_security_policy(CalvinCB(_handle_policy_decision, source_text, appname, verify, security=security, org_cb=org_cb), "application", signer=signer) def _handle_policy_decision(source_text, appname, verify, access_decision, org_cb, security=None): if not access_decision: _log.error("Access denied") # This error reason is detected in calvin control and gives proper REST response _exit_with_error(org_cb) deployable, issutracker = compile_script(source_text, appname) org_cb(deployable, issutracker, security=security) # # Actual code for compile_script # appname = appname_from_filename(filename) # FIXME: if node is None we bypass security even if enabled. Is that the intention? if node is not None and security_enabled(): if credentials: content = Security.verify_signature_get_files(filename, skip_file=True) # content is ALWAYS a dict if skip_file is True content['file'] = source_text else: content = None # FIXME: If cb is None, we will return from this method with None instead of a tuple, failing silently sec = Security(node) sec.authenticate_subject(credentials, callback=CalvinCB( _handle_authentication_decision, source_text, appname, verify, security=sec, org_cb=cb, content=content)) return # # We get here if node is None, or security is disabled # # This used to be # _handle_policy_decision(source_text, filename, verify, access_decision=True, security=None, org_cb=cb) # but since _handle_policy_decision is called with access_decision=True, security=None only compile_script would be called deployable, issuetracker = compile_script(source_text, appname) cb(deployable, issuetracker, security=None)
def compile_script_check_security(source_text, filename, cb, credentials=None, verify=True, node=None): """ Compile a script and return a tuple (deployable, errors, warnings). 'credentials' are optional security credentials(?) 'verify' is deprecated and will be removed 'node' is the runtime performing security check(?) 'cb' is a CalvinCB callback N.B. If callback 'cb' is given, this method calls cb(deployable, errors, warnings) and returns None N.B. If callback 'cb' is given, and method runs to completion, cb is called with additional parameter 'security' (?) """ def _exit_with_error(callback): """Helper method to generate a proper error""" it = IssueTracker() it.add_error("UNAUTHORIZED", info={'status':401}) callback({}, it) def _handle_authentication_decision(source_text, appname, verify, authentication_decision, security, org_cb, content=None): if not authentication_decision: _log.error("Authentication failed") # This error reason is detected in calvin control and gives proper REST response _exit_with_error(org_cb) verified, signer = security.verify_signature_content(content, "application") if not verified: # Verification not OK if sign or cert not OK. _log.error("Failed application verification") # This error reason is detected in calvin control and gives proper REST response _exit_with_error(org_cb) security.check_security_policy( CalvinCB(_handle_policy_decision, source_text, appname, verify, security=security, org_cb=org_cb), "application", signer=signer ) def _handle_policy_decision(source_text, appname, verify, access_decision, org_cb, security=None): if not access_decision: _log.error("Access denied") # This error reason is detected in calvin control and gives proper REST response _exit_with_error(org_cb) deployable, issutracker = compile_script(source_text, appname) org_cb(deployable, issutracker, security=security) # # Actual code for compile_script # appname = appname_from_filename(filename) # FIXME: if node is None we bypass security even if enabled. Is that the intention? if node is not None and security_enabled(): if credentials: content = Security.verify_signature_get_files(filename, skip_file=True) # content is ALWAYS a dict if skip_file is True content['file'] = source_text else: content = None # FIXME: If cb is None, we will return from this method with None instead of a tuple, failing silently sec = Security(node) sec.authenticate_subject( credentials, callback=CalvinCB(_handle_authentication_decision, source_text, appname, verify, security=sec, org_cb=cb, content=content) ) return # # We get here if node is None, or security is disabled # # This used to be # _handle_policy_decision(source_text, filename, verify, access_decision=True, security=None, org_cb=cb) # but since _handle_policy_decision is called with access_decision=True, security=None only compile_script would be called deployable, issuetracker = compile_script(source_text, appname) cb(deployable, issuetracker, security=None)
class Actor(object): """ Base class for all actors Need a name supplied. Subclasses need to declare the parameter calvinsys if they want access to system interface on the node, this parameter will be supplied by the node and not by user """ # Class variable controls action priority order action_priority = tuple() # Internal state (status) class FSM(object): def __init__(self, states, initial, transitions, hooks=None, allow_invalid_transitions=True, disable_transition_checks=False, disable_state_checks=False): self.states = states self._state = initial self.transitions = transitions self.hooks = hooks or {} self.allow_invalid_transitions = allow_invalid_transitions self.disable_transition_checks = disable_transition_checks # disable_state_checks is used in the verify_status decorator self.disable_state_checks = disable_state_checks def state(self): return self._state def transition_to(self, new_state): if new_state in self.transitions[self._state] or self.disable_transition_checks: hook = self.hooks.get((self._state, new_state), None) if hook: hook() self._state = new_state else: msg = "Invalid transition %s -> %s" % (self, self.printable(new_state)) if self.allow_invalid_transitions: _log.warning("ALLOWING " + msg) self._state = new_state else: raise Exception(msg) def printable(self, state): return self.states.reverse_mapping[state] def __str__(self): return self.printable(self._state) STATUS = enum('LOADED', 'READY', 'PENDING', 'ENABLED') VALID_TRANSITIONS = { STATUS.LOADED : [STATUS.READY], STATUS.READY : [STATUS.PENDING, STATUS.ENABLED], STATUS.PENDING: [STATUS.READY, STATUS.PENDING, STATUS.ENABLED], STATUS.ENABLED: [STATUS.READY, STATUS.PENDING] } test_args = () test_kwargs = {} # What are the arguments, really? def __init__(self, actor_type, name='', allow_invalid_transitions=True, disable_transition_checks=False, disable_state_checks=False, actor_id=None): """Should _not_ be overridden in subclasses.""" super(Actor, self).__init__() self._type = actor_type self.name = name # optional: human_readable_name self.id = actor_id or calvinuuid.uuid("ACTOR") _log.debug("New actor id: %s, supplied actor id %s" % (self.id, actor_id)) self._deployment_requirements = [] self._signature = None self._component_members = set([self.id]) # We are only part of component if this is extended self._managed = set(('id', 'name', '_deployment_requirements', '_signature', 'credentials')) self._calvinsys = None self._using = {} self.control = calvincontrol.get_calvincontrol() self.metering = metering.get_metering() self._migrating_to = None # During migration while on the previous node set to the next node id self._last_time_warning = 0.0 self.credentials = None self.authorization_plugins = None self.inports = {p: actorport.InPort(p, self) for p in self.inport_names} self.outports = {p: actorport.OutPort(p, self) for p in self.outport_names} hooks = { (Actor.STATUS.PENDING, Actor.STATUS.ENABLED): self.will_start, (Actor.STATUS.ENABLED, Actor.STATUS.PENDING): self.will_stop, } self.fsm = Actor.FSM(Actor.STATUS, Actor.STATUS.LOADED, Actor.VALID_TRANSITIONS, hooks, allow_invalid_transitions=allow_invalid_transitions, disable_transition_checks=disable_transition_checks, disable_state_checks=disable_state_checks) self.metering.add_actor_info(self) def set_credentials(self, credentials, security=None): """ Set the credentials the actor operates under. This will trigger an authentication of the credentials. Optionally an authenticated Security instance can be supplied, to reduce the needed authentication processing. """ _log.debug("actor.py: set_credentials: %s" % credentials) if credentials is None: return self.credentials = credentials if security: self.sec = security else: if self._calvinsys is not None: self.sec = Security(self._calvinsys._node) self.sec.set_subject(self.credentials) self.sec.authenticate_subject() def get_credentials(self): _log.debug("actor.py: get_credentials: %s" % self.credentials) return self.credentials @verify_status([STATUS.LOADED]) def setup_complete(self): self.fsm.transition_to(Actor.STATUS.READY) def init(self): raise Exception("Implementing 'init()' is mandatory.") def will_start(self): """Override in actor subclass if actions need to be taken before starting.""" pass def will_stop(self): """Override in actor subclass if actions need to be taken before stopping.""" pass def will_migrate(self): """Override in actor subclass if actions need to be taken before migrating.""" pass def did_migrate(self): """Override in actor subclass if actions need to be taken after migrating.""" pass def will_end(self): """Override in actor subclass if actions need to be taken before destruction.""" pass @verify_status([STATUS.LOADED]) def check_requirements(self): """Check that all requirements are available and accessible in calvinsys.""" # Check availability of calvinsys subsystems before checking security policies. if hasattr(self, "requires"): for req in self.requires: if not self._calvinsys.has_capability(req): raise Exception("%s requires %s" % (self.id, req)) # Check the runtime and calvinsys execution access rights. # Note: when no credentials are set, no verification is done. if hasattr(self, 'sec'): access_decision = self.sec.check_security_policy(['runtime'] + (self.requires if hasattr(self, "requires") else [])) if isinstance(access_decision, tuple): # Only a tuple if access was granted. No need to check access_decision[0]. self.authorization_plugins = access_decision[1] elif not access_decision: _log.debug("Security policy check for actor failed") raise Exception("Security policy check for actor failed") def __getitem__(self, attr): if attr in self._using: return self._using[attr] raise KeyError(attr) def use(self, requirement, shorthand): self._using[shorthand] = self._calvinsys.use_requirement(self, requirement) def __str__(self): ip = "" for p in self.inports.values(): ip = ip + str(p) op = "" for p in self.outports.values(): op = op + str(p) s = "Actor: '%s' class '%s'\nstatus: %s\ninports: %s\noutports:%s" % ( self.name, self._type, self.fsm, ip, op) return s @verify_status([STATUS.READY]) def set_port_property(self, port_type, port_name, port_property, value): """Change a port property. Currently, setting 'fanout' on output ports is only allowed operation.""" if port_type not in ('in', 'out'): _log.error("Illegal port type '%s' for actor '%s' of type '%s'" % (port_type, self.name, self._type)) return False ports = self.outports if port_type == 'out' else self.inports if port_name not in ports: _log.error("Illegal %sport name '%s' for actor '%s' of type '%s'" % (port_type, port_name, self.name, self._type)) return False port = ports[port_name] if not hasattr(port, port_property): _log.error("Illegal property '%s' for %sport '%s' in actor '%s' of type '%s'" % (port_property, port_type, port_name, self.name, self._type)) return False setattr(port, port_property, value) return True @verify_status([STATUS.READY, STATUS.PENDING]) def did_connect(self, port): """Called when a port is connected, checks actor is fully connected.""" # If we happen to by in READY, go to PENDING if self.fsm.state() == Actor.STATUS.READY: self.fsm.transition_to(Actor.STATUS.PENDING) # Three non-patological options: # have inports, have outports, or have in- and outports if self.inports: for p in self.inports.values(): if not p.is_connected(): return if self.outports: for p in self.outports.values(): if not p.is_connected(): return # If we made it here, all ports are connected self.fsm.transition_to(Actor.STATUS.ENABLED) # Actor enabled, inform scheduler self._calvinsys.scheduler_wakeup() @verify_status([STATUS.ENABLED, STATUS.PENDING]) def did_disconnect(self, port): """Called when a port is disconnected, checks actor is fully disconnected.""" # If we happen to by in ENABLED, go to PENDING if self.fsm.state() == Actor.STATUS.ENABLED: self.fsm.transition_to(Actor.STATUS.PENDING) # Three non-patological options: # have inports, have outports, or have in- and outports if self.inports: for p in self.inports.values(): if p.is_connected(): return if self.outports: for p in self.outports.values(): if p.is_connected(): return # If we made it here, all ports are disconnected self.fsm.transition_to(Actor.STATUS.READY) @verify_status([STATUS.ENABLED]) def fire(self): start_time = time.time() total_result = ActionResult(did_fire=False) while True: if not self.check_authorization_decision(): # The authorization decision is not valid anymore. # TODO: try to migrate actor. _log.info("Access denied for actor %s(%s)" % ( self._type, self.id)) return total_result # Re-try action in list order after EVERY firing for action_method in self.__class__.action_priority: action_result = action_method(self) total_result.merge(action_result) # Action firing should fire the first action that can fire, # hence when fired start from the beginning if action_result.did_fire: # FIXME: Make this a hook for the runtime to use, don't # import and use calvin_control or metering in actor self.metering.fired(self.id, action_method.__name__) self.control.log_actor_firing( self.id, action_method.__name__, action_result.tokens_produced, action_result.tokens_consumed, action_result.production) _log.debug("Actor %s(%s) did fire %s -> %s" % ( self._type, self.id, action_method.__name__, str(action_result))) break if not action_result.did_fire: diff = time.time() - start_time if diff > 0.2 and start_time - self._last_time_warning > 120.0: # Every other minute warn if an actor runs for longer than 200 ms self._last_time_warning = start_time _log.warning("%s (%s) actor blocked for %f sec" % (self.name, self._type, diff)) # We reached the end of the list without ANY firing => return return total_result # Redundant as of now, kept as reminder for when rewriting exception handling. raise Exception('Exit from fire should ALWAYS be from previous line.') def enabled(self): return self.fsm.state() == Actor.STATUS.ENABLED # DEPRECATED: Only here for backwards compatibility @verify_status([STATUS.ENABLED]) def enable(self): self.fsm.transition_to(Actor.STATUS.ENABLED) @verify_status([STATUS.READY, STATUS.PENDING, STATUS.LOADED]) # DEPRECATED: Only here for backwards compatibility def disable(self): self.fsm.transition_to(Actor.STATUS.PENDING) @verify_status([STATUS.LOADED, STATUS.READY, STATUS.PENDING]) def state(self): state = {} # Manual state handling # Not available until after __init__ completes state['_managed'] = list(self._managed) state['inports'] = {port: self.inports[port]._state() for port in self.inports} state['outports'] = { port: self.outports[port]._state() for port in self.outports} state['_component_members'] = list(self._component_members) # Managed state handling for key in self._managed: obj = self.__dict__[key] if _implements_state(obj): state[key] = obj.state() else: state[key] = obj return state @verify_status([STATUS.LOADED, STATUS.READY, STATUS.PENDING]) def _set_state(self, state): # Managed state handling # Update since if previously a shadow actor the init has been called first # which potentially have altered the managed attributes set compared # with the recorded state self._managed.update(set(state['_managed'])) for key in state['_managed']: if key not in self.__dict__: self.__dict__[key] = state.pop(key) else: obj = self.__dict__[key] if _implements_state(obj): obj.set_state(state.pop(key)) else: self.__dict__[key] = state.pop(key) # Manual state handling for port in state['inports']: # Uses setdefault to support shadow actor self.inports.setdefault(port, actorport.InPort(port, self))._set_state(state['inports'][port]) for port in state['outports']: # Uses setdefault to support shadow actor self.outports.setdefault(port, actorport.OutPort(port, self))._set_state(state['outports'][port]) self._component_members= set(state['_component_members']) # TODO verify status should only allow reading connections when and after being fully connected (enabled) @verify_status([STATUS.ENABLED, STATUS.READY, STATUS.PENDING]) def connections(self, node_id): c = {'actor_id': self.id, 'actor_name': self.name} inports = {} for port in self.inports.values(): peer = port.get_peer() if peer[0] == 'local': peer = (node_id, peer[1]) inports[port.id] = peer c['inports'] = inports outports = {} for port in self.outports.values(): peers = [ (node_id, p[1]) if p[0] == 'local' else p for p in port.get_peers()] outports[port.id] = peers c['outports'] = outports return c def serialize(self): return self.state() def deserialize(self, data): self._set_state(data) def exception_handler(self, action, args, context): """Defult handler when encountering ExceptionTokens""" _log.error("ExceptionToken encountered\n name: %s\n type: %s\n action: %s\n args: %s\n context: %s\n" % (self.name, self._type, action.__name__, args, context)) raise Exception("ExceptionToken NOT HANDLED") def events(self): return [] def component_add(self, actor_ids): if not isinstance(actor_ids, (set, list, tuple)): actor_ids = [actor_ids] self._component_members.update(actor_ids) def component_remove(self, actor_ids): if not isinstance(actor_ids, (set, list, tuple)): actor_ids = [actor_ids] self._component_members -= set(actor_ids) def part_of_component(self): return len(self._component_members - set([self.id]))>0 def component_members(self): return self._component_members def requirements_add(self, deploy_reqs, extend=False): if extend: self._deployment_requirements.extend(deploy_reqs) else: self._deployment_requirements = deploy_reqs def requirements_get(self): return self._deployment_requirements + ( [{'op': 'actor_reqs_match', 'kwargs': {'requires': self.requires}, 'type': '+'}] if hasattr(self, 'requires') else []) def signature_set(self, signature): if self._signature is None: self._signature = signature def check_authorization_decision(self): """Check if authorization decision is still valid""" if self.authorization_plugins: if any(isinstance(elem, list) for elem in self.authorization_plugins): # If list of lists, True must be found in each list. for plugin_list in self.authorization_plugins: if not self.check_authorization_plugin_list(plugin_list): return False return True else: return self.check_authorization_plugin_list(self.authorization_plugins) return True def check_authorization_plugin_list(self, plugin_list): authorization_results = [] for plugin in plugin_list: try: authorization_results.append(authz_plugins[plugin["id"]].authorization_check(**plugin["attributes"])) except Exception: return False # At least one of the authorization checks for the plugins must return True. return True in authorization_results