def handle_in_check(self,lhs,rhs,active_event): ''' Call has form: lhs in rhs rhs can have three basic types: it can be a list, a map, or a string. That means that it can either be a WaldoMapVariable, a WaldoListVariable, a WaldoStringVariable, or a Python string. Instead of using static type inference at compile time to determine, for sake of development, just doing dynamic check to determine which type it is and do the in processing here. FIXME: it is faster to do the static checks with type inference, etc. at compile time rather than at run time. ''' lhs_val = self.get_val_if_waldo(lhs,active_event) # handles Python string case if util.is_string(rhs): return lhs_val in rhs elif is_non_ext_text_var(rhs): return lhs_val in rhs.get_val(active_event) elif is_non_ext_map_var(rhs): return rhs.get_val(active_event).contains_key_called(active_event,lhs_val) elif is_non_ext_list_var(rhs): return rhs.get_val(active_event).contains_val_called(active_event,lhs_val) util.logger_assert( 'Error when calling in: unknown right hand side of expression')
def __init__( self,host_uuid,peered=False,init_val=None): #### DEBUG if peered: util.logger_assert( 'Function variables may not be peered') #### END DEBUG self.MULTI_THREADED_CONSTRUCTOR = LockedFunctionVariable self.SINGLE_THREADED_CONSTRUCTOR = SingleThreadedLockedFunctionVariable def _default_helper_func(*args,**kwargs): pass if init_val == None: init_val = _default_helper_func super(SingleThreadedLockedFunctionVariable,self).__init__(host_uuid,peered,init_val) # {Array} --- Each element is an int. When making a call to a # function object, the function object takes in arguments. # For non-externals, we de-waldoify these arguments. However, # for external arguments, we do not. If an argument is # supposed to be an external, then we just pass it through # directly self.ext_args_array = None
def complete_commit(self,active_event): ''' Both readers and writers can complete commits. If it's a reader, we do not update the internal value of the object. If it's a writer, we do. In either case, we remove the event from holding a lock and check if any other events can be scheduled. ''' self._lock() if ((self.write_lock_holder is not None) and (active_event.uuid == self.write_lock_holder.event.uuid)): self.val.write(self.dirty_val.val) self.write_lock_holder = None self.read_lock_holders = {} else: val = self.read_lock_holders.pop(active_event.uuid,None) #### DEBUG if val is None: util.logger_assert( 'Should not be completing a commit on a missing event') #### END DEBUG self._unlock() # FIXME: may want to actually check whether the change could # have caused another read/write to be scheduled. self.try_next()
def update_event_priority(self,uuid,new_priority): ''' Called when an event with uuid "uuid" is promoted to boosted with priority "priority" ''' util.logger_assert( 'update_event_priority is pure virtual in WaldoLockedObj')
def put_exception(self,error,message_listening_queues_map): ''' Places the appropriate call result in the event complete queue to indicate to the endpoint that an error has occured and the event must be handled. ''' util.logger_assert('put_exception is pure virtual in EventParent')
def serializable_var_tuple_for_network( self,parent_delta,var_name,invalid_listener,force): ''' @see waldoLockedObj.WaldoLockedObj ''' util.logger_assert( 'Serializable var tuple for network is pure virtual ' + 'in SingleThreadedObj.')
def __init__(self,filename,name,host_uuid,peered=False,init_val=None): self.filename = filename if peered: util.logger_assert('Cannot peer a file') WaldoTextVariable.__init__(self,name,host_uuid,False,init_val) self.flush_file(self.val)
def return_internal_val_from_container(self): ''' @returns {bool} --- True if when call get_val_from_key on a container should call get_val on it. False otherwise. ''' util.logger_assert( 'return_internal_val_from_container is pure virtual ' 'in WaldoLockedObj.')
def __init__(self,host_uuid,peered=False,init_val=None): #### DEBUG if peered: util.logger_assert('Cannot have peered endpoint variable') #### END DEBUG self.MULTI_THREADED_CONSTRUCTOR = LockedEndpointVariable self.SINGLE_THREADED_CONSTRUCTOR = SingleThreadedLockedEndpointVariable super(SingleThreadedLockedEndpointVariable,self).__init__(host_uuid,peered,init_val)
def get_and_reset_has_been_written_since_last_msg(self,active_event): ''' @returns {bool} --- True if the object has been written to since we sent the last message. False otherwise. (Including if event has been preempted.) ''' util.logger_assert( 'get_and_reset_has_been_written_since_last_msg is ' + 'pure virtual in WaldoLockedObj')
def get_dirty_wrapped_val(self,active_event): ''' When serializing data to send to other side for peered variables, need to get deltas across lifetime of variable, this method returns a data wrapper that can be used to get those deltas. ''' util.logger_assert( 'Have not determined how to serialize multithreaded peered data.')
def __init__(self,host_uuid,peered=False,init_val=None): if not isinstance(init_val,dict): util.logger_assert( 'User structs must always have init_vals. ' 'Otherwise, not initializing struct data') else: init_val = LockedInternalStructVariable( ensure_locked_obj,host_uuid,peered,init_val) self.MULTI_THREADED_CONSTRUCTOR = LockedStructVariable self.SINGLE_THREADED_CONSTRUCTOR = SingleThreadedLockedStructVariable super(LockedStructVariable,self).__init__(host_uuid,peered,init_val)
def get_write_key_incorporate_deltas(self,container_written_action): if container_written_action.HasField('write_key_text'): index_to_write_to = container_written_action.write_key_text elif container_written_action.HasField('write_key_num'): index_to_write_to = container_written_action.write_key_num elif container_written_action.HasField('write_key_tf'): index_to_write_to = container_written_action.write_key_tf #### DEBUG else: util.logger_assert('Unknown map index') #### END DEBUG return index_to_write_to
def get_delete_key_incorporate_deltas(self,container_deleted_action): if container_deleted_action.HasField('deleted_key_text'): index_to_del_from = container_deleted_action.deleted_key_text elif container_deleted_action.HasField('deleted_key_num'): index_to_del_from = container_deleted_action.deleted_key_num elif container_deleted_action.HasField('deleted_key_tf'): index_to_del_from = container_deleted_action.deleted_key_tf #### DEBUG else: util.logger_assert('Error in delete: unknown key type.') #### END DEBUG return index_to_del_from
def add_all_data_to_delta_list( self,delta_to_add_to,current_internal_val,action_event,for_map): ''' Run through entire list. Create an add action for each element. @param {bool} for_map --- True if performing operations for map. false if performing for list. ''' if for_map: to_iter_over = current_internal_val.keys() else: to_iter_over = range(0,len(current_internal_val)) for key in to_iter_over: if for_map: action = delta_to_add_to.map_actions.add() else: action = delta_to_add_to.list_actions.add() action.container_action = VarStoreDeltas.ContainerAction.ADD_KEY add_action = action.added_key add_action.parent_type = VarStoreDeltas.CONTAINER_ADDED if isinstance(key,numbers.Number): add_action.added_key_num = key elif util.is_string(key): add_action.added_key_text = key else: add_action.added_key_tf = key # now actually add the value to the map list_val = current_internal_val[key] if isinstance(list_val,numbers.Number): add_action.added_what_num = list_val elif util.is_string(list_val): add_action.added_what_text = list_val elif isinstance(list_val,bool): add_action.added_what_tf = list_val elif isinstance(list_val,WaldoLockedObj): list_val.serializable_var_tuple_for_network( add_action,'',action_event,True) #### DEBUG else: util.logger_assert( 'Unkonw type to serialize')
def internal_container_variable_serialize_var_tuple_for_network( locked_container, parent_delta, var_name, active_event, force ): """ @param {Map,List, or Struct Variable} locked_container """ var_data = locked_container.val.val # FIXME: If going to have publicly peered data, need to use # locked_container.dirty_val instead of locked_container.val when # incorporating changes??? .get_dirty_wrapped_val returns # wrapped val that can use for serializing data. dirty_wrapped_val = locked_container.get_dirty_wrapped_val(active_event) sub_element_modified = False if isinstance(var_data, list): list_delta = parent_delta.internal_list_delta list_delta.parent_type = VarStoreDeltas.INTERNAL_LIST_CONTAINER if force: dirty_wrapped_val.add_all_data_to_delta_list(list_delta, var_data, active_event, False) sub_element_modified = True else: # if all subelements have not been modified, then we # do not need to keep track of these changes. # wVariable.waldoMap, wVariable.waldoList, or # wVariable.WaldoUserStruct will get rid of it later. sub_element_modified = dirty_wrapped_val.add_to_delta_list(list_delta, var_data, active_event, False) elif isinstance(var_data, dict): map_delta = parent_delta.internal_map_delta map_delta.parent_type = VarStoreDeltas.INTERNAL_MAP_CONTAINER if force: # perform each operation as a write... dirty_wrapped_val.add_all_data_to_delta_list(map_delta, var_data, active_event, True) sub_element_modified = True else: # if all subelements have not been modified, then we # do not need to keep track of these changes. # wVariable.waldoMap, wVariable.waldoList, or # wVariable.WaldoUserStruct will get rid of it later. sub_element_modified = dirty_wrapped_val.add_to_delta_list(map_delta, var_data, active_event, True) else: # creating deltas for cases where internal data are waldo # references.... should have been overridden in # wVariables.py util.logger_assert("Serializing unknown type.") return sub_element_modified
def add_var(self,unique_name,waldo_variable): ''' @param {String} unique_name --- @param {_WaldoVariable} waldo_variable ''' #### DEBUG if self.get_var_if_exists(unique_name) != None: util.logger_assert( 'Already had an entry for variable trying to ' + 'insert into store.') #### END DEBUG self._name_to_var_map[unique_name] = waldo_variable
def ensure_locked_obj(new_val,host_uuid,single_threaded): ''' @param {Anything} new_val --- If new_val is a non-Waldo object, convert it to a Waldo object. Otherwise, return it unchanged. This method is used to ensure that each individual entry in a map/list is also protected. @param {bool} single_threaded --- True if the variable should be single threaded. ''' # FIXME: Need to include function object check in ensure locked # obj if isinstance(new_val, WaldoLockedObj): return new_val if single_threaded: if isinstance(new_val, bool): return SingleThreadedLockedTrueFalseVariable(host_uuid,False,new_val) elif isinstance(new_val, numbers.Number): return SingleThreadedLockedNumberVariable(host_uuid,False,new_val) elif util.is_string(new_val): return SingleThreadedLockedTextVariable(host_uuid,False,new_val) elif isinstance(new_val,list): return SingleThreadedLockedListVariable(host_uuid,False,new_val) elif isinstance(new_val,dict): return SingleThreadedLockedMapVariable(host_uuid,False,new_val) elif isinstance(new_val,EndpointBase): return SingleThreadedLockedEndpointVariable(host_uuid,False,new_val) else: util.logger_assert('Unknown object type.') else: if isinstance(new_val, bool): return LockedTrueFalseVariable(host_uuid,False,new_val) elif isinstance(new_val, numbers.Number): return LockedNumberVariable(host_uuid,False,new_val) elif util.is_string(new_val): return LockedTextVariable(host_uuid,False,new_val) elif isinstance(new_val,list): return LockedListVariable(host_uuid,False,new_val) elif isinstance(new_val,dict): return LockedMapVariable(host_uuid,False,new_val) elif isinstance(new_val,EndpointBase): return LockedEndpointVariable(host_uuid,False,new_val) else: util.logger_assert('Unknown object type.')
def try_schedule_write_waiting_event(self,waiting_event): ''' CALLED FROM WITHIN LOCK HOLDER Gets called when an event that had not been holding a write lock tries to begin holding a write lock. Three things must happen. 1) Check that the event that is trying to assume the write lock has a higher uuid than any other event that is currently holding a lock (read or read/write). 2) If 1 succeeded, then try to backout all events that currently hold locks. (May not be able to if an event is in the midst of a commit.) If can, roll back changes to events that currently hold locks. 3) If 1 and 2 succeeded, then updates self.write_lock_holder and self.read_lock_holders. Also, unjams waiting_event's queue. @param {Waiting Event object} --- Should be @returns {bool} --- True if could successfully schedule the waiting write. False otherwise. ''' #### DEBUG if not waiting_event.is_write(): util.logger_assert( 'Should only pass writes into try_schedule_write_waiting_event') #### END DEBUG # Stage 1 from above if self.is_gte_than_lock_holding_events(waiting_event.cached_priority): # Stage 2 from above if self.test_and_backout_all(waiting_event.event.uuid): # Stage 3 from above # actually update the read/write lock holders self.read_lock_holders[waiting_event.event.uuid] = ( EventCachedPriorityObj(waiting_event.event,waiting_event.cached_priority)) self.write_lock_holder = ( EventCachedPriorityObj(waiting_event.event,waiting_event.cached_priority)) waiting_event.unwait(self) return True return False
def call_func_obj( self,active_event,func_obj,*args): ''' @param {wVariable.WaldoFunctionVariable} func_obj --- The wrapped function that we are calling. @param {*args} --- The actual arguments that get passed to the function. ''' # {list} external_arg_list --- Each element is a number. # If a number is in this list, then that means that the # corresponding argument to func_obj is external and therefore # should not be de_waldo-ified. If an argument does not have # its corresponding index in the array, then dewaldo-ify it. external_arg_list = func_obj.ext_args_array if external_arg_list == None: util.logger_assert( 'No external arg array for function object') call_arg_list = [] for counter in range(0,len(args)): to_append = args[counter] if counter not in external_arg_list: to_append = self.de_waldoify(to_append,active_event) call_arg_list.append(to_append) internal_func = func_obj.get_val(active_event) returned_val = internal_func( active_event.event_parent.local_endpoint,*call_arg_list) if isinstance(returned_val,list): return SingleThreadedLockedListVariable( active_event.event_parent.local_endpoint._host_uuid, False, # not peered returned_val# used as initial value ) elif isinstance(returned_val,dict): return SingleThreadedLockedMapVariable( active_event.event_parent.local_endpoint._host_uuid, False, # not peered returned_val# used as initial value ) return returned_val
def create_struct_delta(parent_delta): is_var_store = False if parent_delta.parent_type == VarStoreDeltas.VAR_STORE_DELTA: is_var_store = True struct_delta = parent_delta.struct_deltas.add() elif parent_delta.parent_type == VarStoreDeltas.CONTAINER_WRITTEN: struct_delta = parent_delta.what_written_struct elif parent_delta.parent_type == VarStoreDeltas.CONTAINER_ADDED: struct_delta = parent_delta.added_what_struct elif parent_delta.parent_type == VarStoreDeltas.SUB_ELEMENT_ACTION: struct_delta = parent_delta.struct_delta else: util.logger_assert("Unexpected parent container type when serializing struct") struct_delta.parent_type = VarStoreDeltas.STRUCT_CONTAINER return struct_delta, is_var_store
def create_map_delta(parent_delta): is_var_store = False if parent_delta.parent_type == VarStoreDeltas.VAR_STORE_DELTA: is_var_store = True map_delta = parent_delta.map_deltas.add() elif parent_delta.parent_type == VarStoreDeltas.CONTAINER_WRITTEN: map_delta = parent_delta.what_written_map elif parent_delta.parent_type == VarStoreDeltas.CONTAINER_ADDED: map_delta = parent_delta.added_what_map elif parent_delta.parent_type == VarStoreDeltas.SUB_ELEMENT_ACTION: map_delta = parent_delta.map_delta else: util.logger_assert("Unexpected parent container type when serializing map") map_delta.parent_type = VarStoreDeltas.MAP_CONTAINER return map_delta, is_var_store
def container_serializable_var_tuple_for_network(container_obj, parent_delta, var_name, active_event, force, var_type): """ @see waldoReferenceBase.serializable_var_tuple_for_network @param {VAR_TYPE} var_type --- One of the enumerated var types """ # reset has been written to has_been_written_since_last_msg = container_obj.get_and_reset_has_been_written_since_last_msg(active_event) if var_type is FOR_MAP_CONTAINER_SERIALIZABLE: container_delta, is_var_store = create_map_delta(parent_delta) elif var_type is FOR_LIST_CONTAINER_SERIALIZABLE: container_delta, is_var_store = create_list_delta(parent_delta) elif var_type is FOR_STRUCT_CONTAINER_SERIALIZABLE: container_delta, is_var_store = create_struct_delta(parent_delta) else: util.logger_assert("Unknown var type when serializing") container_delta.var_name = var_name container_delta.has_been_written = has_been_written_since_last_msg internal_has_been_written = internal_container_variable_serialize_var_tuple_for_network( container_obj, container_delta, var_name, active_event, # must force the write when we have written a new value over list force or has_been_written_since_last_msg, ) # FIXME: check to ensure that second part of condition will # still hide elements that do not change if (not internal_has_been_written) and is_var_store and (not has_been_written_since_last_msg): # remove the newly added map delta because there were no # changes that it encoded if var_type is FOR_MAP_CONTAINER_SERIALIZABLE: del parent_delta.map_deltas[-1] elif var_type is FOR_LIST_CONTAINER_SERIALIZABLE: del parent_delta.list_deltas[-1] elif var_type is FOR_STRUCT_CONTAINER_SERIALIZABLE: del parent_delta.struct_deltas[-1] else: util.logger_assert("Unknown var type when serializing") return internal_has_been_written or has_been_written_since_last_msg or force
def get_for_iter(self,to_iter_over,active_event): ''' When call for loop on Waldo variables, need to get item to iterate over ''' if (isinstance(to_iter_over,dict) or isinstance(to_iter_over,list) or util.is_string(to_iter_over)): return iter(to_iter_over) if is_non_ext_text_var(to_iter_over): return iter(to_iter_over.get_val(active_event)) if is_non_ext_map_var(to_iter_over): return iter(to_iter_over.get_val(active_event).get_keys(active_event)) if is_non_ext_list_var(to_iter_over): # FIXME: This is an inefficient way of reading all values # over list. to_return = [] internal_val = to_iter_over.get_val(active_event) for i in range(0, internal_val.get_len(active_event)): to_append = internal_val.get_val_on_key(active_event,i) # The reason that we do this here is that in a for # loop, the operation we perform in the compiled code # immediately following the actual for header is # assign to another Waldo variable the variable being # held by to_append. (We do this through a set_val # call. set_val must take in InternalMaps or # InternalLists. Therefore, get_val on the # WaldoList/WaldoMap first. Note this is unnecessary # for all other for iterations because none of the # others possibly return a WaldoObject to iterate over. if (is_non_ext_list_var(to_append) or is_non_ext_map_var(to_append)): to_append = to_append.get_val(active_event) to_return.append(to_append) return iter(to_return) util.logger_assert( 'Calling get_for_iter on an object that does not support iteration')
def value_variable_serializable_var_tuple_for_network( value_variable,parent_delta,var_name,active_event, force): ''' @see waldoLockedObj.serializable_var_tuple_for_network ''' var_data = value_variable.get_val(active_event) has_been_written_since_last_msg = value_variable.get_and_reset_has_been_written_since_last_msg(active_event) if (not force) and (not has_been_written_since_last_msg): # nothing to do because this value has not been # written. NOTE: for list/dict types, must actually # go through to ensure no subelements were written. return False # check if this is a python value type. if it is, append it # to delta. if not value_variable_py_val_serialize( value_variable,parent_delta,var_data,var_name): util.logger_assert('Should only use python values when serializing') return True
def serializable_var_tuple_for_network( self,parent_delta,var_name,invalid_listener,force): ''' The runtime automatically synchronizes data between both endpoints. When one side has updated a peered variable, the other side needs to attempt to apply those changes before doing further work. This method grabs the val and version object of the dirty element associated with invalid_listener. Using these data, plus var_name, it constructs a named tuple for serialization. (@see util._generate_serialization_named_tuple) Note: if the val of this object is another Reference object, then we recursively keep generating named tuples and embed them in the one we return. Note: we only serialize peered data. No other data gets sent over the network; therefore, it should not be serialized. @param {*Delta or VarStoreDeltas} parent_delta --- Append any message that we create here to this message. @param {String} var_name --- Both sides of the connection need to agree on a common name for the variable being serialized. This is to ensure that when the data are received by the other side we know which variable to put them into. This value is only really necessary for the outermost wrapping of the named type tuple, but we pass it through anyways. @param {bool} force --- True if regardless of whether modified or not we should serialize. False otherwise. (We migth want to force for instance the first time we send sequence data.) @returns {bool} --- True if some subelement was modified, False otherwise. ''' util.logger_assert( 'Serializable var tuple for network is pure virtual ' + 'in waldoLockedObj.')
def handle_len(self,what_calling_len_on, active_event): ''' Can support python lists, dicts, strings, or waldo lists, waldo maps, waldo texts. @returns {int} ''' if (isinstance(what_calling_len_on, dict) or isinstance(what_calling_len_on, list) or util.is_string(what_calling_len_on)): return len(what_calling_len_on) if is_non_ext_text_var(what_calling_len_on): return len(what_calling_len_on.get_val(active_event)) if (is_non_ext_list_var(what_calling_len_on) or is_non_ext_map_var(what_calling_len_on)): return what_calling_len_on.get_val(active_event).get_len(active_event) util.logger_assert( 'Calling len on an object that does not support the function')
def func_turn_into_waldo_var( self,val,force_copy,active_event, host_uuid,new_peered, ext_args_array,new_multi_threaded): ''' turn_into_waldo_var works for all non-function types. function-types require additional information (which arguments are and are not external) to populate their ext_args_array. This is passed in in this function. ''' if is_non_ext_func_var(val): if force_copy: # means that it was a WaldoVariable: just call its copy # method return val.copy(active_event,new_peered,new_multi_threaded) # otherwise, just return val return val elif hasattr(val,'__call__'): # python function pass #### DEBUG else: util.logger_assert( 'incorrect type passed into func_turn_into_waldo_var') #### END DEBUG if new_multi_threaded: waldo_func = LockedFunctionVariable( host_uuid, new_peered, val).set_external_args_array(ext_args_array) else: waldo_func = SingleThreadedLockedFunctionVariable( host_uuid, new_peered, val).set_external_args_array(ext_args_array) return waldo_func
def handle_non_first_sequence_msg_from_partner( self,msg,name_of_block_to_exec_next): ''' ASSUMES ALREADY WITHIN LOCK @param {PartnerMessageRequestSequenceBlock.proto} msg --- @param {string or None} name_of_block_to_exec_next --- the name of the sequence block to execute next. None if nothing to execute next (ie, last sequence message). ''' #### DEBUG if msg.reply_to_uuid.data not in self.message_listening_queues_map: util.logger_assert( 'Error: partner response message responding to ' + 'unknown _ActiveEvent message.' + str(msg.reply_to_uuid.data) + ' ' + str(list(self.message_listening_queues_map.keys())) ) #### END DEBUG # unblock waiting listening queue. self.message_listening_queues_map[msg.reply_to_uuid.data].put( _SequenceMessageCallResult( msg.reply_with_uuid.data, name_of_block_to_exec_next, # as soon as read from the listening message # queue, populate sequence local data from context # using sequence_local_var_store_deltas. # FIXME: eventaully, want to move away from # pickle-ing here. msg.sequence_local_var_store_deltas)) # no need holding onto queue waiting on a message response. del self.message_listening_queues_map[msg.reply_to_uuid.data]
def de_waldoify(self,active_event): util.logger_assert( 'de_waldoify is pure virutal in WaldoLockedObj')