def handle_in_check(self,lhs,rhs,active_event):
'''
Call has form:
lhs in rhs
rhs can have three basic types: it can be a list, a map, or a
string. That means that it can either be a WaldoMapVariable,
a WaldoListVariable, a WaldoStringVariable, or a Python
string.
Instead of using static type inference at compile time to
determine, for sake of development, just doing dynamic check
to determine which type it is and do the in processing here.
FIXME: it is faster to do the static checks with type
inference, etc. at compile time rather than at run time.
'''
lhs_val = self.get_val_if_waldo(lhs,active_event)
# handles Python string case
if util.is_string(rhs):
return lhs_val in rhs
elif is_non_ext_text_var(rhs):
return lhs_val in rhs.get_val(active_event)
elif is_non_ext_map_var(rhs):
return rhs.get_val(active_event).contains_key_called(active_event,lhs_val)
elif is_non_ext_list_var(rhs):
return rhs.get_val(active_event).contains_val_called(active_event,lhs_val)
util.logger_assert(
'Error when calling in: unknown right hand side of expression')
def __init__(
self,host_uuid,peered=False,init_val=None):
#### DEBUG
if peered:
util.logger_assert(
'Function variables may not be peered')
#### END DEBUG
self.MULTI_THREADED_CONSTRUCTOR = LockedFunctionVariable
self.SINGLE_THREADED_CONSTRUCTOR = SingleThreadedLockedFunctionVariable
def _default_helper_func(*args,**kwargs):
pass
if init_val == None:
init_val = _default_helper_func
super(SingleThreadedLockedFunctionVariable,self).__init__(host_uuid,peered,init_val)
# {Array} --- Each element is an int. When making a call to a
# function object, the function object takes in arguments.
# For non-externals, we de-waldoify these arguments. However,
# for external arguments, we do not. If an argument is
# supposed to be an external, then we just pass it through
# directly
self.ext_args_array = None
def complete_commit(self,active_event):
'''
Both readers and writers can complete commits. If it's a
reader, we do not update the internal value of the object. If
it's a writer, we do. In either case, we remove the event
from holding a lock and check if any other events can be
scheduled.
'''
self._lock()
if ((self.write_lock_holder is not None) and
(active_event.uuid == self.write_lock_holder.event.uuid)):
self.val.write(self.dirty_val.val)
self.write_lock_holder = None
self.read_lock_holders = {}
else:
val = self.read_lock_holders.pop(active_event.uuid,None)
#### DEBUG
if val is None:
util.logger_assert(
'Should not be completing a commit on a missing event')
#### END DEBUG
self._unlock()
# FIXME: may want to actually check whether the change could
# have caused another read/write to be scheduled.
self.try_next()
def update_event_priority(self,uuid,new_priority):
'''
Called when an event with uuid "uuid" is promoted to boosted
with priority "priority"
'''
util.logger_assert(
'update_event_priority is pure virtual in WaldoLockedObj')
def put_exception(self,error,message_listening_queues_map):
'''
Places the appropriate call result in the event complete queue to
indicate to the endpoint that an error has occured and the event
must be handled.
'''
util.logger_assert('put_exception is pure virtual in EventParent')
def serializable_var_tuple_for_network(
self,parent_delta,var_name,invalid_listener,force):
'''
@see waldoLockedObj.WaldoLockedObj
'''
util.logger_assert(
'Serializable var tuple for network is pure virtual ' +
'in SingleThreadedObj.')
def __init__(self,filename,name,host_uuid,peered=False,init_val=None):
self.filename = filename
if peered:
util.logger_assert('Cannot peer a file')
WaldoTextVariable.__init__(self,name,host_uuid,False,init_val)
self.flush_file(self.val)
def return_internal_val_from_container(self):
'''
@returns {bool} --- True if when call get_val_from_key on a
container should call get_val on it. False otherwise.
'''
util.logger_assert(
'return_internal_val_from_container is pure virtual '
'in WaldoLockedObj.')
def __init__(self,host_uuid,peered=False,init_val=None):
#### DEBUG
if peered:
util.logger_assert('Cannot have peered endpoint variable')
#### END DEBUG
self.MULTI_THREADED_CONSTRUCTOR = LockedEndpointVariable
self.SINGLE_THREADED_CONSTRUCTOR = SingleThreadedLockedEndpointVariable
super(SingleThreadedLockedEndpointVariable,self).__init__(host_uuid,peered,init_val)
def get_and_reset_has_been_written_since_last_msg(self,active_event):
'''
@returns {bool} --- True if the object has been written to
since we sent the last message. False otherwise. (Including
if event has been preempted.)
'''
util.logger_assert(
'get_and_reset_has_been_written_since_last_msg is ' +
'pure virtual in WaldoLockedObj')
def get_dirty_wrapped_val(self,active_event):
'''
When serializing data to send to other side for peered
variables, need to get deltas across lifetime of variable,
this method returns a data wrapper that can be used to get
those deltas.
'''
util.logger_assert(
'Have not determined how to serialize multithreaded peered data.')
def __init__(self,host_uuid,peered=False,init_val=None):
if not isinstance(init_val,dict):
util.logger_assert(
'User structs must always have init_vals. '
'Otherwise, not initializing struct data')
else:
init_val = LockedInternalStructVariable(
ensure_locked_obj,host_uuid,peered,init_val)
self.MULTI_THREADED_CONSTRUCTOR = LockedStructVariable
self.SINGLE_THREADED_CONSTRUCTOR = SingleThreadedLockedStructVariable
super(LockedStructVariable,self).__init__(host_uuid,peered,init_val)
def get_write_key_incorporate_deltas(self,container_written_action):
if container_written_action.HasField('write_key_text'):
index_to_write_to = container_written_action.write_key_text
elif container_written_action.HasField('write_key_num'):
index_to_write_to = container_written_action.write_key_num
elif container_written_action.HasField('write_key_tf'):
index_to_write_to = container_written_action.write_key_tf
#### DEBUG
else:
util.logger_assert('Unknown map index')
#### END DEBUG
return index_to_write_to
def get_delete_key_incorporate_deltas(self,container_deleted_action):
if container_deleted_action.HasField('deleted_key_text'):
index_to_del_from = container_deleted_action.deleted_key_text
elif container_deleted_action.HasField('deleted_key_num'):
index_to_del_from = container_deleted_action.deleted_key_num
elif container_deleted_action.HasField('deleted_key_tf'):
index_to_del_from = container_deleted_action.deleted_key_tf
#### DEBUG
else:
util.logger_assert('Error in delete: unknown key type.')
#### END DEBUG
return index_to_del_from
def add_all_data_to_delta_list(
self,delta_to_add_to,current_internal_val,action_event,for_map):
'''
Run through entire list. Create an add action for each element.
@param {bool} for_map --- True if performing operations for
map. false if performing for list.
'''
if for_map:
to_iter_over = current_internal_val.keys()
else:
to_iter_over = range(0,len(current_internal_val))
for key in to_iter_over:
if for_map:
action = delta_to_add_to.map_actions.add()
else:
action = delta_to_add_to.list_actions.add()
action.container_action = VarStoreDeltas.ContainerAction.ADD_KEY
add_action = action.added_key
add_action.parent_type = VarStoreDeltas.CONTAINER_ADDED
if isinstance(key,numbers.Number):
add_action.added_key_num = key
elif util.is_string(key):
add_action.added_key_text = key
else:
add_action.added_key_tf = key
# now actually add the value to the map
list_val = current_internal_val[key]
if isinstance(list_val,numbers.Number):
add_action.added_what_num = list_val
elif util.is_string(list_val):
add_action.added_what_text = list_val
elif isinstance(list_val,bool):
add_action.added_what_tf = list_val
elif isinstance(list_val,WaldoLockedObj):
list_val.serializable_var_tuple_for_network(
add_action,'',action_event,True)
#### DEBUG
else:
util.logger_assert(
'Unkonw type to serialize')
def internal_container_variable_serialize_var_tuple_for_network(
locked_container, parent_delta, var_name, active_event, force
):
"""
@param {Map,List, or Struct Variable} locked_container
"""
var_data = locked_container.val.val
# FIXME: If going to have publicly peered data, need to use
# locked_container.dirty_val instead of locked_container.val when
# incorporating changes??? .get_dirty_wrapped_val returns
# wrapped val that can use for serializing data.
dirty_wrapped_val = locked_container.get_dirty_wrapped_val(active_event)
sub_element_modified = False
if isinstance(var_data, list):
list_delta = parent_delta.internal_list_delta
list_delta.parent_type = VarStoreDeltas.INTERNAL_LIST_CONTAINER
if force:
dirty_wrapped_val.add_all_data_to_delta_list(list_delta, var_data, active_event, False)
sub_element_modified = True
else:
# if all subelements have not been modified, then we
# do not need to keep track of these changes.
# wVariable.waldoMap, wVariable.waldoList, or
# wVariable.WaldoUserStruct will get rid of it later.
sub_element_modified = dirty_wrapped_val.add_to_delta_list(list_delta, var_data, active_event, False)
elif isinstance(var_data, dict):
map_delta = parent_delta.internal_map_delta
map_delta.parent_type = VarStoreDeltas.INTERNAL_MAP_CONTAINER
if force:
# perform each operation as a write...
dirty_wrapped_val.add_all_data_to_delta_list(map_delta, var_data, active_event, True)
sub_element_modified = True
else:
# if all subelements have not been modified, then we
# do not need to keep track of these changes.
# wVariable.waldoMap, wVariable.waldoList, or
# wVariable.WaldoUserStruct will get rid of it later.
sub_element_modified = dirty_wrapped_val.add_to_delta_list(map_delta, var_data, active_event, True)
else:
# creating deltas for cases where internal data are waldo
# references.... should have been overridden in
# wVariables.py
util.logger_assert("Serializing unknown type.")
return sub_element_modified
def add_var(self,unique_name,waldo_variable):
'''
@param {String} unique_name ---
@param {_WaldoVariable} waldo_variable
'''
#### DEBUG
if self.get_var_if_exists(unique_name) != None:
util.logger_assert(
'Already had an entry for variable trying to ' +
'insert into store.')
#### END DEBUG
self._name_to_var_map[unique_name] = waldo_variable
def ensure_locked_obj(new_val,host_uuid,single_threaded):
'''
@param {Anything} new_val --- If new_val is a non-Waldo object,
convert it to a Waldo object. Otherwise, return it unchanged.
This method is used to ensure that each individual entry in a
map/list is also protected.
@param {bool} single_threaded --- True if the variable should be
single threaded.
'''
# FIXME: Need to include function object check in ensure locked
# obj
if isinstance(new_val, WaldoLockedObj):
return new_val
if single_threaded:
if isinstance(new_val, bool):
return SingleThreadedLockedTrueFalseVariable(host_uuid,False,new_val)
elif isinstance(new_val, numbers.Number):
return SingleThreadedLockedNumberVariable(host_uuid,False,new_val)
elif util.is_string(new_val):
return SingleThreadedLockedTextVariable(host_uuid,False,new_val)
elif isinstance(new_val,list):
return SingleThreadedLockedListVariable(host_uuid,False,new_val)
elif isinstance(new_val,dict):
return SingleThreadedLockedMapVariable(host_uuid,False,new_val)
elif isinstance(new_val,EndpointBase):
return SingleThreadedLockedEndpointVariable(host_uuid,False,new_val)
else:
util.logger_assert('Unknown object type.')
else:
if isinstance(new_val, bool):
return LockedTrueFalseVariable(host_uuid,False,new_val)
elif isinstance(new_val, numbers.Number):
return LockedNumberVariable(host_uuid,False,new_val)
elif util.is_string(new_val):
return LockedTextVariable(host_uuid,False,new_val)
elif isinstance(new_val,list):
return LockedListVariable(host_uuid,False,new_val)
elif isinstance(new_val,dict):
return LockedMapVariable(host_uuid,False,new_val)
elif isinstance(new_val,EndpointBase):
return LockedEndpointVariable(host_uuid,False,new_val)
else:
util.logger_assert('Unknown object type.')
def try_schedule_write_waiting_event(self,waiting_event):
'''
CALLED FROM WITHIN LOCK HOLDER
Gets called when an event that had not been holding a write
lock tries to begin holding a write lock.
Three things must happen.
1) Check that the event that is trying to assume the write
lock has a higher uuid than any other event that is
currently holding a lock (read or read/write).
2) If 1 succeeded, then try to backout all events that
currently hold locks. (May not be able to if an event is
in the midst of a commit.) If can, roll back changes to
events that currently hold locks.
3) If 1 and 2 succeeded, then updates self.write_lock_holder
and self.read_lock_holders. Also, unjams
waiting_event's queue.
@param {Waiting Event object} --- Should be
@returns {bool} --- True if could successfully schedule the
waiting write. False otherwise.
'''
#### DEBUG
if not waiting_event.is_write():
util.logger_assert(
'Should only pass writes into try_schedule_write_waiting_event')
#### END DEBUG
# Stage 1 from above
if self.is_gte_than_lock_holding_events(waiting_event.cached_priority):
# Stage 2 from above
if self.test_and_backout_all(waiting_event.event.uuid):
# Stage 3 from above
# actually update the read/write lock holders
self.read_lock_holders[waiting_event.event.uuid] = (
EventCachedPriorityObj(waiting_event.event,waiting_event.cached_priority))
self.write_lock_holder = (
EventCachedPriorityObj(waiting_event.event,waiting_event.cached_priority))
waiting_event.unwait(self)
return True
return False
def call_func_obj(
self,active_event,func_obj,*args):
'''
@param {wVariable.WaldoFunctionVariable} func_obj --- The
wrapped function that we are calling.
@param {*args} --- The actual arguments that get passed to the
function.
'''
# {list} external_arg_list --- Each element is a number.
# If a number is in this list, then that means that the
# corresponding argument to func_obj is external and therefore
# should not be de_waldo-ified. If an argument does not have
# its corresponding index in the array, then dewaldo-ify it.
external_arg_list = func_obj.ext_args_array
if external_arg_list == None:
util.logger_assert(
'No external arg array for function object')
call_arg_list = []
for counter in range(0,len(args)):
to_append = args[counter]
if counter not in external_arg_list:
to_append = self.de_waldoify(to_append,active_event)
call_arg_list.append(to_append)
internal_func = func_obj.get_val(active_event)
returned_val = internal_func(
active_event.event_parent.local_endpoint,*call_arg_list)
if isinstance(returned_val,list):
return SingleThreadedLockedListVariable(
active_event.event_parent.local_endpoint._host_uuid,
False, # not peered
returned_val# used as initial value
)
elif isinstance(returned_val,dict):
return SingleThreadedLockedMapVariable(
active_event.event_parent.local_endpoint._host_uuid,
False, # not peered
returned_val# used as initial value
)
return returned_val
def create_struct_delta(parent_delta):
is_var_store = False
if parent_delta.parent_type == VarStoreDeltas.VAR_STORE_DELTA:
is_var_store = True
struct_delta = parent_delta.struct_deltas.add()
elif parent_delta.parent_type == VarStoreDeltas.CONTAINER_WRITTEN:
struct_delta = parent_delta.what_written_struct
elif parent_delta.parent_type == VarStoreDeltas.CONTAINER_ADDED:
struct_delta = parent_delta.added_what_struct
elif parent_delta.parent_type == VarStoreDeltas.SUB_ELEMENT_ACTION:
struct_delta = parent_delta.struct_delta
else:
util.logger_assert("Unexpected parent container type when serializing struct")
struct_delta.parent_type = VarStoreDeltas.STRUCT_CONTAINER
return struct_delta, is_var_store
def create_map_delta(parent_delta):
is_var_store = False
if parent_delta.parent_type == VarStoreDeltas.VAR_STORE_DELTA:
is_var_store = True
map_delta = parent_delta.map_deltas.add()
elif parent_delta.parent_type == VarStoreDeltas.CONTAINER_WRITTEN:
map_delta = parent_delta.what_written_map
elif parent_delta.parent_type == VarStoreDeltas.CONTAINER_ADDED:
map_delta = parent_delta.added_what_map
elif parent_delta.parent_type == VarStoreDeltas.SUB_ELEMENT_ACTION:
map_delta = parent_delta.map_delta
else:
util.logger_assert("Unexpected parent container type when serializing map")
map_delta.parent_type = VarStoreDeltas.MAP_CONTAINER
return map_delta, is_var_store
def container_serializable_var_tuple_for_network(container_obj, parent_delta, var_name, active_event, force, var_type):
"""
@see waldoReferenceBase.serializable_var_tuple_for_network
@param {VAR_TYPE} var_type --- One of the enumerated var types
"""
# reset has been written to
has_been_written_since_last_msg = container_obj.get_and_reset_has_been_written_since_last_msg(active_event)
if var_type is FOR_MAP_CONTAINER_SERIALIZABLE:
container_delta, is_var_store = create_map_delta(parent_delta)
elif var_type is FOR_LIST_CONTAINER_SERIALIZABLE:
container_delta, is_var_store = create_list_delta(parent_delta)
elif var_type is FOR_STRUCT_CONTAINER_SERIALIZABLE:
container_delta, is_var_store = create_struct_delta(parent_delta)
else:
util.logger_assert("Unknown var type when serializing")
container_delta.var_name = var_name
container_delta.has_been_written = has_been_written_since_last_msg
internal_has_been_written = internal_container_variable_serialize_var_tuple_for_network(
container_obj,
container_delta,
var_name,
active_event,
# must force the write when we have written a new value over list
force or has_been_written_since_last_msg,
)
# FIXME: check to ensure that second part of condition will
# still hide elements that do not change
if (not internal_has_been_written) and is_var_store and (not has_been_written_since_last_msg):
# remove the newly added map delta because there were no
# changes that it encoded
if var_type is FOR_MAP_CONTAINER_SERIALIZABLE:
del parent_delta.map_deltas[-1]
elif var_type is FOR_LIST_CONTAINER_SERIALIZABLE:
del parent_delta.list_deltas[-1]
elif var_type is FOR_STRUCT_CONTAINER_SERIALIZABLE:
del parent_delta.struct_deltas[-1]
else:
util.logger_assert("Unknown var type when serializing")
return internal_has_been_written or has_been_written_since_last_msg or force
def get_for_iter(self,to_iter_over,active_event):
'''
When call for loop on Waldo variables, need to get item to
iterate over
'''
if (isinstance(to_iter_over,dict) or
isinstance(to_iter_over,list) or
util.is_string(to_iter_over)):
return iter(to_iter_over)
if is_non_ext_text_var(to_iter_over):
return iter(to_iter_over.get_val(active_event))
if is_non_ext_map_var(to_iter_over):
return iter(to_iter_over.get_val(active_event).get_keys(active_event))
if is_non_ext_list_var(to_iter_over):
# FIXME: This is an inefficient way of reading all values
# over list.
to_return = []
internal_val = to_iter_over.get_val(active_event)
for i in range(0, internal_val.get_len(active_event)):
to_append = internal_val.get_val_on_key(active_event,i)
# The reason that we do this here is that in a for
# loop, the operation we perform in the compiled code
# immediately following the actual for header is
# assign to another Waldo variable the variable being
# held by to_append. (We do this through a set_val
# call. set_val must take in InternalMaps or
# InternalLists. Therefore, get_val on the
# WaldoList/WaldoMap first. Note this is unnecessary
# for all other for iterations because none of the
# others possibly return a WaldoObject to iterate over.
if (is_non_ext_list_var(to_append) or
is_non_ext_map_var(to_append)):
to_append = to_append.get_val(active_event)
to_return.append(to_append)
return iter(to_return)
util.logger_assert(
'Calling get_for_iter on an object that does not support iteration')
def value_variable_serializable_var_tuple_for_network(
value_variable,parent_delta,var_name,active_event, force):
'''
@see waldoLockedObj.serializable_var_tuple_for_network
'''
var_data = value_variable.get_val(active_event)
has_been_written_since_last_msg = value_variable.get_and_reset_has_been_written_since_last_msg(active_event)
if (not force) and (not has_been_written_since_last_msg):
# nothing to do because this value has not been
# written. NOTE: for list/dict types, must actually
# go through to ensure no subelements were written.
return False
# check if this is a python value type. if it is, append it
# to delta.
if not value_variable_py_val_serialize(
value_variable,parent_delta,var_data,var_name):
util.logger_assert('Should only use python values when serializing')
return True
def serializable_var_tuple_for_network(
self,parent_delta,var_name,invalid_listener,force):
'''
The runtime automatically synchronizes data between both
endpoints. When one side has updated a peered variable, the
other side needs to attempt to apply those changes before
doing further work. This method grabs the val and version
object of the dirty element associated with invalid_listener.
Using these data, plus var_name, it constructs a named tuple
for serialization. (@see
util._generate_serialization_named_tuple)
Note: if the val of this object is another Reference object,
then we recursively keep generating named tuples and embed
them in the one we return.
Note: we only serialize peered data. No other data gets sent
over the network; therefore, it should not be serialized.
@param {*Delta or VarStoreDeltas} parent_delta --- Append any
message that we create here to this message.
@param {String} var_name --- Both sides of the connection need
to agree on a common name for the variable being serialized.
This is to ensure that when the data are received by the other
side we know which variable to put them into. This value is
only really necessary for the outermost wrapping of the named
type tuple, but we pass it through anyways.
@param {bool} force --- True if regardless of whether modified
or not we should serialize. False otherwise. (We migth want
to force for instance the first time we send sequence data.)
@returns {bool} --- True if some subelement was modified,
False otherwise.
'''
util.logger_assert(
'Serializable var tuple for network is pure virtual ' +
'in waldoLockedObj.')
def handle_len(self,what_calling_len_on, active_event):
'''
Can support python lists, dicts, strings, or waldo lists, waldo maps,
waldo texts.
@returns {int}
'''
if (isinstance(what_calling_len_on, dict) or
isinstance(what_calling_len_on, list) or
util.is_string(what_calling_len_on)):
return len(what_calling_len_on)
if is_non_ext_text_var(what_calling_len_on):
return len(what_calling_len_on.get_val(active_event))
if (is_non_ext_list_var(what_calling_len_on) or
is_non_ext_map_var(what_calling_len_on)):
return what_calling_len_on.get_val(active_event).get_len(active_event)
util.logger_assert(
'Calling len on an object that does not support the function')
def func_turn_into_waldo_var(
self,val,force_copy,active_event, host_uuid,new_peered,
ext_args_array,new_multi_threaded):
'''
turn_into_waldo_var works for all non-function types.
function-types require additional information (which arguments
are and are not external) to populate their ext_args_array.
This is passed in in this function.
'''
if is_non_ext_func_var(val):
if force_copy:
# means that it was a WaldoVariable: just call its copy
# method
return val.copy(active_event,new_peered,new_multi_threaded)
# otherwise, just return val
return val
elif hasattr(val,'__call__'):
# python function
pass
#### DEBUG
else:
util.logger_assert(
'incorrect type passed into func_turn_into_waldo_var')
#### END DEBUG
if new_multi_threaded:
waldo_func = LockedFunctionVariable(
host_uuid,
new_peered,
val).set_external_args_array(ext_args_array)
else:
waldo_func = SingleThreadedLockedFunctionVariable(
host_uuid,
new_peered,
val).set_external_args_array(ext_args_array)
return waldo_func
def handle_non_first_sequence_msg_from_partner(
self,msg,name_of_block_to_exec_next):
'''
ASSUMES ALREADY WITHIN LOCK
@param {PartnerMessageRequestSequenceBlock.proto} msg ---
@param {string or None} name_of_block_to_exec_next --- the
name of the sequence block to execute next. None if nothing to
execute next (ie, last sequence message).
'''
#### DEBUG
if msg.reply_to_uuid.data not in self.message_listening_queues_map:
util.logger_assert(
'Error: partner response message responding to ' +
'unknown _ActiveEvent message.' +
str(msg.reply_to_uuid.data) +
' ' +
str(list(self.message_listening_queues_map.keys()))
)
#### END DEBUG
# unblock waiting listening queue.
self.message_listening_queues_map[msg.reply_to_uuid.data].put(
_SequenceMessageCallResult(
msg.reply_with_uuid.data,
name_of_block_to_exec_next,
# as soon as read from the listening message
# queue, populate sequence local data from context
# using sequence_local_var_store_deltas.
# FIXME: eventaully, want to move away from
# pickle-ing here.
msg.sequence_local_var_store_deltas))
# no need holding onto queue waiting on a message response.
del self.message_listening_queues_map[msg.reply_to_uuid.data]
def de_waldoify(self,active_event):
util.logger_assert(
'de_waldoify is pure virutal in WaldoLockedObj')
