def call_function(func_call_socket, pusher_cache, policy):
# Parse the received protobuf for this function call.
call = FunctionCall()
call.ParseFromString(func_call_socket.recv())
# If there is no response key set for this request, we generate a random
# UUID.
if not call.response_key:
call.response_key = str(uuid.uuid4())
# Filter the arguments for CloudburstReferences, and use the policy engine to
# pick a node for this request.
refs = list(
filter(lambda arg: type(arg) == CloudburstReference,
map(lambda arg: serializer.load(arg), call.arguments.values)))
result = policy.pick_executor(refs)
response = GenericResponse()
if result is None:
response.success = False
response.error = NO_RESOURCES
func_call_socket.send(response.SerializeToString())
return
# Forward the request on to the chosen executor node.
ip, tid = result
sckt = pusher_cache.get(utils.get_exec_address(ip, tid))
sckt.send(call.SerializeToString())
# Send a success response to the user with the response key.
response.success = True
response.response_id = call.response_key
func_call_socket.send(response.SerializeToString())
def test_function_call_no_resources(self):
'''
Constructs a scenario where there are no available resources in the
system, and ensures that the scheduler correctly returns an error to
the user.
'''
# Clear all executors from the system.
self.policy.thread_statuses.clear()
self.policy.unpinned_executors.clear()
# Create a function call.
call = FunctionCall()
call.name = 'function'
call.request_id = 12
val = call.arguments.values.add()
serializer.dump(2, val)
self.socket.inbox.append(call.SerializeToString())
# Execute the scheduling policy.
call_function(self.socket, self.pusher_cache, self.policy)
# Check that the correct number of messages were sent.
self.assertEqual(len(self.socket.outbox), 1)
self.assertEqual(len(self.pusher_cache.socket.outbox), 0)
# Extract and deserialize the messages.
response = GenericResponse()
response.ParseFromString(self.socket.outbox[0])
self.assertFalse(response.success)
self.assertEqual(response.error, NO_RESOURCES)
def call_function_from_queue(func_call_queue_socket, pusher_cache, policy):
call = FunctionCall()
call.ParseFromString(func_call_queue_socket.recv())
# If there is no response key set for this request, we generate a random
# UUID.
if not call.response_key:
call.response_key = str(uuid.uuid4())
if call.source_hint == STORAGE:
# It means the invocation is from storage,
# so we parse the arguments in a different way
# TODO remove loc from the original call
loc_set = set(call.locations)
result = policy.pick_executor_with_loc(call.name, loc_set)
if result is None:
logging.error('No executor available for STORAGE CALL')
return
ip, tid = result
logging.info(
'Pick executor %s:%d for STORAGE CALL %s with locations %s' %
(ip, tid, call.name, loc_set))
sckt = pusher_cache.get(utils.get_exec_address(ip, tid))
sckt.send(call.SerializeToString())
def exec_function(exec_socket, kvs, user_library, cache, function_cache):
call = FunctionCall()
call.ParseFromString(exec_socket.recv())
fargs = [serializer.load(arg) for arg in call.arguments.values]
if call.name in function_cache:
f = function_cache[call.name]
else:
f = utils.retrieve_function(call.name, kvs, user_library,
call.consistency)
if not f:
logging.info('Function %s not found! Returning an error.' %
(call.name))
sutils.error.error = FUNC_NOT_FOUND
result = ('ERROR', sutils.error.SerializeToString())
else:
function_cache[call.name] = f
try:
if call.consistency == NORMAL:
result = _exec_func_normal(kvs, f, fargs, user_library, cache)
logging.info('Finished executing %s: %s!' %
(call.name, str(result)))
else:
dependencies = {}
result = _exec_func_causal(kvs,
f,
fargs,
user_library,
dependencies=dependencies)
except Exception as e:
logging.exception('Unexpected error %s while executing function.' %
(str(e)))
sutils.error.error = EXECUTION_ERROR
result = ('ERROR: ' + str(e), sutils.error.SerializeToString())
if call.consistency == NORMAL:
result = serializer.dump_lattice(result)
succeed = kvs.put(call.response_key, result)
else:
result = serializer.dump_lattice(result,
MultiKeyCausalLattice,
causal_dependencies=dependencies)
succeed = kvs.causal_put(call.response_key, result)
if not succeed:
logging.info(f'Unsuccessful attempt to put key {call.response_key} ' +
'into the KVS.')
def exec_func(self, name, args):
call = FunctionCall()
call.name = name
call.request_id = self.rid
for arg in args:
argobj = call.arguments.values.add()
serializer.dump(arg, argobj)
self.func_call_sock.send(call.SerializeToString())
r = GenericResponse()
r.ParseFromString(self.func_call_sock.recv())
self.rid += 1
return r.response_id
def test_call_function_with_refs(self):
'''
Creates a scenario where the policy should deterministically pick the
same executor to run a request on: There is one reference, and it's
cached only on the node we create in this test.
'''
# Add a new executor for which we will construct cached references.
ip_address = '192.168.0.1'
new_key = (ip_address, 2)
self.policy.unpinned_executors.add(new_key)
# Create a new reference and add its metadata.
ref_name = 'reference'
self.policy.key_locations[ref_name] = [ip_address]
# Create a function call that asks for this reference.
call = FunctionCall()
call.name = 'function'
call.request_id = 12
val = call.arguments.values.add()
serializer.dump(CloudburstReference(ref_name, True), val)
self.socket.inbox.append(call.SerializeToString(0))
# Execute the scheduling policy.
call_function(self.socket, self.pusher_cache, self.policy)
# Check that the correct number of messages were sent.
self.assertEqual(len(self.socket.outbox), 1)
self.assertEqual(len(self.pusher_cache.socket.outbox), 1)
# Extract and deserialize the messages.
response = GenericResponse()
forwarded = FunctionCall()
response.ParseFromString(self.socket.outbox[0])
forwarded.ParseFromString(self.pusher_cache.socket.outbox[0])
self.assertTrue(response.success)
self.assertEqual(response.response_id, forwarded.response_key)
self.assertEqual(forwarded.name, call.name)
self.assertEqual(forwarded.request_id, call.request_id)
# Makes sure that the correct executor was chosen.
self.assertEqual(len(self.pusher_cache.addresses), 1)
self.assertEqual(self.pusher_cache.addresses[0],
utils.get_exec_address(*new_key))
def _create_function_call(self, fname, args, consistency):
call = FunctionCall()
call.name = fname
call.request_id = 1
call.response_key = self.response_key
call.consistency = consistency
for arg in args:
val = call.arguments.values.add()
serializer.dump(arg, val, False)
return call
def test_call_function_no_refs(self):
'''
A basic test that makes sure that an executor is successfully chosen
when there is only one possible executor to choose from.
'''
# Create a new function call for a function that doesn't exist.
call = FunctionCall()
call.name = 'function'
call.request_id = 12
# Add an argument to thhe function.
val = call.arguments.values.add()
serializer.dump(2, val)
self.socket.inbox.append(call.SerializeToString())
# Execute the scheduling policy.
call_function(self.socket, self.pusher_cache, self.policy)
# Check that the correct number of messages were sent.
self.assertEqual(len(self.socket.outbox), 1)
self.assertEqual(len(self.pusher_cache.socket.outbox), 1)
# Extract and deserialize the messages.
response = GenericResponse()
forwarded = FunctionCall()
response.ParseFromString(self.socket.outbox[0])
forwarded.ParseFromString(self.pusher_cache.socket.outbox[0])
self.assertTrue(response.success)
self.assertEqual(response.response_id, forwarded.response_key)
self.assertEqual(forwarded.name, call.name)
self.assertEqual(forwarded.request_id, call.request_id)
# Makes sure that the correct executor was chosen.
self.assertEqual(len(self.pusher_cache.addresses), 1)
self.assertEqual(self.pusher_cache.addresses[0],
utils.get_exec_address(*self.executor_key))
def exec_function(exec_socket,
kvs,
user_library,
cache,
function_cache,
has_ephe=False):
call = FunctionCall()
call.ParseFromString(exec_socket.recv())
fargs = [serializer.load(arg) for arg in call.arguments.values]
if call.name in function_cache:
f = function_cache[call.name]
else:
f = utils.retrieve_function(call.name, kvs, user_library,
call.consistency)
if not f:
logging.info('Function %s not found! Returning an error.' %
(call.name))
sutils.error.error = FUNC_NOT_FOUND
result = ('ERROR', sutils.error.SerializeToString())
else:
function_cache[call.name] = f
# We set the session as the response key from scheduler
# It should be uuid and identical
if has_ephe:
user_library.session = call.response_key
try:
if call.consistency == NORMAL:
result = _exec_func_normal(kvs, f, fargs, user_library, cache)
logging.info('Finished executing %s: %s!' %
(call.name, str(result)))
else:
dependencies = {}
result = _exec_func_causal(kvs,
f,
fargs,
user_library,
dependencies=dependencies)
except Exception as e:
logging.exception('Unexpected error %s while executing function.' %
(str(e)))
sutils.error.error = EXECUTION_ERROR
result = ('ERROR: ' + str(e), sutils.error.SerializeToString())
# When we use ephe kvs for coordination, we do not write the results to anna
# Instead, we presume the function will put the result mannually
if not has_ephe:
if call.consistency == NORMAL:
result = serializer.dump_lattice(result)
succeed = kvs.put(call.response_key, result)
else:
result = serializer.dump_lattice(result,
MultiKeyCausalLattice,
causal_dependencies=dependencies)
succeed = kvs.causal_put(call.response_key, result)
if not succeed:
logging.info(
f'Unsuccessful attempt to put key {call.response_key} ' +
'into the KVS.')