def test_function_call_no_resources(self):
'''
Constructs a scenario where there are no available resources in the
system, and ensures that the scheduler correctly returns an error to
the user.
'''
# Clear all executors from the system.
self.policy.thread_statuses.clear()
self.policy.unpinned_executors.clear()
# Create a function call.
call = FunctionCall()
call.name = 'function'
call.request_id = 12
val = call.arguments.values.add()
serializer.dump(2, val)
self.socket.inbox.append(call.SerializeToString())
# Execute the scheduling policy.
call_function(self.socket, self.pusher_cache, self.policy)
# Check that the correct number of messages were sent.
self.assertEqual(len(self.socket.outbox), 1)
self.assertEqual(len(self.pusher_cache.socket.outbox), 0)
# Extract and deserialize the messages.
response = GenericResponse()
response.ParseFromString(self.socket.outbox[0])
self.assertFalse(response.success)
self.assertEqual(response.error, NO_RESOURCES)
def call_function(func_call_socket, pusher_cache, policy):
# Parse the received protobuf for this function call.
call = FunctionCall()
call.ParseFromString(func_call_socket.recv())
# If there is no response key set for this request, we generate a random
# UUID.
if not call.response_key:
call.response_key = str(uuid.uuid4())
# Filter the arguments for CloudburstReferences, and use the policy engine to
# pick a node for this request.
refs = list(
filter(lambda arg: type(arg) == CloudburstReference,
map(lambda arg: serializer.load(arg), call.arguments.values)))
result = policy.pick_executor(refs)
response = GenericResponse()
if result is None:
response.success = False
response.error = NO_RESOURCES
func_call_socket.send(response.SerializeToString())
return
# Forward the request on to the chosen executor node.
ip, tid = result
sckt = pusher_cache.get(utils.get_exec_address(ip, tid))
sckt.send(call.SerializeToString())
# Send a success response to the user with the response key.
response.success = True
response.response_id = call.response_key
func_call_socket.send(response.SerializeToString())
def call_function_from_queue(func_call_queue_socket, pusher_cache, policy):
call = FunctionCall()
call.ParseFromString(func_call_queue_socket.recv())
# If there is no response key set for this request, we generate a random
# UUID.
if not call.response_key:
call.response_key = str(uuid.uuid4())
if call.source_hint == STORAGE:
# It means the invocation is from storage,
# so we parse the arguments in a different way
# TODO remove loc from the original call
loc_set = set(call.locations)
result = policy.pick_executor_with_loc(call.name, loc_set)
if result is None:
logging.error('No executor available for STORAGE CALL')
return
ip, tid = result
logging.info(
'Pick executor %s:%d for STORAGE CALL %s with locations %s' %
(ip, tid, call.name, loc_set))
sckt = pusher_cache.get(utils.get_exec_address(ip, tid))
sckt.send(call.SerializeToString())
def exec_func(self, name, args):
call = FunctionCall()
call.name = name
call.request_id = self.rid
for arg in args:
argobj = call.arguments.values.add()
serializer.dump(arg, argobj)
self.func_call_sock.send(call.SerializeToString())
r = GenericResponse()
r.ParseFromString(self.func_call_sock.recv())
self.rid += 1
return r.response_id
def test_call_function_with_refs(self):
'''
Creates a scenario where the policy should deterministically pick the
same executor to run a request on: There is one reference, and it's
cached only on the node we create in this test.
'''
# Add a new executor for which we will construct cached references.
ip_address = '192.168.0.1'
new_key = (ip_address, 2)
self.policy.unpinned_executors.add(new_key)
# Create a new reference and add its metadata.
ref_name = 'reference'
self.policy.key_locations[ref_name] = [ip_address]
# Create a function call that asks for this reference.
call = FunctionCall()
call.name = 'function'
call.request_id = 12
val = call.arguments.values.add()
serializer.dump(CloudburstReference(ref_name, True), val)
self.socket.inbox.append(call.SerializeToString(0))
# Execute the scheduling policy.
call_function(self.socket, self.pusher_cache, self.policy)
# Check that the correct number of messages were sent.
self.assertEqual(len(self.socket.outbox), 1)
self.assertEqual(len(self.pusher_cache.socket.outbox), 1)
# Extract and deserialize the messages.
response = GenericResponse()
forwarded = FunctionCall()
response.ParseFromString(self.socket.outbox[0])
forwarded.ParseFromString(self.pusher_cache.socket.outbox[0])
self.assertTrue(response.success)
self.assertEqual(response.response_id, forwarded.response_key)
self.assertEqual(forwarded.name, call.name)
self.assertEqual(forwarded.request_id, call.request_id)
# Makes sure that the correct executor was chosen.
self.assertEqual(len(self.pusher_cache.addresses), 1)
self.assertEqual(self.pusher_cache.addresses[0],
utils.get_exec_address(*new_key))
def test_call_function_no_refs(self):
'''
A basic test that makes sure that an executor is successfully chosen
when there is only one possible executor to choose from.
'''
# Create a new function call for a function that doesn't exist.
call = FunctionCall()
call.name = 'function'
call.request_id = 12
# Add an argument to thhe function.
val = call.arguments.values.add()
serializer.dump(2, val)
self.socket.inbox.append(call.SerializeToString())
# Execute the scheduling policy.
call_function(self.socket, self.pusher_cache, self.policy)
# Check that the correct number of messages were sent.
self.assertEqual(len(self.socket.outbox), 1)
self.assertEqual(len(self.pusher_cache.socket.outbox), 1)
# Extract and deserialize the messages.
response = GenericResponse()
forwarded = FunctionCall()
response.ParseFromString(self.socket.outbox[0])
forwarded.ParseFromString(self.pusher_cache.socket.outbox[0])
self.assertTrue(response.success)
self.assertEqual(response.response_id, forwarded.response_key)
self.assertEqual(forwarded.name, call.name)
self.assertEqual(forwarded.request_id, call.request_id)
# Makes sure that the correct executor was chosen.
self.assertEqual(len(self.pusher_cache.addresses), 1)
self.assertEqual(self.pusher_cache.addresses[0],
utils.get_exec_address(*self.executor_key))