def call_function(func_call_socket, pusher_cache, policy): # Parse the received protobuf for this function call. call = FunctionCall() call.ParseFromString(func_call_socket.recv()) # If there is no response key set for this request, we generate a random # UUID. if not call.response_key: call.response_key = str(uuid.uuid4()) # Filter the arguments for CloudburstReferences, and use the policy engine to # pick a node for this request. refs = list( filter(lambda arg: type(arg) == CloudburstReference, map(lambda arg: serializer.load(arg), call.arguments.values))) result = policy.pick_executor(refs) response = GenericResponse() if result is None: response.success = False response.error = NO_RESOURCES func_call_socket.send(response.SerializeToString()) return # Forward the request on to the chosen executor node. ip, tid = result sckt = pusher_cache.get(utils.get_exec_address(ip, tid)) sckt.send(call.SerializeToString()) # Send a success response to the user with the response key. response.success = True response.response_id = call.response_key func_call_socket.send(response.SerializeToString())
def test_function_call_no_resources(self): ''' Constructs a scenario where there are no available resources in the system, and ensures that the scheduler correctly returns an error to the user. ''' # Clear all executors from the system. self.policy.thread_statuses.clear() self.policy.unpinned_executors.clear() # Create a function call. call = FunctionCall() call.name = 'function' call.request_id = 12 val = call.arguments.values.add() serializer.dump(2, val) self.socket.inbox.append(call.SerializeToString()) # Execute the scheduling policy. call_function(self.socket, self.pusher_cache, self.policy) # Check that the correct number of messages were sent. self.assertEqual(len(self.socket.outbox), 1) self.assertEqual(len(self.pusher_cache.socket.outbox), 0) # Extract and deserialize the messages. response = GenericResponse() response.ParseFromString(self.socket.outbox[0]) self.assertFalse(response.success) self.assertEqual(response.error, NO_RESOURCES)
def call_function_from_queue(func_call_queue_socket, pusher_cache, policy): call = FunctionCall() call.ParseFromString(func_call_queue_socket.recv()) # If there is no response key set for this request, we generate a random # UUID. if not call.response_key: call.response_key = str(uuid.uuid4()) if call.source_hint == STORAGE: # It means the invocation is from storage, # so we parse the arguments in a different way # TODO remove loc from the original call loc_set = set(call.locations) result = policy.pick_executor_with_loc(call.name, loc_set) if result is None: logging.error('No executor available for STORAGE CALL') return ip, tid = result logging.info( 'Pick executor %s:%d for STORAGE CALL %s with locations %s' % (ip, tid, call.name, loc_set)) sckt = pusher_cache.get(utils.get_exec_address(ip, tid)) sckt.send(call.SerializeToString())
def exec_function(exec_socket, kvs, user_library, cache, function_cache): call = FunctionCall() call.ParseFromString(exec_socket.recv()) fargs = [serializer.load(arg) for arg in call.arguments.values] if call.name in function_cache: f = function_cache[call.name] else: f = utils.retrieve_function(call.name, kvs, user_library, call.consistency) if not f: logging.info('Function %s not found! Returning an error.' % (call.name)) sutils.error.error = FUNC_NOT_FOUND result = ('ERROR', sutils.error.SerializeToString()) else: function_cache[call.name] = f try: if call.consistency == NORMAL: result = _exec_func_normal(kvs, f, fargs, user_library, cache) logging.info('Finished executing %s: %s!' % (call.name, str(result))) else: dependencies = {} result = _exec_func_causal(kvs, f, fargs, user_library, dependencies=dependencies) except Exception as e: logging.exception('Unexpected error %s while executing function.' % (str(e))) sutils.error.error = EXECUTION_ERROR result = ('ERROR: ' + str(e), sutils.error.SerializeToString()) if call.consistency == NORMAL: result = serializer.dump_lattice(result) succeed = kvs.put(call.response_key, result) else: result = serializer.dump_lattice(result, MultiKeyCausalLattice, causal_dependencies=dependencies) succeed = kvs.causal_put(call.response_key, result) if not succeed: logging.info(f'Unsuccessful attempt to put key {call.response_key} ' + 'into the KVS.')
def exec_func(self, name, args): call = FunctionCall() call.name = name call.request_id = self.rid for arg in args: argobj = call.arguments.values.add() serializer.dump(arg, argobj) self.func_call_sock.send(call.SerializeToString()) r = GenericResponse() r.ParseFromString(self.func_call_sock.recv()) self.rid += 1 return r.response_id
def test_call_function_with_refs(self): ''' Creates a scenario where the policy should deterministically pick the same executor to run a request on: There is one reference, and it's cached only on the node we create in this test. ''' # Add a new executor for which we will construct cached references. ip_address = '192.168.0.1' new_key = (ip_address, 2) self.policy.unpinned_executors.add(new_key) # Create a new reference and add its metadata. ref_name = 'reference' self.policy.key_locations[ref_name] = [ip_address] # Create a function call that asks for this reference. call = FunctionCall() call.name = 'function' call.request_id = 12 val = call.arguments.values.add() serializer.dump(CloudburstReference(ref_name, True), val) self.socket.inbox.append(call.SerializeToString(0)) # Execute the scheduling policy. call_function(self.socket, self.pusher_cache, self.policy) # Check that the correct number of messages were sent. self.assertEqual(len(self.socket.outbox), 1) self.assertEqual(len(self.pusher_cache.socket.outbox), 1) # Extract and deserialize the messages. response = GenericResponse() forwarded = FunctionCall() response.ParseFromString(self.socket.outbox[0]) forwarded.ParseFromString(self.pusher_cache.socket.outbox[0]) self.assertTrue(response.success) self.assertEqual(response.response_id, forwarded.response_key) self.assertEqual(forwarded.name, call.name) self.assertEqual(forwarded.request_id, call.request_id) # Makes sure that the correct executor was chosen. self.assertEqual(len(self.pusher_cache.addresses), 1) self.assertEqual(self.pusher_cache.addresses[0], utils.get_exec_address(*new_key))
def _create_function_call(self, fname, args, consistency): call = FunctionCall() call.name = fname call.request_id = 1 call.response_key = self.response_key call.consistency = consistency for arg in args: val = call.arguments.values.add() serializer.dump(arg, val, False) return call
def test_call_function_no_refs(self): ''' A basic test that makes sure that an executor is successfully chosen when there is only one possible executor to choose from. ''' # Create a new function call for a function that doesn't exist. call = FunctionCall() call.name = 'function' call.request_id = 12 # Add an argument to thhe function. val = call.arguments.values.add() serializer.dump(2, val) self.socket.inbox.append(call.SerializeToString()) # Execute the scheduling policy. call_function(self.socket, self.pusher_cache, self.policy) # Check that the correct number of messages were sent. self.assertEqual(len(self.socket.outbox), 1) self.assertEqual(len(self.pusher_cache.socket.outbox), 1) # Extract and deserialize the messages. response = GenericResponse() forwarded = FunctionCall() response.ParseFromString(self.socket.outbox[0]) forwarded.ParseFromString(self.pusher_cache.socket.outbox[0]) self.assertTrue(response.success) self.assertEqual(response.response_id, forwarded.response_key) self.assertEqual(forwarded.name, call.name) self.assertEqual(forwarded.request_id, call.request_id) # Makes sure that the correct executor was chosen. self.assertEqual(len(self.pusher_cache.addresses), 1) self.assertEqual(self.pusher_cache.addresses[0], utils.get_exec_address(*self.executor_key))
def exec_function(exec_socket, kvs, user_library, cache, function_cache, has_ephe=False): call = FunctionCall() call.ParseFromString(exec_socket.recv()) fargs = [serializer.load(arg) for arg in call.arguments.values] if call.name in function_cache: f = function_cache[call.name] else: f = utils.retrieve_function(call.name, kvs, user_library, call.consistency) if not f: logging.info('Function %s not found! Returning an error.' % (call.name)) sutils.error.error = FUNC_NOT_FOUND result = ('ERROR', sutils.error.SerializeToString()) else: function_cache[call.name] = f # We set the session as the response key from scheduler # It should be uuid and identical if has_ephe: user_library.session = call.response_key try: if call.consistency == NORMAL: result = _exec_func_normal(kvs, f, fargs, user_library, cache) logging.info('Finished executing %s: %s!' % (call.name, str(result))) else: dependencies = {} result = _exec_func_causal(kvs, f, fargs, user_library, dependencies=dependencies) except Exception as e: logging.exception('Unexpected error %s while executing function.' % (str(e))) sutils.error.error = EXECUTION_ERROR result = ('ERROR: ' + str(e), sutils.error.SerializeToString()) # When we use ephe kvs for coordination, we do not write the results to anna # Instead, we presume the function will put the result mannually if not has_ephe: if call.consistency == NORMAL: result = serializer.dump_lattice(result) succeed = kvs.put(call.response_key, result) else: result = serializer.dump_lattice(result, MultiKeyCausalLattice, causal_dependencies=dependencies) succeed = kvs.causal_put(call.response_key, result) if not succeed: logging.info( f'Unsuccessful attempt to put key {call.response_key} ' + 'into the KVS.')