def test_exec_function_normal(self): ''' Tests creating and executing a function in normal mode, ensuring that no messages are sent outside of the system and that the serialized result is as expected. ''' # Create the function and put it into the KVS. def func(_, x): return x * x fname = 'square' arg = 2 # Put the function into the KVS and create a function call. create_function(func, self.kvs_client, fname) call = self._create_function_call(fname, [arg], NORMAL) self.socket.inbox.append(call.SerializeToString()) # Execute the function call. exec_function(self.socket, self.kvs_client, self.user_library, {}, {}) # Assert that there have been 0 messages sent. self.assertEqual(len(self.socket.outbox), 0) # Retrieve the result, ensure it is a LWWPairLattice, then deserialize # it. result = self.kvs_client.get(self.response_key)[self.response_key] self.assertEqual(type(result), LWWPairLattice) result = serializer.load_lattice(result) # Check that the output is equal to a local function execution. self.assertEqual(result, func('', arg))
def test_exec_function_causal(self): ''' Tests creating and executing a function in causal mode, ensuring that no messages are sent outside of the system and that the serialized result is as expected. ''' # Create the function and serialize it into a lattice. def func(_, x): return x * x fname = 'square' create_function(func, self.kvs_client, fname, SingleKeyCausalLattice) arg = 2 # Put the function into the KVS and create a function call. call = self._create_function_call(fname, [arg], MULTI) self.socket.inbox.append(call.SerializeToString()) # Execute the function call. exec_function(self.socket, self.kvs_client, self.user_library, {}, {}) # Assert that there have been 0 messages sent. self.assertEqual(len(self.socket.outbox), 0) # Retrieve the result, ensure it is a MultiKeyCausalLattice, then # deserialize it. Also check to make sure we have an empty vector clock # because this request populated no dependencies. result = self.kvs_client.get(self.response_key)[self.response_key] self.assertEqual(type(result), MultiKeyCausalLattice) self.assertEqual(result.vector_clock, DEFAULT_VC) result = serializer.load_lattice(result)[0] # Check that the output is equal to a local function execution. self.assertEqual(result, func('', arg))
def test_exec_with_ordered_set(self): ''' Tests a single function execution with an ordered set input as an argument to validate that ordered sets are correctly handled. ''' def func(_, x): return len(x) >= 2 and x[0] < x[1] fname = 'set_order' arg_value = [2, 3] arg_name = 'set' self.kvs_client.put(arg_name, serializer.dump_lattice(arg_value)) # Put the function into the KVS and create a function call. create_function(func, self.kvs_client, fname) call = self._create_function_call( fname, [CloudburstReference(arg_name, True)], NORMAL) self.socket.inbox.append(call.SerializeToString()) # Execute the function call. exec_function(self.socket, self.kvs_client, self.user_library, {}, {}) # Assert that there have been 0 messages sent. self.assertEqual(len(self.socket.outbox), 0) # Retrieve the result, ensure it is a LWWPairLattice, then deserialize # it. result = self.kvs_client.get(self.response_key)[self.response_key] self.assertEqual(type(result), LWWPairLattice) result = serializer.load_lattice(result) # Check that the output is equal to a local function execution. self.assertEqual(result, func('', arg_value))
def test_exec_function_with_error(self): ''' Attempts to executet a function that raises an error during its execution. Ensures that an error is returned to the user. ''' e_msg = 'This is a broken_function!' def func(_, x): raise ValueError(e_msg) fname = 'func' create_function(func, self.kvs_client, fname) # Put the functin into the KVS and create a function call. call = self._create_function_call(fname, [''], NORMAL) self.socket.inbox.append(call.SerializeToString()) # Execute the function call. exec_function(self.socket, self.kvs_client, self.user_library, {}, {}) # Retrieve the result from the KVS and ensure that it is the correct # lattice type. result = self.kvs_client.get(self.response_key)[self.response_key] self.assertEqual(type(result), LWWPairLattice) result = serializer.load_lattice(result) # Check the type and values of the error. self.assertEqual(type(result), tuple) self.assertTrue(e_msg in result[0]) # Unpack the GenericResponse and check its values. response = GenericResponse() response.ParseFromString(result[1]) self.assertEqual(response.success, False) self.assertEqual(response.error, EXECUTION_ERROR)
def test_exec_function_nonexistent(self): ''' Attempts to execute a non-existent function and ensures that an error is thrown and returned to the user. ''' # Create a call to a function that does not exist. call = self._create_function_call('bad_func', [1], NORMAL) self.socket.inbox.append(call.SerializeToString()) # Attempt to execute the nonexistent function. exec_function(self.socket, self.kvs_client, self.user_library, {}, {}) # Assert that there have been 0 messages sent. self.assertEqual(len(self.socket.outbox), 0) # Retrieve the result from the KVS and ensure that it is a lattice as # we expect. Deserialize it and check for an error. result = self.kvs_client.get(self.response_key)[self.response_key] self.assertEqual(type(result), LWWPairLattice) result = serializer.load_lattice(result) # Check the type and values of the error. self.assertEqual(type(result), tuple) self.assertEqual(result[0], 'ERROR') # Unpack the GenericResponse and check its values. response = GenericResponse() response.ParseFromString(result[1]) self.assertEqual(response.success, False) self.assertEqual(response.error, FUNC_NOT_FOUND)
def test_exec_func_with_causal_ref(self): ''' Tests a function execution where the argument is a reference to the KVS in causal mode. Ensures that the result has the correct causal dependencies and metadata. ''' # Create the function and serialize it into a lattice. def func(_, x): return x * x fname = 'square' create_function(func, self.kvs_client, fname, SingleKeyCausalLattice) # Put an argument value into the KVS. arg_value = 2 arg_name = 'key' self.kvs_client.put( arg_name, serializer.dump_lattice(arg_value, MultiKeyCausalLattice)) # Create and serialize the function call. call = self._create_function_call( fname, [CloudburstReference(arg_name, True)], MULTI) self.socket.inbox.append(call.SerializeToString()) # Execute the function call. exec_function(self.socket, self.kvs_client, self.user_library, {}, {}) # Assert that there have been 0 messages sent. self.assertEqual(len(self.socket.outbox), 0) # Retrieve the result, ensure it is a MultiKeyCausalLattice, then # deserialize it. result = self.kvs_client.get(self.response_key)[self.response_key] self.assertEqual(type(result), MultiKeyCausalLattice) self.assertEqual(result.vector_clock, DEFAULT_VC) self.assertEqual(len(result.dependencies.reveal()), 1) self.assertTrue(arg_name in result.dependencies.reveal()) self.assertEqual(result.dependencies.reveal()[arg_name], DEFAULT_VC) result = serializer.load_lattice(result)[0] # Check that the output is equal to a local function execution. self.assertEqual(result, func('', arg_value))
def test_exec_class_function(self): ''' Tests creating and executing a class method in normal mode, ensuring that no messages are sent outside of the system and that the serialized result is as expected. ''' # Create the function and put it into the KVS. class Test: def __init__(self, cloudburst, num): self.num = num def run(self, cloudburst, inp): return inp + self.num fname = 'class' init_arg = 3 arg = 2 # Put the function into the KVS and create a function call. create_function((Test, (init_arg, )), self.kvs_client, fname) call = self._create_function_call(fname, [arg], NORMAL) self.socket.inbox.append(call.SerializeToString()) # Execute the function call. exec_function(self.socket, self.kvs_client, self.user_library, {}, {}) # Assert that there have been 0 messages sent. self.assertEqual(len(self.socket.outbox), 0) # Retrieve the result, ensure it is a LWWPairLattice, then deserialize # it. result = self.kvs_client.get(self.response_key)[self.response_key] self.assertEqual(type(result), LWWPairLattice) result = serializer.load_lattice(result) # Check that the output is equal to a local function execution. self.assertEqual(result, Test(None, init_arg).run('', arg))
def test_exec_func_with_ref(self): ''' Tests a function execution where the argument is a reference to the KVS in normal mode. ''' # Create the function and serialize it into a lattice. def func(_, x): return x * x fname = 'square' create_function(func, self.kvs_client, fname) # Put an argument value into the KVS. arg_value = 2 arg_name = 'key' self.kvs_client.put(arg_name, serializer.dump_lattice(arg_value)) # Create and serialize the function call. call = self._create_function_call( fname, [CloudburstReference(arg_name, True)], NORMAL) self.socket.inbox.append(call.SerializeToString()) # Execute the function call. exec_function(self.socket, self.kvs_client, self.user_library, {}, {}) # Assert that there have been 0 messages sent. self.assertEqual(len(self.socket.outbox), 0) # Retrieve the result, ensure it is a LWWPairLattice, then deserialize # it. result = self.kvs_client.get(self.response_key)[self.response_key] self.assertEqual(type(result), LWWPairLattice) result = serializer.load_lattice(result) # Check that the output is equal to a local function execution. self.assertEqual(result, func('', arg_value))
def executor(ip, mgmt_ip, schedulers, thread_id): # logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(asctime)s %(message)s') logging.basicConfig(filename='log_executor.txt', level=logging.INFO, filemode="w", format='%(asctime)s %(message)s') # Check what resources we have access to, set as an environment variable. if os.getenv('EXECUTOR_TYPE', 'CPU') == 'GPU': exec_type = GPU else: exec_type = CPU context = zmq.Context(1) poller = zmq.Poller() pin_socket = context.socket(zmq.PULL) pin_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.PIN_PORT + thread_id)) unpin_socket = context.socket(zmq.PULL) unpin_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.UNPIN_PORT + thread_id)) exec_socket = context.socket(zmq.PULL) exec_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.FUNC_EXEC_PORT + thread_id)) dag_queue_socket = context.socket(zmq.PULL) dag_queue_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.DAG_QUEUE_PORT + thread_id)) dag_exec_socket = context.socket(zmq.PULL) dag_exec_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.DAG_EXEC_PORT + thread_id)) self_depart_socket = context.socket(zmq.PULL) self_depart_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.SELF_DEPART_PORT + thread_id)) pusher_cache = SocketCache(context, zmq.PUSH) poller = zmq.Poller() poller.register(pin_socket, zmq.POLLIN) poller.register(unpin_socket, zmq.POLLIN) poller.register(exec_socket, zmq.POLLIN) poller.register(dag_queue_socket, zmq.POLLIN) poller.register(dag_exec_socket, zmq.POLLIN) poller.register(self_depart_socket, zmq.POLLIN) # If the management IP is set to None, that means that we are running in # local mode, so we use a regular AnnaTcpClient rather than an IPC client. has_ephe = False if mgmt_ip: if 'STORAGE_OR_DEFAULT' in os.environ and os.environ[ 'STORAGE_OR_DEFAULT'] == '0': client = AnnaTcpClient(os.environ['ROUTE_ADDR'], ip, local=False, offset=thread_id) has_ephe = True else: client = AnnaIpcClient(thread_id, context) # force_remote_anna = 1 # if 'FORCE_REMOTE' in os.environ: # force_remote_anna = int(os.environ['FORCE_REMOTE']) # if force_remote_anna == 0: # remote anna only # client = AnnaTcpClient(os.environ['ROUTE_ADDR'], ip, local=False, offset=thread_id) # elif force_remote_anna == 1: # anna cache # client = AnnaIpcClient(thread_id, context) # elif force_remote_anna == 2: # control both cache and remote anna # remote_client = AnnaTcpClient(os.environ['ROUTE_ADDR'], ip, local=False, offset=thread_id) # cache_client = AnnaIpcClient(thread_id, context) # client = cache_client # user_library = CloudburstUserLibrary(context, pusher_cache, ip, thread_id, (cache_client, remote_client)) local = False else: client = AnnaTcpClient('127.0.0.1', '127.0.0.1', local=True, offset=1) local = True user_library = CloudburstUserLibrary(context, pusher_cache, ip, thread_id, client, has_ephe=has_ephe) status = ThreadStatus() status.ip = ip status.tid = thread_id status.running = True status.type = exec_type utils.push_status(schedulers, pusher_cache, status) departing = False # Maintains a request queue for each function pinned on this executor. Each # function will have a set of request IDs mapped to it, and this map stores # a schedule for each request ID. queue = {} # Tracks the actual function objects that are pinned to this executor. function_cache = {} # Tracks runtime cost of excuting a DAG function. runtimes = {} # If multiple triggers are necessary for a function, track the triggers as # we receive them. This is also used if a trigger arrives before its # corresponding schedule. received_triggers = {} # Tracks when we received a function request, so we can report end-to-end # latency for the whole executio. receive_times = {} # Tracks the number of requests we are finishing for each function pinned # here. exec_counts = {} # Tracks the end-to-end runtime of each DAG request for which we are the # sink function. dag_runtimes = {} # A map with KVS keys and their corresponding deserialized payloads. cache = {} # A map which tracks the most recent DAGs for which we have finished our # work. finished_executions = {} # The set of pinned functions and whether they support batching. NOTE: This # is only a set for local mode -- in cluster mode, there will only be one # pinned function per executor. batching = False # Internal metadata to track thread utilization. report_start = time.time() event_occupancy = { 'pin': 0.0, 'unpin': 0.0, 'func_exec': 0.0, 'dag_queue': 0.0, 'dag_exec': 0.0 } total_occupancy = 0.0 while True: socks = dict(poller.poll(timeout=1000)) if pin_socket in socks and socks[pin_socket] == zmq.POLLIN: work_start = time.time() batching = pin(pin_socket, pusher_cache, client, status, function_cache, runtimes, exec_counts, user_library, local, batching) utils.push_status(schedulers, pusher_cache, status) elapsed = time.time() - work_start event_occupancy['pin'] += elapsed total_occupancy += elapsed if unpin_socket in socks and socks[unpin_socket] == zmq.POLLIN: work_start = time.time() unpin(unpin_socket, status, function_cache, runtimes, exec_counts) utils.push_status(schedulers, pusher_cache, status) elapsed = time.time() - work_start event_occupancy['unpin'] += elapsed total_occupancy += elapsed if exec_socket in socks and socks[exec_socket] == zmq.POLLIN: work_start = time.time() # logging.info(f'Executor timer. exec_socket recv: {work_start}') exec_function(exec_socket, client, user_library, cache, function_cache, has_ephe=has_ephe) user_library.close() utils.push_status(schedulers, pusher_cache, status) elapsed = time.time() - work_start event_occupancy['func_exec'] += elapsed total_occupancy += elapsed if dag_queue_socket in socks and socks[dag_queue_socket] == zmq.POLLIN: work_start = time.time() logging.info( f'Executor timer. dag_queue_socket recv: {work_start}') # In order to effectively support batching, we have to make sure we # dequeue lots of schedules in addition to lots of triggers. Right # now, we're not going to worry about supporting batching here, # just on the trigger dequeue side, but we still have to dequeue # all schedules we've received. We just process them one at a time. while True: schedule = DagSchedule() try: msg = dag_queue_socket.recv(zmq.DONTWAIT) except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: break # There are no more messages. else: raise e # Unexpected error. schedule.ParseFromString(msg) fname = schedule.target_function logging.info( 'Received a schedule for DAG %s (%s), function %s.' % (schedule.dag.name, schedule.id, fname)) if fname not in queue: queue[fname] = {} queue[fname][schedule.id] = schedule if (schedule.id, fname) not in receive_times: receive_times[(schedule.id, fname)] = time.time() # In case we receive the trigger before we receive the schedule, we # can trigger from this operation as well. trkey = (schedule.id, fname) fref = None # Check to see what type of execution this function is. for ref in schedule.dag.functions: if ref.name == fname: fref = ref if (trkey in received_triggers and ((len(received_triggers[trkey]) == len(schedule.triggers)) or (fref.type == MULTIEXEC))): triggers = list(received_triggers[trkey].values()) if fname not in function_cache: logging.error('%s not in function cache', fname) utils.generate_error_response(schedule, client, fname) continue exec_start = time.time() # logging.info(f'Executor timer. dag_queue_socket exec_dag: {exec_start}') # We don't support actual batching for when we receive a # schedule before a trigger, so everything is just a batch of # size 1 if anything. success = exec_dag_function(pusher_cache, client, [triggers], function_cache[fname], [schedule], user_library, dag_runtimes, cache, schedulers, batching)[0] user_library.close() del received_triggers[trkey] if success: del queue[fname][schedule.id] fend = time.time() fstart = receive_times[(schedule.id, fname)] runtimes[fname].append(fend - work_start) exec_counts[fname] += 1 finished_executions[(schedule.id, fname)] = time.time() elapsed = time.time() - work_start event_occupancy['dag_queue'] += elapsed total_occupancy += elapsed if dag_exec_socket in socks and socks[dag_exec_socket] == zmq.POLLIN: work_start = time.time() # logging.info(f'Executor timer. dag_exec_socket recv: {work_start}') # How many messages to dequeue -- BATCH_SIZE_MAX or 1 depending on # the function configuration. if batching: count = BATCH_SIZE_MAX else: count = 1 trigger_keys = set() for _ in range(count): # Dequeue count number of messages. trigger = DagTrigger() try: msg = dag_exec_socket.recv(zmq.DONTWAIT) except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: # There are no more messages. break else: raise e # Unexpected error. trigger.ParseFromString(msg) # We have received a repeated trigger for a function that has # already finished executing. if trigger.id in finished_executions: continue fname = trigger.target_function logging.info( 'Received a trigger for schedule %s, function %s.' % (trigger.id, fname)) key = (trigger.id, fname) trigger_keys.add(key) if key not in received_triggers: received_triggers[key] = {} if (trigger.id, fname) not in receive_times: receive_times[(trigger.id, fname)] = time.time() received_triggers[key][trigger.source] = trigger # Only execute the functions for which we have received a schedule. # Everything else will wait. for tid, fname in list(trigger_keys): if fname not in queue or tid not in queue[fname]: trigger_keys.remove((tid, fname)) if len(trigger_keys) == 0: continue fref = None schedule = queue[fname][list(trigger_keys)[0] [0]] # Pick a random schedule to check. # Check to see what type of execution this function is. for ref in schedule.dag.functions: if ref.name == fname: fref = ref break # Compile a list of all the trigger sets for which we have # enough triggers. trigger_sets = [] schedules = [] for key in trigger_keys: if (len(received_triggers[key]) == len(schedule.triggers)) or \ fref.type == MULTIEXEC: if fref.type == MULTIEXEC: triggers = [trigger] else: triggers = list(received_triggers[key].values()) if fname not in function_cache: logging.error('%s not in function cache', fname) utils.generate_error_response(schedule, client, fname) continue trigger_sets.append(triggers) schedule = queue[fname][key[0]] schedules.append(schedule) exec_start = time.time() # logging.info(f'Executor timer. dag_exec_socket exec_dag: {exec_start}') # Pass all of the trigger_sets into exec_dag_function at once. # We also include the batching variaible to make sure we know # whether to pass lists into the fn or not. if len(trigger_sets) > 0: successes = exec_dag_function(pusher_cache, client, trigger_sets, function_cache[fname], schedules, user_library, dag_runtimes, cache, schedulers, batching) user_library.close() del received_triggers[key] for key, success in zip(trigger_keys, successes): if success: del queue[fname][key[0]] # key[0] is trigger.id. fend = time.time() fstart = receive_times[key] average_time = (fend - work_start) / len(trigger_keys) runtimes[fname].append(average_time) exec_counts[fname] += 1 finished_executions[(schedule.id, fname)] = time.time() elapsed = time.time() - work_start event_occupancy['dag_exec'] += elapsed total_occupancy += elapsed if self_depart_socket in socks and socks[self_depart_socket] == \ zmq.POLLIN: # This message does not matter. self_depart_socket.recv() logging.info('Preparing to depart. No longer accepting requests ' + 'and clearing all queues.') status.ClearField('functions') status.running = False utils.push_status(schedulers, pusher_cache, status) departing = True # periodically report function occupancy report_end = time.time() if report_end - report_start > REPORT_THRESH: if len(cache) > 100: extra_keys = list(cache.keys())[:len(cache) - 100] for key in extra_keys: del cache[key] utilization = total_occupancy / (report_end - report_start) status.utilization = utilization # Periodically report my status to schedulers with the utilization # set. utils.push_status(schedulers, pusher_cache, status) logging.debug('Total thread occupancy: %.6f' % (utilization)) for event in event_occupancy: occ = event_occupancy[event] / (report_end - report_start) logging.debug('\tEvent %s occupancy: %.6f' % (event, occ)) event_occupancy[event] = 0.0 stats = ExecutorStatistics() for fname in runtimes: if exec_counts[fname] > 0: fstats = stats.functions.add() fstats.name = fname fstats.call_count = exec_counts[fname] fstats.runtime.extend(runtimes[fname]) runtimes[fname].clear() exec_counts[fname] = 0 for dname in dag_runtimes: dstats = stats.dags.add() dstats.name = dname dstats.runtimes.extend(dag_runtimes[dname]) dag_runtimes[dname].clear() # If we are running in cluster mode, mgmt_ip will be set, and we # will report our status and statistics to it. Otherwise, we will # write to the local conf file if mgmt_ip: sckt = pusher_cache.get( sutils.get_statistics_report_address(mgmt_ip)) sckt.send(stats.SerializeToString()) sckt = pusher_cache.get(utils.get_util_report_address(mgmt_ip)) sckt.send(status.SerializeToString()) else: logging.info(stats) status.ClearField('utilization') report_start = time.time() total_occupancy = 0.0 # Periodically clear any old functions we have cached that we are # no longer accepting requests for. del_list = [] for fname in queue: if len(queue[fname]) == 0 and fname not in status.functions: del_list.append(fname) del function_cache[fname] del runtimes[fname] del exec_counts[fname] for fname in del_list: del queue[fname] del_list = [] for tid in finished_executions: if (time.time() - finished_executions[tid]) > 10: del_list.append(tid) for tid in del_list: del finished_executions[tid] # If we are departing and have cleared our queues, let the # management server know, and exit the process. if departing and len(queue) == 0: sckt = pusher_cache.get(utils.get_depart_done_addr(mgmt_ip)) sckt.send_string(ip) # We specifically pass 1 as the exit code when ending our # process so that the wrapper script does not restart us. sys.exit(1)
def executor(ip, mgmt_ip, schedulers, thread_id): logging.basicConfig(filename='log_executor.txt', level=logging.INFO, format='%(asctime)s %(message)s') context = zmq.Context(1) poller = zmq.Poller() pin_socket = context.socket(zmq.PULL) pin_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.PIN_PORT + thread_id)) unpin_socket = context.socket(zmq.PULL) unpin_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.UNPIN_PORT + thread_id)) exec_socket = context.socket(zmq.PULL) exec_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.FUNC_EXEC_PORT + thread_id)) dag_queue_socket = context.socket(zmq.PULL) dag_queue_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.DAG_QUEUE_PORT + thread_id)) dag_exec_socket = context.socket(zmq.PULL) dag_exec_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.DAG_EXEC_PORT + thread_id)) self_depart_socket = context.socket(zmq.PULL) self_depart_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.SELF_DEPART_PORT + thread_id)) pusher_cache = SocketCache(context, zmq.PUSH) poller = zmq.Poller() poller.register(pin_socket, zmq.POLLIN) poller.register(unpin_socket, zmq.POLLIN) poller.register(exec_socket, zmq.POLLIN) poller.register(dag_queue_socket, zmq.POLLIN) poller.register(dag_exec_socket, zmq.POLLIN) poller.register(self_depart_socket, zmq.POLLIN) # If the management IP is set to None, that means that we are running in # local mode, so we use a regular AnnaTcpClient rather than an IPC client. if mgmt_ip: client = AnnaIpcClient(thread_id, context) else: client = AnnaTcpClient('127.0.0.1', '127.0.0.1', local=True, offset=1) user_library = CloudburstUserLibrary(context, pusher_cache, ip, thread_id, client) status = ThreadStatus() status.ip = ip status.tid = thread_id status.running = True utils.push_status(schedulers, pusher_cache, status) departing = False # Maintains a request queue for each function pinned on this executor. Each # function will have a set of request IDs mapped to it, and this map stores # a schedule for each request ID. queue = {} # Tracks the actual function objects that are pinned to this executor. function_cache = {} # Tracks runtime cost of excuting a DAG function. runtimes = {} # If multiple triggers are necessary for a function, track the triggers as # we receive them. This is also used if a trigger arrives before its # corresponding schedule. received_triggers = {} # Tracks when we received a function request, so we can report end-to-end # latency for the whole executio. receive_times = {} # Tracks the number of requests we are finishing for each function pinned # here. exec_counts = {} # Tracks the end-to-end runtime of each DAG request for which we are the # sink function. dag_runtimes = {} # A map with KVS keys and their corresponding deserialized payloads. cache = {} # Internal metadata to track thread utilization. report_start = time.time() event_occupancy = { 'pin': 0.0, 'unpin': 0.0, 'func_exec': 0.0, 'dag_queue': 0.0, 'dag_exec': 0.0 } total_occupancy = 0.0 while True: socks = dict(poller.poll(timeout=1000)) if pin_socket in socks and socks[pin_socket] == zmq.POLLIN: work_start = time.time() pin(pin_socket, pusher_cache, client, status, function_cache, runtimes, exec_counts, user_library) utils.push_status(schedulers, pusher_cache, status) elapsed = time.time() - work_start event_occupancy['pin'] += elapsed total_occupancy += elapsed if unpin_socket in socks and socks[unpin_socket] == zmq.POLLIN: work_start = time.time() unpin(unpin_socket, status, function_cache, runtimes, exec_counts) utils.push_status(schedulers, pusher_cache, status) elapsed = time.time() - work_start event_occupancy['unpin'] += elapsed total_occupancy += elapsed if exec_socket in socks and socks[exec_socket] == zmq.POLLIN: work_start = time.time() exec_function(exec_socket, client, user_library, cache, function_cache) user_library.close() utils.push_status(schedulers, pusher_cache, status) elapsed = time.time() - work_start event_occupancy['func_exec'] += elapsed total_occupancy += elapsed if dag_queue_socket in socks and socks[dag_queue_socket] == zmq.POLLIN: work_start = time.time() schedule = DagSchedule() schedule.ParseFromString(dag_queue_socket.recv()) fname = schedule.target_function logging.info('Received a schedule for DAG %s (%s), function %s.' % (schedule.dag.name, schedule.id, fname)) if fname not in queue: queue[fname] = {} queue[fname][schedule.id] = schedule if (schedule.id, fname) not in receive_times: receive_times[(schedule.id, fname)] = time.time() # In case we receive the trigger before we receive the schedule, we # can trigger from this operation as well. trkey = (schedule.id, fname) if (trkey in received_triggers and (len(received_triggers[trkey]) == len(schedule.triggers))): exec_dag_function(pusher_cache, client, received_triggers[trkey], function_cache[fname], schedule, user_library, dag_runtimes, cache) user_library.close() del received_triggers[trkey] del queue[fname][schedule.id] fend = time.time() fstart = receive_times[(schedule.id, fname)] runtimes[fname].append(fend - fstart) exec_counts[fname] += 1 elapsed = time.time() - work_start event_occupancy['dag_queue'] += elapsed total_occupancy += elapsed if dag_exec_socket in socks and socks[dag_exec_socket] == zmq.POLLIN: work_start = time.time() trigger = DagTrigger() trigger.ParseFromString(dag_exec_socket.recv()) fname = trigger.target_function logging.info('Received a trigger for schedule %s, function %s.' % (trigger.id, fname)) key = (trigger.id, fname) if key not in received_triggers: received_triggers[key] = {} if (trigger.id, fname) not in receive_times: receive_times[(trigger.id, fname)] = time.time() received_triggers[key][trigger.source] = trigger if fname in queue and trigger.id in queue[fname]: schedule = queue[fname][trigger.id] if len(received_triggers[key]) == len(schedule.triggers): exec_dag_function(pusher_cache, client, received_triggers[key], function_cache[fname], schedule, user_library, dag_runtimes, cache) user_library.close() del received_triggers[key] del queue[fname][trigger.id] fend = time.time() fstart = receive_times[(trigger.id, fname)] runtimes[fname].append(fend - fstart) exec_counts[fname] += 1 elapsed = time.time() - work_start event_occupancy['dag_exec'] += elapsed total_occupancy += elapsed if self_depart_socket in socks and socks[self_depart_socket] == \ zmq.POLLIN: # This message does not matter. self_depart_socket.recv() logging.info('Preparing to depart. No longer accepting requests ' + 'and clearing all queues.') status.ClearField('functions') status.running = False utils.push_status(schedulers, pusher_cache, status) departing = True # periodically report function occupancy report_end = time.time() if report_end - report_start > REPORT_THRESH: cache.clear() utilization = total_occupancy / (report_end - report_start) status.utilization = utilization # Periodically report my status to schedulers with the utilization # set. utils.push_status(schedulers, pusher_cache, status) logging.info('Total thread occupancy: %.6f' % (utilization)) for event in event_occupancy: occ = event_occupancy[event] / (report_end - report_start) logging.info('\tEvent %s occupancy: %.6f' % (event, occ)) event_occupancy[event] = 0.0 stats = ExecutorStatistics() for fname in runtimes: if exec_counts[fname] > 0: fstats = stats.functions.add() fstats.name = fname fstats.call_count = exec_counts[fname] fstats.runtime.extend(runtimes[fname]) runtimes[fname].clear() exec_counts[fname] = 0 for dname in dag_runtimes: dstats = stats.dags.add() dstats.name = dname dstats.runtimes.extend(dag_runtimes[dname]) dag_runtimes[dname].clear() # If we are running in cluster mode, mgmt_ip will be set, and we # will report our status and statistics to it. Otherwise, we will # write to the local conf file if mgmt_ip: sckt = pusher_cache.get( sutils.get_statistics_report_address(mgmt_ip)) sckt.send(stats.SerializeToString()) sckt = pusher_cache.get(utils.get_util_report_address(mgmt_ip)) sckt.send(status.SerializeToString()) else: logging.info(stats) status.ClearField('utilization') report_start = time.time() total_occupancy = 0.0 # Periodically clear any old functions we have cached that we are # no longer accepting requests for. del_list = [] for fname in queue: if len(queue[fname]) == 0 and fname not in status.functions: del_list.append(fname) del function_cache[fname] del runtimes[fname] del exec_counts[fname] for fname in del_list: del queue[fname] # If we are departing and have cleared our queues, let the # management server know, and exit the process. if departing and len(queue) == 0: sckt = pusher_cache.get(utils.get_depart_done_addr(mgmt_ip)) sckt.send_string(ip) # We specifically pass 1 as the exit code when ending our # process so that the wrapper script does not restart us. os._exit(1)