def test_exec_dag_sink(self): ''' Tests that the last function in a DAG executes correctly and stores the result in the KVS. ''' def func(_, x): return x * x fname = 'square' arg = 2 dag = create_linear_dag([func], [fname], self.kvs_client, 'dag') schedule, triggers = self._create_fn_schedule(dag, arg, fname, [fname]) exec_dag_function(self.pusher_cache, self.kvs_client, triggers, func, schedule, self.user_library, {}, {}) # Assert that there have been 0 messages sent. self.assertEqual(len(self.socket.outbox), 0) # Retrieve the result, ensure it is a LWWPairLattice, then deserialize # it. result = self.kvs_client.get(schedule.id)[schedule.id] self.assertEqual(type(result), LWWPairLattice) result = serializer.load_lattice(result) # Check that the output is equal to a local function execution. self.assertEqual(result, func('', arg))
def test_exec_causal_dag_sink(self): ''' Tests that the last function in a causal DAG executes correctly and stores the result in the KVS. Also checks to make sure that causal metadata is properly created. ''' def func(_, x): return x * x fname = 'square' arg = 2 dag = create_linear_dag([func], [fname], self.kvs_client, 'dag', MultiKeyCausalLattice) schedule, triggers = self._create_fn_schedule(dag, arg, fname, [fname], MULTI) schedule.output_key = 'output_key' schedule.client_id = '12' # We know that there is only one trigger. We populate dependencies # explicitly in this trigger message to make sure that they are # reflected in the final result. kv = triggers['BEGIN'].dependencies.add() kv.key = 'dependency' DEFAULT_VC.serialize(kv.vector_clock) exec_dag_function(self.pusher_cache, self.kvs_client, triggers, func, schedule, self.user_library, {}, {}) # Assert that there have been 0 messages sent. self.assertEqual(len(self.socket.outbox), 0) # Retrieve the result and check its value and its metadata. result = self.kvs_client.get(schedule.output_key)[schedule.output_key] self.assertEqual(type(result), MultiKeyCausalLattice) # Check that the vector clock of the output corresponds ot the client # ID. self.assertEqual(result.vector_clock, VectorClock({schedule.client_id: 1}, True)) # Check that the dependencies of the output match those specified in # the trigger. self.assertEqual(len(result.dependencies.reveal()), 1) self.assertTrue(kv.key in result.dependencies.reveal()) self.assertEqual(result.dependencies.reveal()[kv.key], DEFAULT_VC) # Check that the output is equal to a local function execution. result = serializer.load_lattice(result)[0] self.assertEqual(result, func('', arg))
def test_exec_causal_dag_non_sink(self): ''' Creates and executes a non-sink function in a causal-mode DAG. This should be exactly the same as the non-causal version of the test, except we ensure that the causal metadata is empty, because we don't have any KVS accesses. ''' # Create two functions intended to be used in sequence. def incr(_, x): x + 1 iname = 'incr' def square(_, x): return x * x sname = 'square' arg = 1 # Create a DAG and a trigger for the first function in the DAG. dag = create_linear_dag([incr, square], [iname, sname], self.kvs_client, 'dag', MultiKeyCausalLattice) schedule, triggers = self._create_fn_schedule(dag, arg, iname, [iname, sname], MULTI) exec_dag_function(self.pusher_cache, self.kvs_client, triggers, incr, schedule, self.user_library, {}, {}) # Assert that there has been a message sent. self.assertEqual(len(self.pusher_cache.socket.outbox), 1) # Extract that message and check its contents. trigger = DagTrigger() trigger.ParseFromString(self.pusher_cache.socket.outbox[0]) self.assertEqual(trigger.id, schedule.id) self.assertEqual(trigger.target_function, sname) self.assertEqual(trigger.source, iname) self.assertEqual(len(trigger.arguments.values), 1) self.assertEqual(len(trigger.version_locations), 0) self.assertEqual(len(trigger.dependencies), 0) val = serializer.load(trigger.arguments.values[0]) self.assertEqual(val, incr('', arg))
def test_exec_dag_non_sink(self): ''' Executes a non-sink function in a DAG and ensures that the correct downstream trigger was sent with a correct execution of the function. ''' # Create two functions intended to be used in sequence. def incr(_, x): x + 1 iname = 'incr' def square(_, x): return x * x sname = 'square' arg = 1 # Create a DAG and a trigger for the first function in the DAG. dag = create_linear_dag([incr, square], [iname, sname], self.kvs_client, 'dag') schedule, triggers = self._create_fn_schedule(dag, arg, iname, [iname, sname]) exec_dag_function(self.pusher_cache, self.kvs_client, triggers, incr, schedule, self.user_library, {}, {}) # Assert that there has been a message sent. self.assertEqual(len(self.pusher_cache.socket.outbox), 1) # Extract that message and check its contents. trigger = DagTrigger() trigger.ParseFromString(self.pusher_cache.socket.outbox[0]) self.assertEqual(trigger.id, schedule.id) self.assertEqual(trigger.target_function, sname) self.assertEqual(trigger.source, iname) self.assertEqual(len(trigger.arguments.values), 1) val = serializer.load(trigger.arguments.values[0]) self.assertEqual(val, incr('', arg))
def test_exec_causal_dag_non_sink_with_ref(self): ''' Creates and executes a non-sink function in a causal-mode DAG. This version accesses a KVS key, so we ensure that data is appropriately cached and the metadata is passed downstream. ''' # Create two functions intended to be used in sequence. def incr(_, x): x + 1 iname = 'incr' def square(_, x): return x * x sname = 'square' # Put tthe argument into the KVS. arg_name = 'arg' arg_value = 1 arg = serializer.dump_lattice(arg_value, MultiKeyCausalLattice) self.kvs_client.put(arg_name, arg) # Create a DAG and a trigger for the first function in the DAG. dag = create_linear_dag([incr, square], [iname, sname], self.kvs_client, 'dag', MultiKeyCausalLattice) schedule, triggers = self._create_fn_schedule( dag, DropletReference(arg_name, True), iname, [iname, sname], MULTI) exec_dag_function(self.pusher_cache, self.kvs_client, triggers, incr, schedule, self.user_library, {}, {}) # Assert that there has been a message sent. self.assertEqual(len(self.pusher_cache.socket.outbox), 1) # Extract that message and check its contents. trigger = DagTrigger() trigger.ParseFromString(self.pusher_cache.socket.outbox[0]) self.assertEqual(trigger.id, schedule.id) self.assertEqual(trigger.target_function, sname) self.assertEqual(trigger.source, iname) self.assertEqual(len(trigger.arguments.values), 1) # Check the metadata of the key that is cached here after execution. locs = trigger.version_locations self.assertEqual(len(locs), 1) self.assertTrue(self.ip in locs.keys()) self.assertEqual(len(locs[self.ip].keys), 1) kv = locs[self.ip].keys[0] self.assertEqual(kv.key, arg_name) self.assertEqual(VectorClock(dict(kv.vector_clock), True), arg.vector_clock) # Check the metatada of the causal dependency passed downstream. self.assertEqual(len(trigger.dependencies), 1) kv = trigger.dependencies[0] self.assertEqual(kv.key, arg_name) self.assertEqual(VectorClock(dict(kv.vector_clock), True), arg.vector_clock) val = serializer.load(trigger.arguments.values[0]) self.assertEqual(val, incr('', arg_value))
def executor(ip, mgmt_ip, schedulers, thread_id): logging.basicConfig(filename='log_executor.txt', level=logging.INFO, format='%(asctime)s %(message)s') context = zmq.Context(1) poller = zmq.Poller() pin_socket = context.socket(zmq.PULL) pin_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.PIN_PORT + thread_id)) unpin_socket = context.socket(zmq.PULL) unpin_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.UNPIN_PORT + thread_id)) exec_socket = context.socket(zmq.PULL) exec_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.FUNC_EXEC_PORT + thread_id)) dag_queue_socket = context.socket(zmq.PULL) dag_queue_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.DAG_QUEUE_PORT + thread_id)) dag_exec_socket = context.socket(zmq.PULL) dag_exec_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.DAG_EXEC_PORT + thread_id)) self_depart_socket = context.socket(zmq.PULL) self_depart_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.SELF_DEPART_PORT + thread_id)) pusher_cache = SocketCache(context, zmq.PUSH) poller = zmq.Poller() poller.register(pin_socket, zmq.POLLIN) poller.register(unpin_socket, zmq.POLLIN) poller.register(exec_socket, zmq.POLLIN) poller.register(dag_queue_socket, zmq.POLLIN) poller.register(dag_exec_socket, zmq.POLLIN) poller.register(self_depart_socket, zmq.POLLIN) # If the management IP is set to None, that means that we are running in # local mode, so we use a regular AnnaTcpClient rather than an IPC client. if mgmt_ip: client = AnnaIpcClient(thread_id, context) else: client = AnnaTcpClient('127.0.0.1', '127.0.0.1', local=True, offset=1) user_library = DropletUserLibrary(context, pusher_cache, ip, thread_id, client) status = ThreadStatus() status.ip = ip status.tid = thread_id status.running = True utils.push_status(schedulers, pusher_cache, status) departing = False # Maintains a request queue for each function pinned on this executor. Each # function will have a set of request IDs mapped to it, and this map stores # a schedule for each request ID. queue = {} # Tracks the actual function objects that are pinned to this executor. pinned_functions = {} # Tracks runtime cost of excuting a DAG function. runtimes = {} # If multiple triggers are necessary for a function, track the triggers as # we receive them. This is also used if a trigger arrives before its # corresponding schedule. received_triggers = {} # Tracks when we received a function request, so we can report end-to-end # latency for the whole executio. receive_times = {} # Tracks the number of requests we are finishing for each function pinned # here. exec_counts = {} # Tracks the end-to-end runtime of each DAG request for which we are the # sink function. dag_runtimes = {} # A map with KVS keys and their corresponding deserialized payloads. cache = {} # Internal metadata to track thread utilization. report_start = time.time() event_occupancy = {'pin': 0.0, 'unpin': 0.0, 'func_exec': 0.0, 'dag_queue': 0.0, 'dag_exec': 0.0} total_occupancy = 0.0 while True: socks = dict(poller.poll(timeout=1000)) if pin_socket in socks and socks[pin_socket] == zmq.POLLIN: work_start = time.time() pin(pin_socket, pusher_cache, client, status, pinned_functions, runtimes, exec_counts) utils.push_status(schedulers, pusher_cache, status) elapsed = time.time() - work_start event_occupancy['pin'] += elapsed total_occupancy += elapsed if unpin_socket in socks and socks[unpin_socket] == zmq.POLLIN: work_start = time.time() unpin(unpin_socket, status, pinned_functions, runtimes, exec_counts) utils.push_status(schedulers, pusher_cache, status) elapsed = time.time() - work_start event_occupancy['unpin'] += elapsed total_occupancy += elapsed if exec_socket in socks and socks[exec_socket] == zmq.POLLIN: work_start = time.time() exec_function(exec_socket, client, user_library, cache) user_library.close() utils.push_status(schedulers, pusher_cache, status) elapsed = time.time() - work_start event_occupancy['func_exec'] += elapsed total_occupancy += elapsed if dag_queue_socket in socks and socks[dag_queue_socket] == zmq.POLLIN: work_start = time.time() schedule = DagSchedule() schedule.ParseFromString(dag_queue_socket.recv()) fname = schedule.target_function logging.info('Received a schedule for DAG %s (%s), function %s.' % (schedule.dag.name, schedule.id, fname)) if fname not in queue: queue[fname] = {} queue[fname][schedule.id] = schedule if (schedule.id, fname) not in receive_times: receive_times[(schedule.id, fname)] = time.time() # In case we receive the trigger before we receive the schedule, we # can trigger from this operation as well. trkey = (schedule.id, fname) if (trkey in received_triggers and (len(received_triggers[trkey]) == len(schedule.triggers))): exec_dag_function(pusher_cache, client, received_triggers[trkey], pinned_functions[fname], schedule, user_library, dag_runtimes, cache) user_library.close() del received_triggers[trkey] del queue[fname][schedule.id] fend = time.time() fstart = receive_times[(schedule.id, fname)] runtimes[fname].append(fend - fstart) exec_counts[fname] += 1 elapsed = time.time() - work_start event_occupancy['dag_queue'] += elapsed total_occupancy += elapsed if dag_exec_socket in socks and socks[dag_exec_socket] == zmq.POLLIN: work_start = time.time() trigger = DagTrigger() trigger.ParseFromString(dag_exec_socket.recv()) fname = trigger.target_function logging.info('Received a trigger for schedule %s, function %s.' % (trigger.id, fname)) key = (trigger.id, fname) if key not in received_triggers: received_triggers[key] = {} if (trigger.id, fname) not in receive_times: receive_times[(trigger.id, fname)] = time.time() received_triggers[key][trigger.source] = trigger if fname in queue and trigger.id in queue[fname]: schedule = queue[fname][trigger.id] if len(received_triggers[key]) == len(schedule.triggers): exec_dag_function(pusher_cache, client, received_triggers[key], pinned_functions[fname], schedule, user_library, dag_runtimes, cache) user_library.close() del received_triggers[key] del queue[fname][trigger.id] fend = time.time() fstart = receive_times[(trigger.id, fname)] runtimes[fname].append(fend - fstart) exec_counts[fname] += 1 elapsed = time.time() - work_start event_occupancy['dag_exec'] += elapsed total_occupancy += elapsed if self_depart_socket in socks and socks[self_depart_socket] == \ zmq.POLLIN: # This message does not matter. self_depart_socket.recv() logging.info('Preparing to depart. No longer accepting requests ' + 'and clearing all queues.') status.ClearField('functions') status.running = False utils.push_status(schedulers, pusher_cache, status) departing = True # periodically report function occupancy report_end = time.time() if report_end - report_start > REPORT_THRESH: cache.clear() utilization = total_occupancy / (report_end - report_start) status.utilization = utilization # Periodically report my status to schedulers with the utilization # set. utils.push_status(schedulers, pusher_cache, status) logging.info('Total thread occupancy: %.6f' % (utilization)) for event in event_occupancy: occ = event_occupancy[event] / (report_end - report_start) logging.info('\tEvent %s occupancy: %.6f' % (event, occ)) event_occupancy[event] = 0.0 stats = ExecutorStatistics() for fname in runtimes: if exec_counts[fname] > 0: fstats = stats.functions.add() fstats.name = fname fstats.call_count = exec_counts[fname] fstats.runtime.extend(runtimes[fname]) runtimes[fname].clear() exec_counts[fname] = 0 for dname in dag_runtimes: dstats = stats.dags.add() dstats.name = dname dstats.runtimes.extend(dag_runtimes[dname]) dag_runtimes[dname].clear() # If we are running in cluster mode, mgmt_ip will be set, and we # will report our status and statistics to it. Otherwise, we will # write to the local conf file if mgmt_ip: sckt = pusher_cache.get(sutils.get_statistics_report_address (mgmt_ip)) sckt.send(stats.SerializeToString()) sckt = pusher_cache.get(utils.get_util_report_address(mgmt_ip)) sckt.send(status.SerializeToString()) else: logging.info(stats) status.ClearField('utilization') report_start = time.time() total_occupancy = 0.0 # Periodically clear any old functions we have cached that we are # no longer accepting requests for. for fname in queue: if len(queue[fname]) == 0 and fname not in status.functions: del queue[fname] del pinned_functions[fname] del runtimes[fname] del exec_counts[fname] # If we are departing and have cleared our queues, let the # management server know, and exit the process. if departing and len(queue) == 0: sckt = pusher_cache.get(utils.get_depart_done_addr(mgmt_ip)) sckt.send_string(ip) # We specifically pass 1 as the exit code when ending our # process so that the wrapper script does not restart us. os._exit(1)