def simple_executor(f_all, args_all, kwargs_all, num_tasks): serialization_times = [] exec_times = [] results = [] for i in range(num_tasks): task_id = i start_time = time.time() buf = pack_apply_message(f=next(f_all), args=next(args_all), kwargs=next(kwargs_all), buffer_threshold=1024 * 1024, item_threshold=1024) serialization_times.append(time.time() - start_time) start_time = time.time() user_ns = locals() user_ns.update({'__builtins__': __builtins__}) f, args, kwargs = unpack_apply_message(buf, user_ns, copy=False) result = execute_task(f, args, kwargs, user_ns) exec_times.append(time.time() - start_time) results.append(result) avg_serialization_time = sum(serialization_times) / len( serialization_times) * 10**6 avg_execution_time = sum(exec_times) / len(exec_times) * 10**6 return { "avg_serialization_time": avg_serialization_time, "avg_execution_time": avg_execution_time, "results": results }
def dealer_worker(worker_id, ip="localhost", port=5560): context = zmq.Context() socket = context.socket(zmq.REP) socket.connect("tcp://{}:{}".format(ip, port)) print("Starting worker {}".format(worker_id)) task_ids_received = [] while True: bufs = socket.recv_multipart() task_id = int.from_bytes(bufs[0], "little") task_ids_received.append(task_id) user_ns = locals() user_ns.update({'__builtins__': __builtins__}) f, args, kwargs = unpack_apply_message(bufs[1:], user_ns, copy=False) logger.debug("Worker {} received task {}".format(worker_id, task_id)) result = execute_task(f, args, kwargs, user_ns) logger.debug("Worker result: {}".format(result)) reply = {"result": result, "worker_id": worker_id} socket.send_multipart([bufs[0]] + serialize_object(reply)) print("Worker {} received {} tasks".format(worker_id, len(task_ids_received)))
def worker(worker_id, task_url, debug=True, logdir="workers", uid="1"): """ TODO: docstring TODO : Cleanup debug, logdir and uid to function correctly """ start_file_logger('{}/{}/worker_{}.log'.format(logdir, uid, worker_id), 0, level=logging.DEBUG if debug is True else logging.INFO) logger.info("Starting worker {}".format(worker_id)) task_ids_received = [] message_q = zmq_pipes.WorkerMessages(task_url) while True: print("Worker loop iteration starting") task_id, buf = message_q.get() task_ids_received.append(task_id) user_ns = locals() user_ns.update({'__builtins__': __builtins__}) f, args, kwargs = unpack_apply_message(buf, user_ns, copy=False) logger.debug("Worker {} received task {}".format(worker_id, task_id)) result = execute_task(f, args, kwargs, user_ns) logger.debug("Worker {} completed task {}".format(worker_id, task_id)) reply = {"result": result, "worker_id": worker_id} message_q.put(task_id, serialize_object(reply)) logger.debug("Result sent")
def StandAloneServer(data_dir='.', server_id=0): context = zmq.Context() socket = context.socket(zmq.REP) socket.connect('tcp://localhost:5555') while True: message = socket.recv().decode('utf-8') #print('Received message : %s' % message) try: filepath = os.path.join(data_dir, message) input_function = open(filepath, "rb") #print('File opened') function_tuple = pickle.load(input_function) input_function.close() except Exception: exit(2) user_ns = locals() user_ns.update({'__builtins__': __builtins__}) f, args, kwargs = unpack_apply_message(function_tuple, user_ns, copy=False) result = f(*args, **kwargs) # TODO: Support for large object socket.send(serialize_object(result)[0]) print('Server %d Finish Sending Result : %s' %(server_id, message))
def do_apply(self, content, bufs, msg_id, reply_metadata): shell = self.shell try: working = shell.user_ns prefix = "_" + str(msg_id).replace("-", "") + "_" f, args, kwargs = unpack_apply_message(bufs, working, copy=False) fname = getattr(f, '__name__', 'f') fname = prefix + "f" argname = prefix + "args" kwargname = prefix + "kwargs" resultname = prefix + "result" ns = {fname: f, argname: args, kwargname: kwargs, resultname: None} # print ns working.update(ns) code = "%s = %s(*%s,**%s)" % (resultname, fname, argname, kwargname) try: exec(code, shell.user_global_ns, shell.user_ns) result = working.get(resultname) finally: for key in ns: working.pop(key) result_buf = serialize_object( result, buffer_threshold=self.session.buffer_threshold, item_threshold=self.session.item_threshold, ) except: # invoke IPython traceback formatting shell.showtraceback() # FIXME - fish exception info out of shell, possibly left there by # run_code. We'll need to clean up this logic later. reply_content = {} if shell._reply_content is not None: reply_content.update(shell._reply_content) # reset after use shell._reply_content = None self.send_response(self.iopub_socket, u'error', reply_content, ident=self._topic('error')) self.log.info("Exception in apply request:\n%s", '\n'.join(reply_content['traceback'])) result_buf = [] else: reply_content = {'status': 'ok'} return reply_content, result_buf
def do_apply(self, content, bufs, msg_id, reply_metadata): shell = self.shell try: working = shell.user_ns prefix = "_"+str(msg_id).replace("-","")+"_" f,args,kwargs = unpack_apply_message(bufs, working, copy=False) fname = getattr(f, '__name__', 'f') fname = prefix+"f" argname = prefix+"args" kwargname = prefix+"kwargs" resultname = prefix+"result" ns = { fname : f, argname : args, kwargname : kwargs , resultname : None } # print ns working.update(ns) code = "%s = %s(*%s,**%s)" % (resultname, fname, argname, kwargname) try: exec(code, shell.user_global_ns, shell.user_ns) result = working.get(resultname) finally: for key in ns: working.pop(key) result_buf = serialize_object(result, buffer_threshold=self.session.buffer_threshold, item_threshold=self.session.item_threshold, ) except: # invoke IPython traceback formatting shell.showtraceback() # FIXME - fish exception info out of shell, possibly left there by # run_code. We'll need to clean up this logic later. reply_content = {} if shell._reply_content is not None: reply_content.update(shell._reply_content) # reset after use shell._reply_content = None self.send_response(self.iopub_socket, u'error', reply_content, ident=self._topic('error')) self.log.info("Exception in apply request:\n%s", '\n'.join(reply_content['traceback'])) result_buf = [] else: reply_content = {'status' : 'ok'} return reply_content, result_buf
def execute_task(bufs): """Deserialize the buffer and execute the task. Returns the result or throws exception. """ logger.debug("Inside execute_task function") user_ns = locals() user_ns.update({'__builtins__': __builtins__}) logger.debug(bufs) f, actual_args, kwargs = unpack_apply_message(bufs, user_ns, copy=False) logger.debug("Message unpacked") # We might need to look into callability of the function from itself # since we change it's name in the new namespace prefix = "parsl_" fname = prefix + "f" argname = prefix + "args" kwargname = prefix + "kwargs" resultname = prefix + "result" user_ns.update({ fname: f, argname: actual_args, kwargname: kwargs, resultname: resultname }) logger.debug("Namespace updated") code = "{0} = {1}(*{2}, **{3})".format(resultname, fname, argname, kwargname) try: exec(code, user_ns, user_ns) except Exception as e: raise e else: return user_ns.get(resultname)
def execute_task(bufs): ''' Deserialize the buf, and execute the task. Returns the serialized result/exception ''' all_names = dir(__builtins__) user_ns = locals() user_ns.update( {'__builtins__': {k: getattr(__builtins__, k) for k in all_names}}) f, args, kwargs = unpack_apply_message(bufs, user_ns, copy=False) fname = getattr(f, '__name__', 'f') prefix = "parsl_" fname = prefix + "f" argname = prefix + "args" kwargname = prefix + "kwargs" resultname = prefix + "result" user_ns.update({ fname: f, argname: args, kwargname: kwargs, resultname: resultname }) code = "{0} = {1}(*{2}, **{3})".format(resultname, fname, argname, kwargname) try: print("[RUNNER] Executing : {0}".format(code)) exec(code, user_ns, user_ns) except Exception as e: logger.warning("Caught errors but will not handled %s", e) raise e else: # print("Done : {0}".format(locals())) print("[RUNNER] Result : {0}".format(user_ns.get(resultname))) return user_ns.get(resultname)
def execute_task(bufs): """Deserialize the buffer and execute the task. Returns the result or throws exception. """ user_ns = locals() user_ns.update({'__builtins__': __builtins__}) f, args, kwargs = unpack_apply_message(bufs, user_ns, copy=False) # We might need to look into callability of the function from itself # since we change it's name in the new namespace prefix = "parsl_" fname = prefix + "f" argname = prefix + "args" kwargname = prefix + "kwargs" resultname = prefix + "result" user_ns.update({ fname: f, argname: args, kwargname: kwargs, resultname: resultname }) code = "{0} = {1}(*{2}, **{3})".format(resultname, fname, argname, kwargname) try: # logger.debug("[RUNNER] Executing: {0}".format(code)) exec(code, user_ns, user_ns) except Exception as e: logger.warning("Caught exception; will raise it: {}".format(e), exc_info=True) raise e else: # logger.debug("[RUNNER] Result: {0}".format(user_ns.get(resultname))) return user_ns.get(resultname)
def execute_task(bufs): """Deserialize the buffer and execute the task. Returns the serialized result or exception. """ user_ns = locals() user_ns.update({'__builtins__': __builtins__}) f, args, kwargs = unpack_apply_message(bufs, user_ns, copy=False) fname = getattr(f, '__name__', 'f') prefix = "parsl_" fname = prefix + "f" argname = prefix + "args" kwargname = prefix + "kwargs" resultname = prefix + "result" user_ns.update({ fname: f, argname: args, kwargname: kwargs, resultname: resultname }) code = "{0} = {1}(*{2}, **{3})".format(resultname, fname, argname, kwargname) try: logger.debug("[RUNNER] Executing: {0}".format(code)) exec(code, user_ns, user_ns) except Exception as e: logger.warning("Caught exception; will raise it: {}".format(e)) raise e else: logger.debug("[RUNNER] Result: {0}".format( user_ns.get(resultname))) return user_ns.get(resultname)
def do_apply(self, content, bufs, msg_id, reply_metadata): shell = self.shell try: working = shell.user_ns prefix = "_" + str(msg_id).replace("-", "") + "_" f, args, kwargs = unpack_apply_message(bufs, working, copy=False) fname = getattr(f, '__name__', 'f') fname = prefix + "f" argname = prefix + "args" kwargname = prefix + "kwargs" resultname = prefix + "result" ns = {fname: f, argname: args, kwargname: kwargs, resultname: None} # print ns working.update(ns) code = "%s = %s(*%s,**%s)" % (resultname, fname, argname, kwargname) try: exec(code, shell.user_global_ns, shell.user_ns) result = working.get(resultname) finally: for key in ns: working.pop(key) result_buf = serialize_object( result, buffer_threshold=self.session.buffer_threshold, item_threshold=self.session.item_threshold, ) except BaseException as e: # invoke IPython traceback formatting shell.showtraceback() reply_content = { 'traceback': [], 'ename': unicode_type(type(e).__name__), 'evalue': safe_unicode(e), } # get formatted traceback, which ipykernel recorded if hasattr(shell, '_last_traceback'): # ipykernel 4.4 reply_content['traceback'] = shell._last_traceback or [] elif hasattr(shell, '_reply_content'): # ipykernel <= 4.3 if shell._reply_content and 'traceback' in shell._reply_content: reply_content['traceback'] = shell._reply_content[ 'traceback'] else: self.log.warning("Didn't find a traceback where I expected to") shell._last_traceback = None e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='apply') reply_content['engine_info'] = e_info self.send_response(self.iopub_socket, u'error', reply_content, ident=self._topic('error')) self.log.info("Exception in apply request:\n%s", '\n'.join(reply_content['traceback'])) result_buf = [] reply_content['status'] = 'error' else: reply_content = {'status': 'ok'} return reply_content, result_buf
def unpack_byte_code_function(function_info, user_namespace): from ipyparallel.serialize import unpack_apply_message func, args, kwargs = unpack_apply_message(function_info["byte code"], user_namespace, copy=False) return (func, 'parsl_function_name', args, kwargs)
index += 1 except Exception as e: print(e) exit(1) try: input_function = open(input_function_file, "rb") function_tuple = pickle.load(input_function) input_function.close() except Exception as e: print(e) exit(2) user_ns = locals() user_ns.update({'__builtins__': __builtins__}) f, args, kwargs = unpack_apply_message(function_tuple, user_ns, copy=False) mapping = {} try: if shared_fs is False and remapping_string is not None: for i in remapping_string.split(","): split_mapping = i.split(":") mapping[split_mapping[0]] = split_mapping[1] func_inputs = kwargs.get("inputs", []) for inp in func_inputs: check_file(inp, mapping, file_type_string) for kwarg, potential_f in kwargs.items():
def do_apply(self, content, bufs, msg_id, reply_metadata): shell = self.shell try: working = shell.user_ns prefix = "_"+str(msg_id).replace("-","")+"_" f,args,kwargs = unpack_apply_message(bufs, working, copy=False) fname = getattr(f, '__name__', 'f') fname = prefix+"f" argname = prefix+"args" kwargname = prefix+"kwargs" resultname = prefix+"result" ns = { fname : f, argname : args, kwargname : kwargs , resultname : None } # print ns working.update(ns) code = "%s = %s(*%s,**%s)" % (resultname, fname, argname, kwargname) try: exec(code, shell.user_global_ns, shell.user_ns) result = working.get(resultname) finally: for key in ns: working.pop(key) result_buf = serialize_object(result, buffer_threshold=self.session.buffer_threshold, item_threshold=self.session.item_threshold, ) except BaseException as e: # invoke IPython traceback formatting shell.showtraceback() reply_content = { 'traceback': [], 'ename': unicode_type(type(e).__name__), 'evalue': safe_unicode(e), } # get formatted traceback, which ipykernel recorded if hasattr(shell, '_last_traceback'): # ipykernel 4.4 reply_content['traceback'] = shell._last_traceback or [] elif hasattr(shell, '_reply_content'): # ipykernel <= 4.3 if shell._reply_content and 'traceback' in shell._reply_content: reply_content['traceback'] = shell._reply_content['traceback'] else: self.log.warning("Didn't find a traceback where I expected to") shell._last_traceback = None e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='apply') reply_content['engine_info'] = e_info self.send_response(self.iopub_socket, u'error', reply_content, ident=self._topic('error')) self.log.info("Exception in apply request:\n%s", '\n'.join(reply_content['traceback'])) result_buf = [] reply_content['status'] = 'error' else: reply_content = {'status' : 'ok'} return reply_content, result_buf
# Load function data try: input_function = open(input_function_file, "rb") function_info = pickle.load(input_function) # Extract information from transferred source code if source: source_code = function_info["source code"] name = function_info["name"] args = function_info["args"] kwargs = function_info["kwargs"] # Extract information from function pointer else: from ipyparallel.serialize import unpack_apply_message func, args, kwargs = unpack_apply_message(function_info, user_ns, copy=False) input_function.close() except Exception as e: print(e) exit(2) # Remapping file names using remapping string mapping = {} try: if shared_fs is False and remapping_string is not None: # Parse the remapping string into a dictionary for i in remapping_string.split(","): split_mapping = i.split(":") mapping[split_mapping[0]] = split_mapping[1]
def do_apply(self, content, bufs, msg_id, reply_metadata): shell = self.shell try: working = shell.user_ns prefix = "_" + str(msg_id).replace("-", "") + "_" f, args, kwargs = unpack_apply_message(bufs, working, copy=False) fname = getattr(f, '__name__', 'f') fname = prefix + "f" argname = prefix + "args" kwargname = prefix + "kwargs" resultname = prefix + "result" ns = {fname: f, argname: args, kwargname: kwargs, resultname: None} # print ns working.update(ns) code = f"{resultname} = {fname}(*{argname},**{kwargname})" try: exec(code, shell.user_global_ns, shell.user_ns) result = working.get(resultname) finally: for key in ns: working.pop(key) result_buf = serialize_object( result, buffer_threshold=self.session.buffer_threshold, item_threshold=self.session.item_threshold, ) except BaseException as e: # invoke IPython traceback formatting # this sends the 'error' message shell.showtraceback() try: str_evalue = str(e) except Exception as str_error: str_evalue = f"Failed to cast exception to string: {str_error}" reply_content = { 'traceback': [], 'ename': str(type(e).__name__), 'evalue': str_evalue, } # get formatted traceback, which ipykernel recorded if hasattr(shell, '_last_traceback'): # ipykernel 4.4 reply_content['traceback'] = shell._last_traceback or [] else: self.log.warning("Didn't find a traceback where I expected to") shell._last_traceback = None e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='apply') reply_content['engine_info'] = e_info self.log.info( "Exception in apply request:\n%s", '\n'.join(reply_content['traceback']) ) result_buf = [] reply_content['status'] = 'error' else: reply_content = {'status': 'ok'} return reply_content, result_buf