def fetch_and_register_remote_function(self, key): """Import a remote function.""" vals = self._worker.gcs_client.internal_kv_get( key, KV_NAMESPACE_FUNCTION_TABLE) if vals is None: vals = {} else: vals = pickle.loads(vals) fields = [ "job_id", "function_id", "function_name", "function", "module", "max_calls" ] (job_id_str, function_id_str, function_name, serialized_function, module, max_calls) = (vals.get(field) for field in fields) function_id = ray.FunctionID(function_id_str) job_id = ray.JobID(job_id_str) max_calls = int(max_calls) # This function is called by ImportThread. This operation needs to be # atomic. Otherwise, there is race condition. Another thread may use # the temporary function above before the real function is ready. with self.lock: self._num_task_executions[function_id] = 0 try: function = pickle.loads(serialized_function) except Exception: # If an exception was thrown when the remote function was # imported, we record the traceback and notify the scheduler # of the failure. traceback_str = format_error_message(traceback.format_exc()) def f(*args, **kwargs): raise RuntimeError( "The remote function failed to import on the " "worker. This may be because needed library " "dependencies are not installed in the worker " "environment:\n\n{}".format(traceback_str)) # Use a placeholder method when function pickled failed self._function_execution_info[function_id] = ( FunctionExecutionInfo( function=f, function_name=function_name, max_calls=max_calls)) # Log the error message. Log at DEBUG level to avoid overly # spamming the log on import failure. The user gets the error # via the RuntimeError message above. logger.debug("Failed to unpickle the remote function " f"'{function_name}' with " f"function ID {function_id.hex()}. " f"Job ID:{job_id}." f"Traceback:\n{traceback_str}. ") else: # The below line is necessary. Because in the driver process, # if the function is defined in the file where the python # script was started from, its module is `__main__`. # However in the worker process, the `__main__` module is a # different module, which is `default_worker.py` function.__module__ = module self._function_execution_info[function_id] = ( FunctionExecutionInfo( function=function, function_name=function_name, max_calls=max_calls))
def fetch_and_register_remote_function(self, key): """Import a remote function.""" (job_id_str, function_id_str, function_name, serialized_function, num_return_vals, module, resources, max_calls) = self._worker.redis_client.hmget(key, [ "job_id", "function_id", "name", "function", "num_return_vals", "module", "resources", "max_calls" ]) function_id = ray.FunctionID(function_id_str) job_id = ray.JobID(job_id_str) function_name = decode(function_name) max_calls = int(max_calls) module = decode(module) # This is a placeholder in case the function can't be unpickled. This # will be overwritten if the function is successfully registered. def f(): raise Exception("This function was not imported properly.") # This function is called by ImportThread. This operation needs to be # atomic. Otherwise, there is race condition. Another thread may use # the temporary function above before the real function is ready. with self.lock: self._function_execution_info[job_id][function_id] = ( FunctionExecutionInfo( function=f, function_name=function_name, max_calls=max_calls)) self._num_task_executions[job_id][function_id] = 0 try: function = pickle.loads(serialized_function) except Exception: # If an exception was thrown when the remote function was # imported, we record the traceback and notify the scheduler # of the failure. traceback_str = format_error_message(traceback.format_exc()) # Log the error message. push_error_to_driver( self._worker, ray_constants.REGISTER_REMOTE_FUNCTION_PUSH_ERROR, "Failed to unpickle the remote function '{}' with " "function ID {}. Traceback:\n{}".format( function_name, function_id.hex(), traceback_str), job_id=job_id) else: # The below line is necessary. Because in the driver process, # if the function is defined in the file where the python # script was started from, its module is `__main__`. # However in the worker process, the `__main__` module is a # different module, which is `default_worker.py` function.__module__ = module self._function_execution_info[job_id][function_id] = ( FunctionExecutionInfo( function=function, function_name=function_name, max_calls=max_calls)) # Add the function to the function table. self._worker.redis_client.rpush( b"FunctionTable:" + function_id.binary(), self._worker.worker_id)
def fetch_and_register_remote_function(self, key): """Import a remote function.""" (job_id_str, function_id_str, function_name, serialized_function, module, max_calls) = self._worker.redis_client.hmget( key, [ "job_id", "function_id", "function_name", "function", "module", "max_calls" ]) if ray_constants.ISOLATE_EXPORTS and \ job_id_str != self._worker.current_job_id.binary(): # A worker only executes tasks from the assigned job. # TODO(jjyao): If fetching unrelated remote functions # becomes a perf issue, we can also consider having export # queue per job. return function_id = ray.FunctionID(function_id_str) job_id = ray.JobID(job_id_str) function_name = decode(function_name) max_calls = int(max_calls) module = decode(module) # This function is called by ImportThread. This operation needs to be # atomic. Otherwise, there is race condition. Another thread may use # the temporary function above before the real function is ready. with self.lock: self._num_task_executions[function_id] = 0 try: function = pickle.loads(serialized_function) except Exception: # If an exception was thrown when the remote function was # imported, we record the traceback and notify the scheduler # of the failure. traceback_str = format_error_message(traceback.format_exc()) def f(*args, **kwargs): raise RuntimeError( "The remote function failed to import on the " "worker. This may be because needed library " "dependencies are not installed in the worker " "environment:\n\n{}".format(traceback_str)) # Use a placeholder method when function pickled failed self._function_execution_info[function_id] = ( FunctionExecutionInfo( function=f, function_name=function_name, max_calls=max_calls)) # Log the error message. Log at DEBUG level to avoid overly # spamming the log on import failure. The user gets the error # via the RuntimeError message above. logger.debug("Failed to unpickle the remote function " f"'{function_name}' with " f"function ID {function_id.hex()}. " f"Job ID:{job_id}." f"Traceback:\n{traceback_str}. ") else: # The below line is necessary. Because in the driver process, # if the function is defined in the file where the python # script was started from, its module is `__main__`. # However in the worker process, the `__main__` module is a # different module, which is `default_worker.py` function.__module__ = module self._function_execution_info[function_id] = ( FunctionExecutionInfo( function=function, function_name=function_name, max_calls=max_calls)) # Add the function to the function table. self._worker.redis_client.rpush( b"FunctionTable:" + function_id.binary(), self._worker.worker_id)