def guess_preprocessor(entry_point, *args, **kwargs): if get_execution_obj_type(entry_point) != ObjectType.NOT_SUPPORTED: return FunctionPreProcessor(entry_point, *args, **kwargs) else: # TODO (karthikv2k) Handle other entrypoints like python files and full notebooks raise NotImplementedError( "obj param should be a function or a class, got {}".format(type(entry_point)))
def __init__(self, function_obj, path_prefix=constants.DEFAULT_DEST_PREFIX, output_map=None, input_files=[]): super().__init__(output_map=output_map, path_prefix=path_prefix, input_files=input_files) if get_execution_obj_type(function_obj) == ObjectType.NOT_SUPPORTED: raise RuntimeError("Object must of type function or a class") fairing_dir = os.path.dirname(fairing.__file__) self.output_map[os.path.join(fairing_dir, "functions", FUNCTION_SHIM)] = \ os.path.join(path_prefix, FUNCTION_SHIM) # Make sure the user code can be imported as a module self.output_map[os.path.join(fairing_dir, '__init__.py')] = \ os.path.join(path_prefix, '__init__.py') # Make sure fairing can use imported as a module self.output_map[os.path.join(fairing_dir, '__init__.py')] = \ os.path.join(path_prefix, "fairing", '__init__.py') # Make sure cloudpickle can be imported as a module cloudpickle_dir = os.path.dirname(cloudpickle.__file__) self.output_map[os.path.join(cloudpickle_dir, '__init__.py')] = \ os.path.join(path_prefix, "cloudpickle", '__init__.py') self.output_map[os.path.join(cloudpickle_dir, 'cloudpickle.py')] = \ os.path.join(path_prefix, "cloudpickle", 'cloudpickle.py') _, temp_payload_file = tempfile.mkstemp() with open(temp_payload_file, "wb") as f: cloudpickle.dump(function_obj, f) # Adding the serialized file to the context payload_file_in_context = os.path.join(path_prefix, SERIALIZED_FN_FILE) self.output_map[temp_payload_file] = payload_file_in_context # TODO(@karthikv2k): Ref #122 Find a better way to support deployer specific preprocessing _, temp_payload_wrapper_file = tempfile.mkstemp() with open(temp_payload_wrapper_file, "w") as f: contents = OUTPUT_FILE.format( OBJ_NAME=function_obj.__name__, SERIALIZED_FN_FILE=SERIALIZED_FN_FILE) f.write(contents) # Adding the serialized file to the context payload_wrapper_file_in_context = os.path.join( path_prefix, function_obj.__name__ + ".py") self.output_map[ temp_payload_wrapper_file] = payload_wrapper_file_in_context self.command = [ "python", os.path.join(self.path_prefix, FUNCTION_SHIM), "--serialized_fn_file", payload_file_in_context ]
def __init__(self, function_obj, path_prefix=constants.DEFAULT_DEST_PREFIX, output_map=None, input_files=None): super().__init__( output_map=output_map, path_prefix=path_prefix, input_files=input_files) if not notebook_util.is_in_notebook(): logger.warning("The FunctionPreProcessor is optimized for using in a notebook or IPython environment. " "For it to work, the python version should be same for both local python and the python in " "the docker. Please look at alternatives like BasePreprocessor or FullNotebookPreprocessor.") if get_execution_obj_type(function_obj) == ObjectType.NOT_SUPPORTED: raise RuntimeError("Object must of type function or a class") fairing_dir = os.path.dirname(fairing.__file__) self.output_map[os.path.join(fairing_dir, "functions", FUNCTION_SHIM)] = \ os.path.join(path_prefix, FUNCTION_SHIM) # Make sure fairing can use imported as a module self.output_map[os.path.join(fairing_dir, '__init__.py')] = \ os.path.join(path_prefix, "fairing", '__init__.py') # Make sure cloudpickle can be imported as a module cloudpickle_dir = os.path.dirname(cloudpickle.__file__) self.output_map[os.path.join(cloudpickle_dir, '__init__.py')] = \ os.path.join(path_prefix, "cloudpickle", '__init__.py') self.output_map[os.path.join(cloudpickle_dir, 'cloudpickle.py')] = \ os.path.join(path_prefix, "cloudpickle", 'cloudpickle.py') _, temp_payload_file = tempfile.mkstemp() with open(temp_payload_file, "wb") as f: cloudpickle.dump(function_obj, f) # Adding the serialized file to the context payload_file_in_context = os.path.join(path_prefix, SERIALIZED_FN_FILE) self.output_map[temp_payload_file] = payload_file_in_context # TODO(@karthikv2k): Ref #122 Find a better way to support deployer specific preprocessing _, temp_payload_wrapper_file = tempfile.mkstemp() with open(temp_payload_wrapper_file, "w") as f: contents = OUTPUT_FILE.format(OBJ_NAME=function_obj.__name__, SERIALIZED_FN_FILE=SERIALIZED_FN_FILE) f.write(contents) # Adding the serialized file to the context payload_wrapper_file_in_context = os.path.join(path_prefix, function_obj.__name__ + ".py") self.output_map[temp_payload_wrapper_file] = payload_wrapper_file_in_context local_python_version = ".".join([str(x) for x in sys.version_info[0:3]]) self.command = ["python", os.path.join(self.path_prefix, FUNCTION_SHIM), "--serialized_fn_file", payload_file_in_context, "--python_version", local_python_version]
def guess_preprocessor(entry_point, input_files, output_map): if get_execution_obj_type(entry_point) != ObjectType.NOT_SUPPORTED: return FunctionPreProcessor(function_obj=entry_point, input_files=input_files, output_map=output_map) elif isinstance(entry_point, str) and entry_point.endswith(".py"): input_files.append(entry_point) return BasePreProcessor(executable=entry_point, input_files=input_files, output_map=output_map) elif isinstance(entry_point, str) and entry_point.endswith(".ipynb"): return FullNotebookPreProcessor(notebook_file=entry_point, input_files=input_files , output_map=output_map) else: # TODO (karthikv2k) Handle other entrypoints like python files and full notebooks raise NotImplementedError( "obj param should be a function or a class, got {}".format(type(entry_point)))