def _prepare_transformer_assets(fn: Callable, assets: Dict = None): notebook_path = jputils.get_notebook_path() processor = NotebookProcessor(nb_path=notebook_path, skip_validation=True) fn_source = astutils.get_function_source(fn, strip_signature=False) missing_names = flakeutils.pyflakes_report( processor.get_imports_and_functions() + "\n" + fn_source) if not assets: assets = dict() if not isinstance(assets, dict): ValueError("Please provide preprocessing assets as a dictionary" " mapping variables *names* to their objects") missing_assets = [x not in assets.keys() for x in missing_names] if any(missing_assets): raise RuntimeError( "The following abjects are a dependency for the" " provided preprocessing function. Please add the" " to the `preprocessing_assets` dictionary: %s" % [a for a, m in zip(missing_names, missing_assets) if m]) # save function and assets utils.clean_dir(TRANSFORMER_ASSETS_DIR) marshal.set_data_dir(TRANSFORMER_ASSETS_DIR) marshal.save(fn, TRANSFORMER_FN_ASSET_NAME) for asset_name, asset_value in assets.items(): marshal.save(asset_value, asset_name) # save notebook as well shutil.copy( notebook_path, os.path.join(TRANSFORMER_ASSETS_DIR, TRANSFORMER_SRC_NOTEBOOK_NAME))
def _load_transformer_assets(self): marshal.set_data_dir(serveutils.TRANSFORMER_ASSETS_DIR) log.info("Loading transformer function...") _fn = marshal.load(serveutils.TRANSFORMER_FN_ASSET_NAME) # create a new function monkey patching the original function's # __globals__. The marshalled function would not be scoped under # the current module, thus its __globals__ dict would be empty. # In this way we create the same function but binding it to the # module's globals(). self.fn = types.FunctionType(_fn.__code__, globals(), _fn.__name__, _fn.__defaults__, _fn.__closure__) log.info("Processing source notebook for imports and functions...") processor = NotebookProcessor(nb_path=os.path.join( serveutils.TRANSFORMER_ASSETS_DIR, serveutils.TRANSFORMER_SRC_NOTEBOOK_NAME), skip_validation=True) self.init_code = processor.get_imports_and_functions() log.info("Initialization code:\n%s" % self.init_code) log.info("Running initialization code...") exec(self.init_code, globals()) log.info("Loading transformer's assets...") for file in os.listdir(serveutils.TRANSFORMER_ASSETS_DIR): if file in [ serveutils.TRANSFORMER_SRC_NOTEBOOK_NAME, serveutils.TRANSFORMER_FN_ASSET_NAME ]: continue # The marshal mechanism works by looking at the name of the files # without extensions. basename = os.path.splitext(file)[0] # remove extension self.assets[basename] = marshal.load(basename) log.info("Assets successfully loaded: %s" % self.assets.keys()) log.info("Initializing assets...") for asset_name, asset_value in self.assets.items(): globals()[asset_name] = asset_value