def _load(self): loads = [] # load in the same order as in self._ins. for var_name in self._ins: if var_name not in self._parameters: loads.append(marshal_utils.load(var_name)) else: loads.append(self._parameters[var_name].param_value) return loads
def unmarshal_data(source_notebook_path): """Unmarshal data from the marshal directory.""" source_notebook_path = os.path.expanduser(source_notebook_path) kale_marshal_dir = _get_kale_marshal_dir(source_notebook_path) if not os.path.exists(kale_marshal_dir): return {} marshal.set_data_dir(kale_marshal_dir) return {os.path.splitext(f)[0]: marshal.load(os.path.splitext(f)[0]) for f in os.listdir(kale_marshal_dir)}
def _load_transformer_assets(self): marshal.set_data_dir(serveutils.TRANSFORMER_ASSETS_DIR) log.info("Loading transformer function...") _fn = marshal.load(serveutils.TRANSFORMER_FN_ASSET_NAME) # create a new function monkey patching the original function's # __globals__. The marshalled function would not be scoped under # the current module, thus its __globals__ dict would be empty. # In this way we create the same function but binding it to the # module's globals(). self.fn = types.FunctionType(_fn.__code__, globals(), _fn.__name__, _fn.__defaults__, _fn.__closure__) log.info("Processing source notebook for imports and functions...") processor = NotebookProcessor(nb_path=os.path.join( serveutils.TRANSFORMER_ASSETS_DIR, serveutils.TRANSFORMER_SRC_NOTEBOOK_NAME), skip_validation=True) self.init_code = processor.get_imports_and_functions() log.info("Initialization code:\n%s" % self.init_code) log.info("Running initialization code...") exec(self.init_code, globals()) log.info("Loading transformer's assets...") for file in os.listdir(serveutils.TRANSFORMER_ASSETS_DIR): if file in [ serveutils.TRANSFORMER_SRC_NOTEBOOK_NAME, serveutils.TRANSFORMER_FN_ASSET_NAME ]: continue # The marshal mechanism works by looking at the name of the files # without extensions. basename = os.path.splitext(file)[0] # remove extension self.assets[basename] = marshal.load(basename) log.info("Assets successfully loaded: %s" % self.assets.keys()) log.info("Initializing assets...") for asset_name, asset_value in self.assets.items(): globals()[asset_name] = asset_value