def _load(self):
loads = [] # load in the same order as in self._ins.
for var_name in self._ins:
if var_name not in self._parameters:
loads.append(marshal_utils.load(var_name))
else:
loads.append(self._parameters[var_name].param_value)
return loads
def unmarshal_data(source_notebook_path):
"""Unmarshal data from the marshal directory."""
source_notebook_path = os.path.expanduser(source_notebook_path)
kale_marshal_dir = _get_kale_marshal_dir(source_notebook_path)
if not os.path.exists(kale_marshal_dir):
return {}
marshal.set_data_dir(kale_marshal_dir)
return {os.path.splitext(f)[0]:
marshal.load(os.path.splitext(f)[0])
for f in os.listdir(kale_marshal_dir)}
def _load_transformer_assets(self):
marshal.set_data_dir(serveutils.TRANSFORMER_ASSETS_DIR)
log.info("Loading transformer function...")
_fn = marshal.load(serveutils.TRANSFORMER_FN_ASSET_NAME)
# create a new function monkey patching the original function's
# __globals__. The marshalled function would not be scoped under
# the current module, thus its __globals__ dict would be empty.
# In this way we create the same function but binding it to the
# module's globals().
self.fn = types.FunctionType(_fn.__code__, globals(), _fn.__name__,
_fn.__defaults__, _fn.__closure__)
log.info("Processing source notebook for imports and functions...")
processor = NotebookProcessor(nb_path=os.path.join(
serveutils.TRANSFORMER_ASSETS_DIR,
serveutils.TRANSFORMER_SRC_NOTEBOOK_NAME),
skip_validation=True)
self.init_code = processor.get_imports_and_functions()
log.info("Initialization code:\n%s" % self.init_code)
log.info("Running initialization code...")
exec(self.init_code, globals())
log.info("Loading transformer's assets...")
for file in os.listdir(serveutils.TRANSFORMER_ASSETS_DIR):
if file in [
serveutils.TRANSFORMER_SRC_NOTEBOOK_NAME,
serveutils.TRANSFORMER_FN_ASSET_NAME
]:
continue
# The marshal mechanism works by looking at the name of the files
# without extensions.
basename = os.path.splitext(file)[0] # remove extension
self.assets[basename] = marshal.load(basename)
log.info("Assets successfully loaded: %s" % self.assets.keys())
log.info("Initializing assets...")
for asset_name, asset_value in self.assets.items():
globals()[asset_name] = asset_value