def _prepare_transformer_assets(fn: Callable, assets: Dict = None):
notebook_path = jputils.get_notebook_path()
processor = NotebookProcessor(nb_path=notebook_path, skip_validation=True)
fn_source = astutils.get_function_source(fn, strip_signature=False)
missing_names = flakeutils.pyflakes_report(
processor.get_imports_and_functions() + "\n" + fn_source)
if not assets:
assets = dict()
if not isinstance(assets, dict):
ValueError("Please provide preprocessing assets as a dictionary"
" mapping variables *names* to their objects")
missing_assets = [x not in assets.keys() for x in missing_names]
if any(missing_assets):
raise RuntimeError(
"The following abjects are a dependency for the"
" provided preprocessing function. Please add the"
" to the `preprocessing_assets` dictionary: %s" %
[a for a, m in zip(missing_names, missing_assets) if m])
# save function and assets
utils.clean_dir(TRANSFORMER_ASSETS_DIR)
marshal.set_data_dir(TRANSFORMER_ASSETS_DIR)
marshal.save(fn, TRANSFORMER_FN_ASSET_NAME)
for asset_name, asset_value in assets.items():
marshal.save(asset_value, asset_name)
# save notebook as well
shutil.copy(
notebook_path,
os.path.join(TRANSFORMER_ASSETS_DIR, TRANSFORMER_SRC_NOTEBOOK_NAME))
def _load_transformer_assets(self):
marshal.set_data_dir(serveutils.TRANSFORMER_ASSETS_DIR)
log.info("Loading transformer function...")
_fn = marshal.load(serveutils.TRANSFORMER_FN_ASSET_NAME)
# create a new function monkey patching the original function's
# __globals__. The marshalled function would not be scoped under
# the current module, thus its __globals__ dict would be empty.
# In this way we create the same function but binding it to the
# module's globals().
self.fn = types.FunctionType(_fn.__code__, globals(), _fn.__name__,
_fn.__defaults__, _fn.__closure__)
log.info("Processing source notebook for imports and functions...")
processor = NotebookProcessor(nb_path=os.path.join(
serveutils.TRANSFORMER_ASSETS_DIR,
serveutils.TRANSFORMER_SRC_NOTEBOOK_NAME),
skip_validation=True)
self.init_code = processor.get_imports_and_functions()
log.info("Initialization code:\n%s" % self.init_code)
log.info("Running initialization code...")
exec(self.init_code, globals())
log.info("Loading transformer's assets...")
for file in os.listdir(serveutils.TRANSFORMER_ASSETS_DIR):
if file in [
serveutils.TRANSFORMER_SRC_NOTEBOOK_NAME,
serveutils.TRANSFORMER_FN_ASSET_NAME
]:
continue
# The marshal mechanism works by looking at the name of the files
# without extensions.
basename = os.path.splitext(file)[0] # remove extension
self.assets[basename] = marshal.load(basename)
log.info("Assets successfully loaded: %s" % self.assets.keys())
log.info("Initializing assets...")
for asset_name, asset_value in self.assets.items():
globals()[asset_name] = asset_value