def _run_hook(self, batch_response: Dict[str, Any],
hook_function_path: str):
"""running a callable after submitting the batch to Livy"""
user_callable_path = hook_function_path
if user_callable_path is not None:
logger.debug("Executing Livy hook; callable at {callable}".format(
callable=user_callable_path))
try:
user_callable = load_python_callable(user_callable_path)
except DatabandError:
logger.error("Failed to load callable at `{path}`".format(
path=user_callable_path))
else:
try:
# callable loaded successfully
# user_callable interface:
# (LivySparkCtrl, Dict[str, Any]) -> None
user_callable(self, batch_response)
except Exception as e:
logger.error(
"Failed to execute callable at `{path}` with the fallowing error: {exception}"
.format(path=user_callable_path, exception=e))
raise
logger.debug(
"Successfully executed Livy hook; callable at `{callable}`"
.format(callable=user_callable_path))
def load_run(cls, dump_file, disable_tracking_api):
# type: (FileTarget, bool) -> DatabandRun
logger.info("Loading dbnd run from %s", dump_file)
with dump_file.open("rb") as fp:
databand_run = cloudpickle.load(file=fp)
if disable_tracking_api:
databand_run.context.tracking_store.disable_tracking_api()
logger.info("Tracking has been disabled")
try:
if databand_run.context.settings.core.pickle_handler:
pickle_handler = load_python_callable(
databand_run.context.settings.core.pickle_handler)
pickle_handler(databand_run)
except Exception as e:
logger.warning(
"error while trying to handle pickle with custom handler:", e)
return databand_run
def parse_from_str(self, s):
return load_python_callable(callable_path=s)