def _build_inline_root_task(root_task_name, airflow_context):
# create "root task" with default name as current process executable file name
if not root_task_name:
root_task_name = sys.argv[0].split(os.path.sep)[-1]
class InplaceTask(Task):
_conf__task_family = root_task_name
try:
user_frame = UserCodeDetector.build_code_detector(
).find_user_side_frame(user_side_only=True)
if user_frame:
module_code = open(user_frame.filename).read()
InplaceTask.task_definition.task_module_code = module_code
if not airflow_context:
# we don't set DAGs task's source code
InplaceTask.task_definition.task_source_code = module_code
except Exception as ex:
logger.info("Failed to find source code: %s", str(ex))
root_task = InplaceTask(task_version="now", task_name=root_task_name)
if airflow_context:
# we generate specific cmd values for airflow tasks in sync time
root_task.task_is_system = True
root_task.task_meta.task_command_line = ""
root_task.task_meta.task_functional_call = ""
else:
root_task.task_meta.task_command_line = list2cmdline(sys.argv)
root_task.task_meta.task_functional_call = "bash_cmd(args=%s)" % repr(
sys.argv)
return root_task
def __init__(self, name="global"):
"""
Initialize cmd line args
"""
super(DatabandContext, self).__init__()
# double checking on bootstrap, as we can run from all kind of locations
# usually we should be bootstraped already as we run from cli.
dbnd_bootstrap()
self.name = name
self.current_context_uid = "%s_%s" % (
utcnow().strftime("%Y%m%d_%H%M%S"),
str(uuid.uuid1())[:8],
)
self.config = config
# we are running from python notebook, let start to print to stdout
if self.name == "interactive" or is_unit_test_mode():
self.config.set("log", "stream_stdout", "True", source="log")
self.task_instance_cache = TaskInstanceCache()
self.user_code_detector = UserCodeDetector.build_code_detector()
# will set up in __enter__
# we can't initialize settings without having self defined as context
# we assign real object only in _on_enter, however it's great for auto completion
from dbnd._core.settings import DatabandSettings
self.settings = None # type: DatabandSettings
self._is_initialized = False
self._tracking_store = None
def __init__(
self,
user_code_detector=None,
isolate=True,
show_vars=False,
max_length=120,
skip_non_user_on_isolate=True,
no_colors=False,
color_scheme="monokai",
):
self._user_code_detector = (user_code_detector
or UserCodeDetector.build_code_detector())
self.skip_non_user_on_isolate = skip_non_user_on_isolate and isolate
# Whether to print interesting lines in color or not. If False,
# all lines are printed in color.
self._isolate = isolate
self._no_colors = no_colors
# Whether to print variables for stack frames.
self._show_vars = show_vars and isolate
# Max length of printed variable lines
self._max_length = max_length
self.color_scheme = color_scheme
try:
self.pygments_lexer = PythonLexer()
self.pygments_formatter = TerminalFormatter(
style=self.color_scheme)
except Exception:
self.pygments_lexer = None
self.pygments_formatter = None
def _build_inline_root_task(root_task_name):
# create "root task" with default name as current process executable file name
if not root_task_name:
root_task_name = sys.argv[0].split(os.path.sep)[-1]
class InplaceTask(TrackingTask):
_conf__task_family = root_task_name
try:
user_frame = UserCodeDetector.build_code_detector().find_user_side_frame(
user_side_only=True
)
if user_frame:
module_code = open(user_frame.filename).read()
InplaceTask.task_definition.task_module_code = module_code
InplaceTask.task_definition.task_source_code = module_code
except Exception as ex:
logger.info("Failed to find source code: %s", str(ex))
root_task = InplaceTask(task_version="now", task_name=root_task_name)
root_task.ctrl.task_repr.task_command_line = list2cmdline(sys.argv)
root_task.ctrl.task_repr.task_functional_call = "bash_cmd(args=%s)" % repr(sys.argv)
return root_task
def get_user_module_code():
try:
user_frame = UserCodeDetector.build_code_detector(
).find_user_side_frame(user_side_only=True)
if user_frame:
module_code = open(user_frame.filename).read()
return module_code
except Exception as ex:
return
def from_callstack(cls):
try:
user_frame = UserCodeDetector.build_code_detector(
).find_user_side_frame(user_side_only=True)
if user_frame:
module_code = open(user_frame.filename).read()
return TaskSourceCode(task_module_code=module_code,
task_source_file=user_frame.filename)
except Exception as ex:
logger.debug("Failed to find source code: %s", str(ex))
return NO_SOURCE_CODE
def test_windows_stack(self):
stack = r"""
Traceback (most recent call last):
File "C:\Users\Databand\Anaconda2\lib\site-packages\dbnd_airflow\simple_executor.py", line 62, in sync
self._run_task_instance(command[0], **command[1])
File "C:\Users\Databand\Anaconda2\lib\site-packages\airflow\utils\db.py", line 74, in wrapper
return func(*args, **kwargs)
File "C:\Users\Databand\Anaconda2\lib\site-packages\dbnd_airflow\simple_executor.py", line 142, in _run_task_instance
ti._run_raw_task(mark_success=mark_success, job_id=ti.job_id, pool=pool)
File "C:\Users\Databand\Anaconda2\lib\site-packages\airflow\utils\db.py", line 74, in wrapper
return func(*args, **kwargs)
File "C:\Users\Databand\Anaconda2\lib\site-packages\dbnd_web\models\dbnd_task_instance.py", line 226, in _run_raw_task
result = databand_execute(task_copy, context=context)
File "C:\Users\Databand\Anaconda2\lib\site-packages\dbnd_web\models\dbnd_task_instance.py", line 39, in databand_execute
return airflow_operator.task._task_scheduler_execute(airflow_context=context)
File "C:\Users\Databand\Anaconda2\lib\site-packages\databand\core\task\task.py", line 171, in _task_scheduler_execute
return self.ctrl.task_runner.execute(airflow_context)
File "C:\Users\Databand\Anaconda2\lib\site-packages\databand\core\task_ctrl\task_runner.py", line 47, in execute
return self._execute()
File "C:\Users\Databand\Anaconda2\lib\site-packages\databand\core\task_ctrl\task_runner.py", line 97, in _execute
result = self.task._task_submit()
File "C:\Users\Databand\Anaconda2\lib\site-packages\databand\core\task\task.py", line 182, in _task_submit
return self._task_run()
File "C:\Users\Databand\Anaconda2\lib\site-packages\databand\core\task\task.py", line 188, in _task_run
return self.run()
File "C:\Users\Databand\Anaconda2\lib\site-packages\databand\core\decorator\func_task.py", line 48, in run
self._invoke_func()
File "C:\Users\Databand\Anaconda2\lib\site-packages\databand\core\decorator\func_task.py", line 28, in _invoke_func
result = func_call(**invoke_kwargs)
File "C:/Users/Databand/Documents/source/user_package/user_code.py", line 42, in user_function
max, min = find_min_max(data, data)
AttributeError: 'list' object has no attribute 'max'
"""
user_code_detector = UserCodeDetector(
system_code_dirs=[r"C:\Users\Databand\Anaconda2\lib\site-packages"],
code_dir="C:/Users/Databand/Documents/source",
)
tbvaccine = TBVaccine(
user_code_detector=user_code_detector,
skip_non_user_on_isolate=True,
isolate=True,
no_colors=True,
)
stack = stack.split("\n")
tb_print = tbvaccine._new_tb_print()
tb_print._process(stack)
exception_message = tb_print._buffer
logging.warning("Actual exception message: \n%s", exception_message)
assert "func_task" not in exception_message
assert "user_code" in exception_message
def __init__(self, *args, **kwargs):
help_msg = kwargs.pop("help_msg", None)
nested_exceptions = kwargs.pop("nested_exceptions", None)
show_exc_info = kwargs.pop("show_exc_info",
self._default_show_exc_info)
super(DatabandError, self).__init__(*args, **kwargs)
self.help_msg = help_msg
self.show_exc_info = show_exc_info
from dbnd._core.errors.errors_utils import UserCodeDetector
self.user_frame_info = (
UserCodeDetector.build_code_detector().find_user_side_frame(
user_side_only=True))
if nested_exceptions:
if not isinstance(nested_exceptions, list):
nested_exceptions = [nested_exceptions]
else:
nested_exceptions = []
try:
self.nested_exceptions = nested_exceptions
from dbnd._core.configuration.environ_config import is_unit_test_mode
if is_unit_test_mode():
from dbnd._core.errors.errors_utils import nested_exceptions_str
if nested_exceptions:
logger.exception(
"PYTEST INFO: There are nested exceptions : %s",
nested_exceptions_str(self),
)
except Exception:
logger.exception(
"Failed to print nested exceptions for unit test mode")
def user_code_detector(self):
return UserCodeDetector(
system_code_dirs=[current_test], code_dir=raise_example_dir
)
def user_code_detector(self):
system_code_dirs = [databand_lib_path(), databand_lib_path("../targets")]
return UserCodeDetector(
code_dir=self.code_dir, system_code_dirs=system_code_dirs
)