class _BaseSparkTask(Task):
_conf__task_type_name = TaskType.spark
# will get it from env
spark_config = parameter(
config_path=from_task_env("spark_config"))[SparkConfig]
spark_engine = parameter(config_path=from_task_env("spark_engine"))[
SparkEngineConfig] # type: Union[EmrConfig]
python_script = None
main_class = None
def application_args(self):
"""
'Arguments for the application being submitted'
:return: list
"""
return []
def _get_spark_ctrl(self):
# type: ()-> SparkCtrl
return self.spark_engine.get_spark_ctrl(self.current_task_run)
def _task_banner(self, banner, verbosity):
b = banner
b.new_section()
try:
spark_command_line = subprocess.list2cmdline(
list_of_strings(self.application_args()))
b.column("SPARK CMD LINE", spark_command_line)
except Exception:
logger.exception("Failed to get spark command line from %s" % self)
class EngineConfig(config.Config):
"""Databand's engine configuration (where tasks are executed)"""
require_submit = parameter(
description="Should the task engine be forced to submit tasks").value(
False)
dbnd_local_root = parameter(
default=None,
description="Local dbnd home directory at the engine environment"
)[DirTarget]
dbnd_executable = parameter(
default=[sys.executable, "-m", "dbnd"],
description="'dbnd' executable path at engine environment",
)[typing.List[str]]
def cleanup_after_run(self):
pass
def submit_to_engine_task(self, env, task_name, args, interactive=True):
raise local_engine_not_accept_remote_jobs(self.env, self)
def prepare_for_run(self, run):
# type: (DatabandRun) -> None
return
def _should_wrap_with_submit_task(self, task_run):
return self.require_submit
class OutputConfig(Config):
"""(Advanced) Databand's core task's output behaviour"""
_conf__task_family = "output"
path_task = parameter(description="default path for every Task")[str]
path_prod_immutable_task = parameter(
description=
"format of the path to be used by Production Immutable tasks")[str]
hdf_format = (parameter.choices([
"table", "fixed"
]).help("Default format to save DataFrame to hdf").value("fixed"))
deploy_id = parameter(
default=VersionAlias.context_uid,
description="deploy prefix to use for remote deployments",
)[VersionStr]
def get_value_target_config(self, value_type):
# type: (Type) -> TargetConfig
type_handler = get_value_type_of_type(value_type)
for possible_option in [str(type_handler), type_handler.config_name]:
config_value = config.get_config_value(
section=self._conf__task_family, key=possible_option)
if config_value:
return parse_target_config(config_value.value)
return file.pickle
class RunInfoConfig(config.Config):
"""(Advanced) Databand's run information gatherer"""
_conf__task_family = "run_info"
source_version = parameter(
default="git", description="gather version control via git/None"
)[VersionStr]
user_data = parameter(default=None, description="UserData")[str]
user = parameter(default=None, description="override user name with the value")[str]
def build_task_run_info(self):
task_run_env_uid = get_uuid()
import dbnd
logging.debug("Created new task run env with uid '%s'", task_run_env_uid)
machine = environ.get(ENV_DBND__ENV_MACHINE, "")
if environ.get(ENV_DBND__ENV_IMAGE, None):
machine += " image=%s" % environ.get(ENV_DBND__ENV_IMAGE)
return TaskRunEnvInfo(
uid=task_run_env_uid,
databand_version=dbnd.__version__,
user_code_version=self.source_version,
user_code_committed=True,
cmd_line=subprocess.list2cmdline(sys.argv),
user=self.user or dbnd_getuser(),
machine=machine,
project_root=project_path(),
user_data=safe_string(self.user_data, max_value_len=500),
heartbeat=utcnow(),
)
class GitConfig(config.Config):
"""Databand's git validator"""
_conf__task_family = "git"
enforce_clean = parameter(
description="Enforce project's git to be clean. Can be overridden by allow_dirty or --git-allow-dirty"
).value(False)
allow_dirty = parameter(
description="Permit git to be dirty when enforce_clean or --git-enforce-clean is on"
).value(False)
def validate_git_policy(self):
if not self.enforce_clean:
return
if is_git_dirty():
if self.allow_dirty:
logger.warning("Runing with not commited files")
return
raise DatabandBuildError(
help_msg="Git workspace must be clean."
"\nYou see this message because enforce_clean in git section is enabled."
"\nTo temporarily disable this message use --git-allow-dirty."
)
class SchedulerConfig(config.Config):
"""(Advanced) Databand's scheduler"""
_conf__task_family = "scheduler"
config_file = parameter(
default=None,
description="path to a file defining scheduled jobs to execute")[str]
# by default the scheduler drop-in will decide whether to sync or not based on if it's running inside the scheduler or not (and not the websrver)
# the next two params can be used to force it one way or the other
never_file_sync = parameter(
default=False,
description="disable syncing the scheduler config_file to the db"
)[bool]
always_file_sync = parameter(
default=False,
description="force syncing the scheduler config_file to the db")[bool]
no_ui_cli_edit = parameter(
default=False,
description=
"disables creating, editing and deleting scheduled jobs from the cli and ui. Scheduled job definitions will"
"only be taken from the scheduler config file",
)
refresh_interval = parameter(
default=1,
description=
"interval to refresh the scheduled job list (from both the db and/or config file)",
)[int]
active_by_default = parameter(
default=True,
description="whether new scheduled jobs will be active by default"
)[bool]
default_retries = parameter(
description=
"number of times to retry a failed run, unless set to a different value on the scheduled job"
)[int]
dbnd_user = parameter(
description="user used to connect to the dbnd web server")[str]
dbnd_password = parameter(
description="password used to connect to the dbnd web server")[str]
shell_cmd = parameter(
description=
"If shell_cmd is True, the specified command will be executed through the shell. "
"This can be useful if you are using Python primarily "
"for the enhanced control flow it offers "
"over most system shells and still want convenient access to other shell features "
"such as shell pipes, filename wildcards, environment variable expansion, "
"and expansion of ~ to a user's home directory.")[bool]
class DescribeConfig(Config):
"""(Advanced) Databand's --describe behaviour"""
_conf__task_family = "describe"
dry = parameter(
default=False,
description="Describe without pushing to databand-web")[bool]
no_checks = parameter(
default=False,
description="Describe without doing copleteness and other checks"
)[bool]
no_tree = parameter(
default=False, description="Describe without showing tasks tree")[bool]
class PySparkTask(_BaseSparkTask):
_conf__task_type_name = TaskType.pyspark
python_script = parameter(
description="The application that submitted as a job *.py file")[str]
def _task_submit(self):
return self._get_spark_ctrl().run_pyspark(
pyspark_script=self.python_script)
class DescribeConfig(Config):
"""(Advanced) Databand's --describe behaviour"""
_conf__task_family = "describe"
dry = parameter(
default=False, description="Describe without pushing to databand-web"
)[bool]
no_checks = parameter(
default=False, description="Describe without doing copleteness and other checks"
)[bool]
no_tree = parameter(
default=False, description="Describe without showing tasks tree"
)[bool]
console_value_preview_size = parameter(
description="Maximum length of string previewed in TaskVisualiser"
)[int]
class GitConfig(config.Config):
"""Databand's git validator"""
_conf__task_family = "git"
enforce_clean = parameter(
description="Enforce project's git to be clean. Can be overridden by allow_dirty or --git-allow-dirty"
).value(False)
allow_dirty = parameter(
description="Permit git to be dirty when enforce_clean or --git-enforce-clean is on"
).value(False)
def _raise_enforce_clean_error(self, msg):
help_text = (
"\nYou see this message because enforce_clean in git section is enabled."
"\nTo temporarily disable this message use --git-allow-dirty."
)
raise DatabandBuildError(help_msg=msg + help_text)
def validate_git_policy(self):
if not self.enforce_clean:
return
is_dirty = is_git_dirty()
if is_dirty is False:
return
if is_dirty:
if self.allow_dirty:
logger.warning("Runing with not commited files.")
else:
self._raise_enforce_clean_error("Git repo must be clean.")
else: # is_dirty is None
if self.allow_dirty:
logger.warning("Failed to get git status.")
else:
self._raise_enforce_clean_error("Failed to get git status.")
def ingest_partner_data(
data=parameter(log_histograms=True)[pd.DataFrame],
name="customer",
dedup_columns=None,
columns_to_impute=None,
pii_columns=None,
) -> Tuple[pd.DataFrame, pd.DataFrame]:
pii_columns = pii_columns or ["name", "address", "phone"]
dedup_columns = dedup_columns or ["phone"]
columns_to_impute = columns_to_impute or ["10"]
clean = clean_pii(data, pii_columns)
enriched = enrich_missing_fields(clean, columns_to_impute)
deduped = dedup_records(enriched, columns=dedup_columns)
report = create_report(deduped)
return report, deduped
class SparkTask(_BaseSparkTask):
main_class = parameter(default=None,
description="The entry point for your application",
system=True)[str]
def _task_submit(self):
if not self.spark_config.main_jar:
raise DatabandConfigError("main_jar is not configured for %s" %
self)
return self._get_spark_ctrl().run_spark(main_class=self.main_class)
def run(self):
# we don't actually have run function except for inline
#
# we don't want to read params automatically
# most likely our run function just creates a command line to run on remote compute
raise DatabandBuildError(
"You should not call or override run functions")
def enrich_missing_fields(
raw_data=parameter(log_histograms=True)[pd.DataFrame],
columns_to_impute=None,
columns_min_max_scaler=None,
fill_with=0,
) -> pd.DataFrame:
columns_to_impute = columns_to_impute or ["10"]
columns_min_max_scaler = columns_min_max_scaler or []
counter = int(raw_data[columns_to_impute].copy().isna().sum())
noise = randint(-counter, counter)
log_metric("Replaced NaNs",
int(raw_data[columns_to_impute].copy().isna().sum()) + noise)
raw_data[columns_to_impute] = raw_data[columns_to_impute].fillna(fill_with)
for column_name in columns_min_max_scaler:
scaler = preprocessing.MinMaxScaler()
raw_data[column_name + "_norm"] = scaler.fit_transform(
raw_data[[column_name]].values.astype(float))
return raw_data
class _AirflowRuntimeTask(Task):
task_is_system = True
_conf__track_source_code = False
dag_id = parameter[str]
execution_date = parameter[datetime.datetime]
is_dag = parameter(system=True).value(False)
def _initialize(self):
super(_AirflowRuntimeTask, self)._initialize()
self.ctrl.force_task_run_uid = TaskRunUidGen_TaskAfId(self.dag_id)
self.task_meta.task_functional_call = ""
self.task_meta.task_command_line = generate_airflow_cmd(
dag_id=self.dag_id,
task_id=self.task_id,
execution_date=self.execution_date,
is_root_task=self.is_dag,
)
class RunConfig(config.Config):
"""Databand's per run settings (e.g. execution date)"""
_conf__task_family = "run"
# on none generated at DatabandRun
name = parameter.value(default=None, description="Specify run name")[str]
description = parameter.value(default=None, description="Specify run description")[
Optional[str]
]
parallel = parameter(default=None)[bool]
task_executor_type = parameter(
default=None,
description="Alternate executor type: "
" local/airflow_inprocess/airflow_multiprocess_local/airflow_kubernetes,"
" see docs for more options",
)[str]
submit_driver = parameter(
description="override env.submit_driver for specific environment"
).none[bool]
submit_tasks = parameter(
description="override env.submit_tasks for specific environment"
).none[bool]
enable_airflow_kubernetes = parameter(
default=True,
description="Enable use of kubernetes executor for kubebernetes engine submission",
)[bool]
execution_date = parameter(default=None, description="Override execution date")[
datetime
]
# Execution specific
id = parameter(default=None, description="The list of task ids to run")[List[str]]
selected_tasks_regex = parameter(
default=None, description="Run only specified tasks (regular expresion)"
)[List[str]]
ignore_dependencies = parameter(
description="The regex to filter specific task_ids"
).value(False)
ignore_first_depends_on_past = parameter(
description="The regex to filter specific task_ids"
).value(False)
pool = parameter(default=None, description="Resource pool to use")[str]
donot_pickle = parameter(
description="Do not attempt to pickle the DAG object to send over "
"to the workers, just tell the workers to run their version "
"of the code."
).value(False)
mark_success = parameter(
description="Mark jobs as succeeded without running them"
).value(False)
skip_completed = parameter(
description="Mark jobs as succeeded without running them"
).value(True)
fail_fast = parameter(
description="Skip all remaining tasks if a task has failed"
).value(True)
enable_prod = parameter(description="Enable production tasks").value(False)
is_archived = parameter(description="Save this run in the archive").value(False)
heartbeat_interval_s = parameter(
description="How often a run should send a heartbeat to the server. Set -1 to disable"
)[int]
heartbeat_timeout_s = parameter(
description="How old can a run's last heartbeat be before we consider it failed. Set -1 to disable"
)[int]
heartbeat_sender_log_to_file = parameter(
description="create a separate log file for the heartbeat sender and don't log the run process stdout"
)[bool]
open_web_tracker_in_browser = parameter(
description="If True, open web tracker in browser during task run."
).value(False)
enable_concurent_sqlite = parameter(
description="Enable concurrent execution with sqlite db (use only for debug!)"
).value(False)
interactive = parameter(
default=False,
description="When submitting driver to remote execution keep tracking of submitted process and wait for completion",
)[bool]
skip_completed_on_run = parameter(default=True).help(
"Should dbnd task check that task is completed and mark it as resued on task execution"
)[bool]
validate_task_inputs = parameter(default=True).help(
"Should dbnd task check that all input files exist"
)[bool]
validate_task_outputs = parameter(default=True).help(
"Should dbnd task check that all outputs exist after task has been executed"
)[bool]
validate_task_outputs_on_build = parameter(default=False).help(
"Should dbnd task check that there are no incomplete outputs before task executes"
)[bool]
tracking_with_cache = parameter(default=False).help(
"Should dbnd cache results during tracking"
)[bool]
pipeline_band_only_check = parameter(default=False).help(
"When checking if pipeline is completed, check only if the band file exist (skip the tasks)"
)[bool]
task_complete_parallelism_level = parameter(default=1).help(
"Number of threads to use when checking if tasks are already complete"
)[int]
dry = parameter(default=False).help(
"Do not execute tasks, stop before sending them to the execution, and print their status"
)[bool]
class _BaseSparkTask(Task):
_conf__task_type_name = TaskType.spark
# will get it from env
spark_config = parameter(
config_path=from_task_env("spark_config"))[SparkConfig]
spark_engine = parameter(config_path=from_task_env("spark_engine"))[
SparkEngineConfig] # type: Union[EmrConfig]
python_script = None
main_class = None
spark_conf_extension = parameter(
default={},
description="This is an extension for SparkConfig.conf dict, "
"every config added to this dict will be merged to spark_config.conf",
significant=False,
)[dict]
spark_resources = parameter.c(default=None, system=True)[Dict[str,
FileTarget]]
def band(self):
result = super(_BaseSparkTask, self).band()
if self.spark_config.include_user_project:
fat_wheel_task = fat_wheel_building_task(
task_version=try_get_databand_context().current_context_uid,
task_target_date="today",
)
self.spark_resources = {"user_project": fat_wheel_task}
if self.spark_engine.disable_task_band:
logger.debug("Task band is disabled due to disable_task_band flag")
self.task_band = None
return result
def get_py_files(self):
py_files = self.spark_config.py_files.copy()
if self.spark_resources and "user_project" in self.spark_resources:
project_files = self.spark_resources["user_project"].load(str)
py_files.append(project_files)
return py_files
def application_args(self):
"""
'Arguments for the application being submitted'
:return: list
"""
return []
def _get_spark_ctrl(self):
# type: ()-> SparkCtrl
return self.spark_engine.get_spark_ctrl(self.current_task_run)
def _task_banner(self, banner, verbosity):
b = banner
b.new_section()
try:
spark_command_line = subprocess.list2cmdline(
list_of_strings(self.application_args()))
b.column("SPARK CMD LINE", spark_command_line)
except Exception:
logger.exception("Failed to get spark command line from %s" % self)
def get_root(self):
return self.spark_engine.root or super(_BaseSparkTask, self).get_root()
def _initialize(self):
super(_BaseSparkTask, self)._initialize()
if self.spark_conf_extension:
# adds the last layer for SparkConfig.conf
self.spark_config.conf.update(self.spark_conf_extension)
class LoggingConfig(config.Config):
"""Databand's logger configuration"""
_conf__task_family = "log"
disabled = parameter(description="Should logging be disabled").value(False)
debug_log_config = parameter(
description="Debug our logging configuration system").value(False)
capture_stdout_stderr = parameter(
description="Should logger retransmit all output wrtten to stdout/stderr"
).value(True)
capture_task_run_log = parameter.help(
"Capture task output into log").value(True)
override_airflow_logging_on_task_run = parameter(
description="Replace airflow logger with databand logger").value(True)
support_jupyter = parameter(
description="Support logging output to Jupiter UI").value(False)
level = parameter(
description="Logging level. DEBUG/INFO/WARN/ERROR").value("INFO")
formatter = parameter(
description="Log formatting string (logging library convention)")[str]
formatter_colorlog = parameter(
description="Log formatting string (logging library convention)")[str]
formatter_simple = parameter(
description="Log formatting string (logging library convention)")[str]
console_formatter_name = parameter(
description="The name of the formatter logging to console output")[str]
file_formatter_name = parameter(
description="The name of the formatter logging to file output")[str]
# sentry config
sentry_url = parameter(
default=None,
description=
"URL for setting up sentry logger. Notice - make sure the url is exposed to dbnd run environment",
)[str]
sentry_env = parameter(default="dev",
description="Environment for sentry logger")[str]
sentry_release = parameter(default="",
description="Release for sentry logger")[str]
sentry_debug = parameter(default=False,
description="Unable debug flag for sentry")[bool]
file_log = parameter(default=None,
description="Log to file (off by default)")[str]
stream_stdout = parameter(
description="Should databand'a logger stream stdout instead of stderr"
).value(False)
custom_dict_config = parameter(
default=None,
description="Advanced: Customized logging configuration")[Callable]
at_warn = parameter.help("name of loggers to put in WARNING mode").c[
List[str]]
at_debug = parameter.help("name of loggers to put in DEBUG mode").c[
List[str]]
exception_no_color = parameter(
default=False,
description="Do not use colors in exception handling")[bool]
exception_simple = parameter(
default=False, description="Simple mode of exception handling")[bool]
send_body_to_server = parameter(
default=True,
description="Enable or disable sending log file to server.")[bool]
preview_head_bytes = parameter(
default=0, # Disabled
description=
"Max head size of the log file, bytes to be sent to server.\n"
"Default: 0KB.",
)[int]
preview_tail_bytes = parameter(
default=0, # Disabled
description=
"Max tail size of the log file, bytes to be sent to server.\n"
"Default: 0KB.",
)[int]
remote_logging_disabled = parameter.help(
"for tasks using a cloud environment, don't copy the task log to cloud storage"
).value(False)
targets_log_level = parameter(
default="DEBUG",
description=
"Should log the time it takes for marshalling and unmarshalling targets",
)[str]
disable_colors = parameter(default=False,
description="Disabling any colored logs.")
sqlalchemy_print = parameter(
description="enable sqlalchemy logger").value(False)
sqlalchemy_trace = parameter(
description="trace sqlalchemy queries").value(False)
def _initialize(self):
super(LoggingConfig, self)._initialize()
self.task_log_file_formatter = None
def format_exception_as_str(self, exc_info, isolate=True):
if self.exception_simple:
return format_exception_as_str(exc_info)
try:
from dbnd._vendor.tbvaccine import TBVaccine
tbvaccine = TBVaccine(
no_colors=self.exception_no_color,
show_vars=False,
skip_non_user_on_isolate=True,
isolate=isolate,
)
return tbvaccine.format_tb(*exc_info)
except Exception as ex:
logger.info("Failed to format exception: %s", ex)
return format_exception_as_str(exc_info)
def get_dbnd_logging_config(self, filename=None):
if self.custom_dict_config:
if not in_quiet_mode():
logger.info("Using user provided logging config")
self.log_debug("Using log.custom_dict_config")
return self.settings.log.custom_dict_config()
return self.get_dbnd_logging_config_base(filename=filename)
def get_dbnd_logging_config_base(self, filename=None):
# type: (LoggingConfig, Optional[str]) -> Optional[dict]
self.log_debug("Using log.get_dbnd_logging_config_base")
log_settings = self
log_level = log_settings.level
# we want to have "real" output, so nothing can catch our handler
# in opposite to what airflow is doing
console_stream = (sys.__stdout__
if log_settings.stream_stdout else sys.__stderr__)
if "ipykernel" in sys.modules and self.support_jupyter:
# we can not use __stdout__ or __stderr__ as it will not be printed into jupyter web UI
# at the same time using sys.stdout when airflow is active is very dangerous
# as it can create dangerous loop from airflow redirection into root logger
self.log_debug("ipykernel: checking on console_stream again")
console_stream = sys.stdout if log_settings.stream_stdout else sys.stderr
# dummy path, we will not write to this file
task_file_handler_file = databand_system_path("logs", "task.log")
self.log_debug("task_file_handler_file: %s", task_file_handler_file)
setup_log_file(task_file_handler_file)
config = {
"version": 1,
"disable_existing_loggers": False,
"filters": {
"task_context_filter": {
"()": "dbnd._core.log.logging_utils.TaskContextFilter"
}
},
"formatters": {
"formatter": {
"format": log_settings.formatter
},
"formatter_simple": {
"format": log_settings.formatter_simple
},
"formatter_colorlog": {
"()": "dbnd._vendor.colorlog.ColoredFormatter",
"format": log_settings.formatter_colorlog,
"reset": True,
},
},
"handlers": {
"console": {
"class": "logging.StreamHandler",
"stream": console_stream,
"formatter": log_settings.console_formatter_name,
"filters": ["task_context_filter"],
}
},
"root": {
"handlers": ["console"],
"level": log_level
},
}
if filename:
setup_log_file(filename)
config["handlers"]["file"] = {
"class": "logging.FileHandler",
"formatter": log_settings.file_formatter_name,
"filename": filename,
"encoding": "utf-8",
}
config["root"]["handlers"].append("file")
loggers = config.setdefault("loggers", {})
for logger_warn in log_settings.at_warn:
loggers[logger_warn] = {
"level": logging.WARNING,
"propagate": True
}
for logger_debug in log_settings.at_debug:
loggers[logger_debug] = {"level": logging.DEBUG, "propagate": True}
if log_settings.sqlalchemy_print:
loggers["sqlalchemy.engine"] = {
"level": logging.INFO,
"propagate": True
}
self.log_debug("Log config: %s", config)
return config
def configure_dbnd_logging(self):
if self.disabled:
self.log_debug("Log is disabled, skipping configure_dbnd_logging")
return
# start by trying to initiate Sentry setup - has side effect of changing the logging config
self.log_debug("Initialize Sentry setup")
try_init_sentry()
if self.disable_colors:
self.log_debug("Colors are disabled")
self.disable_color_logs()
dict_config = self.get_dbnd_logging_config(filename=self.file_log)
airflow_task_log_handler = None
if self.override_airflow_logging_on_task_run:
airflow_task_log_handler = self.dbnd_override_airflow_logging_on_task_run(
)
try:
self.log_debug("configure_logging_dictConfig: %s", dict_config)
configure_logging_dictConfig(dict_config=dict_config)
except Exception as e:
# we print it this way, as it could be that now "logging" is down!
print(
"Failed to load reload logging configuration with dbnd settings! Exception: %s"
% (e, ),
file=sys.__stderr__,
)
raise
if airflow_task_log_handler:
self.log_debug(
"logging.root.handlers.append(airflow_task_log_handler)")
logging.root.handlers.append(airflow_task_log_handler)
self.log_debug("Databand logging is up!")
def dbnd_override_airflow_logging_on_task_run(self):
# EXISTING STATE:
# root logger use Console handler -> prints to current sys.stdout
# on `airflow run` without interactive -> we have `redirect_stderr` applied that will redirect sys.stdout
# into logger `airflow.task`, that will save everything into file.
# EVERY output of root logger will go through CONSOLE handler into AIRFLOW.TASK without being printed to screen
self.log_debug("dbnd_override_airflow_logging_on_task_run")
if not sys.stderr or not _safe_is_typeof(sys.stderr,
"StreamLogWriter"):
self.log_debug(
"Airflow logging is already replaced by dbnd stream log writer! sys.stderr=%s",
sys.stderr,
)
return
# NEW STATE
# we will move airflow.task file handler to root level
# we will set propogate
# we will stop redirect of airflow logging
# this will disable stdout ,stderr redirection
sys.stderr = sys.__stderr__
sys.stdout = sys.__stdout__
airflow_root_console_handler = find_handler(logging.root, "console")
self.log_debug("airflow_root_console_handler:%s",
airflow_root_console_handler)
if _safe_is_typeof(airflow_root_console_handler, "RedirectStdHandler"):
# we are removing this console logger
# this is the logger that capable to create self loop
# as it writes to "latest" sys.stdout,
# if you have stdout redirection into any of loggers, that will propogate into root
# you get very busy message loop that is really hard to debug
self.log_debug("airflow_root_console_handler has been removed")
logging.root.handlers.remove(airflow_root_console_handler)
airflow_task_logger = logging.getLogger("airflow.task")
self.log_debug("airflow_task_logger: %s", airflow_task_logger)
airflow_task_log_handler = find_handler(airflow_task_logger, "task")
if airflow_task_log_handler:
self.log_debug("airflow_task_log_handler: %s",
airflow_task_log_handler)
logging.root.handlers.append(airflow_task_log_handler)
airflow_task_logger.propagate = True
airflow_task_logger.handlers = []
self.log_debug(
"dbnd_override_airflow_logging_on_task_run logging.root: %s",
logging.root)
return airflow_task_log_handler
def get_task_log_file_handler(self, log_file):
if not self.task_log_file_formatter:
config = self.get_dbnd_logging_config()
configurator = DictConfigurator(config)
file_formatter_config = configurator.config.get("formatters").get(
self.file_formatter_name)
self.task_log_file_formatter = configurator.configure_formatter(
file_formatter_config)
# "formatter": log_settings.file_formatter,
log_file = str(log_file)
setup_log_file(log_file)
handler = logging.FileHandler(filename=log_file, encoding="utf-8")
handler.setFormatter(self.task_log_file_formatter)
handler.setLevel(self.level)
return handler
def disable_color_logs(self):
"""Removes colors from any console related config"""
logger.debug("disabling color logs")
os.environ[
"ANSI_COLORS_DISABLED"] = "True" # disabling termcolor.colored
self.exception_no_color = True
if self.console_formatter_name == "formatter_colorlog":
self.console_formatter_name = "formatter_simple"
def log_debug(self, msg, *args):
if not self.debug_log_config:
if not self.disabled:
# we don't want to print ANYTHING if we are disabled
logger.debug(msg, *args)
return
try:
# we print to stderr as well in case logging is broken
print("DEBUG_LOG_CONFIG:" + msg % args, file=sys.__stderr__)
logger.info("DEBUG_LOG_CONFIG:" + msg, *args)
except Exception:
pass
class Task(_TaskWithParams, _TaskCtrlMixin, _TaskParamContainer):
"""
This is the base class of all dbnd Tasks, the base unit of work in databand.
A dbnd Task describes a unit or work.
A ``run`` method must be present in a subclass
Each ``parameter`` of the Task should be declared as members::
class MyTask(dbnd.Task):
count = dbnd.parameter[int]
second_param = dbnd.parameter[str]
"""
_conf_confirm_on_kill_msg = None # get user confirmation on task kill if not empty
_conf__require_run_dump_file = False
_task_band_result = output(default=None, system=True)
_meta_output = output(
system=True,
output_name="meta",
output_ext="",
target_config=folder,
significant=False,
description="Location of all internal outputs (e.g. metrics)",
)
task_band = output.json(output_name="band", system=True)
task_enabled = system_passthrough_param(default=True)[bool]
task_enabled_in_prod = system_passthrough_param(default=True)[bool]
validate_no_extra_params = ParamValidation.error
# for permanent bump of task version use Task.task_class_version
task_version = parameter(
default="1",
description="task version, directly affects task signature ",
scope=ParameterScope.children,
)[VersionStr]
task_class_version = parameter.value(
default=DEFAULT_CLASS_VERSION,
system=True,
description="task code version, "
"use while you want persistent change in your task version",
)
task_env = parameter.value(
default="local",
description="task environment name",
scope=ParameterScope.children,
)[EnvConfig]
task_target_date = parameter(
default="today",
description="task data target date",
scope=ParameterScope.children,
)[datetime.date]
task_airflow_op_kwargs = parameter.system(
default=None, description="airflow operator kwargs"
)[Dict[str, object]]
task_config = parameter.system(empty_default=True)[Dict]
task_is_system = parameter.system(default=False)[bool]
task_in_memory_outputs = system_passthrough_param(
default=False, description="Store all task outputs in memory"
)[bool]
task_output_path_format = system_passthrough_param(
default=None, description="Format string used to generate task output paths"
)[str]
task_is_dynamic = system_passthrough_param(
default=False,
scope=ParameterScope.children,
description="task was executed from within another task",
)[bool]
# for example: if task.run doesn't have access to databand, we can't run runtime tasks
task_supports_dynamic_tasks = parameter.system(
default=True, description="indicates if task can run dynamic databand tasks"
)[bool]
task_retries = parameter.system(
default=0,
description="Total number of attempts to run the task. So task_retries=3 -> task can fail 3 times before we give up",
)[int]
task_retry_delay = parameter.system(
default="15s",
description="timedelta to wait before retrying a task. Example: 5s",
)[datetime.timedelta]
task_essence = TaskEssence.ORCHESTRATION
def __init__(self, **kwargs):
super(Task, self).__init__(**kwargs)
# used to communicate return value of "user function"
self._dbnd_call_state = None # type: Optional[TaskCallState]
self.ctrl = TaskCtrl(self)
def band(self):
"""
Please, do not override this function only in Pipeline/External tasks!
We do all wiring work in Meta classes only.
Our implementation should never be coupled to code!
"""
return
def run(self):
"""
The task run method, to be overridden in a subclass.
See :ref:`Task.run`
"""
pass # default impl
@property
def task_outputs(self):
"""
The output that this Task produces.
The output of the Task determines if the Task needs to be run--the task
is considered finished iff the outputs all exist.
"""
return self.ctrl.relations.task_outputs_user
@property
def task_dag(self):
# type: (...)->_TaskDagNode
return self.ctrl.task_dag
@property
def descendants(self):
return self.ctrl.descendants
def _complete(self):
"""
If the task has any outputs, return ``True`` if all outputs exist. Otherwise, return ``False``.
However, you may freely override this method with custom logic.
"""
# we check only user side task outputs
# all system tasks outputs are not important (if the exists or not)
# user don't see them
outputs = [
o for o in flatten(self.task_outputs) if not o.config.overwrite_target
]
if len(outputs) == 0:
if not self.task_band:
warnings.warn(
"Task %r without outputs has no custom complete() and no task band!"
% self,
stacklevel=2,
)
return False
else:
return self.task_band.exists()
incomplete_outputs = [str(o) for o in outputs if not o.exists()]
num_of_incomplete_outputs = len(incomplete_outputs)
if 0 < num_of_incomplete_outputs < len(outputs):
complete_outputs = [str(o) for o in outputs if o.exists()]
exc = incomplete_output_found_for_task(
self.task_name, complete_outputs, incomplete_outputs
)
if self.settings.run.validate_task_outputs_on_build:
raise exc
else:
logger.warning(str(exc))
return num_of_incomplete_outputs == 0
@property
def current_task_run(self):
# type: ()->TaskRun
return get_databand_run().get_task_run(self.task_id)
def _output(self):
"""
The default output that this Task produces.
Use outputs! Override only if you are writing "base" class.
"""
return NOTHING
def _requires(self):
"""
Override in "template" tasks which themselves are supposed to be subclassed.
Must return an iterable which, among others, contains the _requires() of
the superclass.
"""
def _task_submit(self):
"""Task submission logic, by default we just call -> ``_task_run()`` -> ``run()``."""
return self._task_run()
def _task_run(self):
# bring all relevant files
self.current_task_run.sync_local.sync_pre_execute()
param_values = self.task_params.get_param_values()
with auto_load_save_params(
task=self, auto_read=self._conf_auto_read_params, param_values=param_values
):
result = self.run()
self.current_task_run.sync_local.sync_post_execute()
# publish all relevant files
return result
@property
def tracker(self):
return self.current_task_run.tracker
@property
def metrics(self):
# backward compatible code
return self.tracker
def get_template_vars(self):
# TODO: move to cached version, (after relations are built)
base = {
"task": self,
"task_family": self.task_family,
"task_name": self.task_name,
"task_signature": self.task_signature,
"task_id": self.task_id,
}
base.update(self._params.get_params_serialized(ParameterFilters.INPUTS))
if self.task_target_date is None:
base["task_target_date"] = "input"
return base
def on_kill(self):
"""
Override this method to cleanup subprocesses when a task instance gets killed.
Any use of the threading, subprocess or multiprocessing
module within an operator needs to be cleaned up or it will leave
ghost processes behind.
"""
def _get_task_output_path_format(self, output_mode):
"""
Defines the format string used to generate all task outputs.
For example:
{root}/{env_label}/{task_target_date}/{task_name}/{task_name}{task_class_version}_{task_signature}/{output_name}{output_ext}
"""
if self.task_output_path_format:
# explicit input - first priority
return self.task_output_path_format
if self._conf__base_output_path_fmt:
# from class definition
return self._conf__base_output_path_fmt
# default behaviour
if self.task_env.production and output_mode == OutputMode.prod_immutable:
return self.settings.output.path_prod_immutable_task
return self.settings.output.path_task
def get_target(self, name, config=None, output_ext=None, output_mode=None):
name = name or "tmp/dbnd-tmp-%09d" % random.randint(0, 999999999)
config = config or TargetConfig()
path_pattern = self._get_task_output_path_format(output_mode)
path = calculate_path(
task=self,
name=name,
output_ext=output_ext,
is_dir=config.folder,
path_pattern=path_pattern,
)
return target(path, config=config)
def get_root(self):
return self.task_env.root
def _initialize(self):
super(Task, self)._initialize()
self.ctrl._initialize_task()
def _should_run(self):
if not self.task_enabled:
return False
if self.task_env.production:
return self.task_enabled_in_prod or self.settings.run.enable_prod
return True
def _save_param(self, parameter, original_value, current_value):
# type: (ParameterDefinition, Any, Any) -> None
# it's output! we are going to save it.
# task run doesn't always exist
task_run = try_get_current_task_run()
access_status = DbndTargetOperationStatus.OK
try:
if isinstance(original_value, InMemoryTarget):
parameter.value_type = get_value_type_of_obj(
current_value, parameter.value_type
)
parameter.dump_to_target(original_value, current_value)
# it's a workaround, we don't want to change parameter for outputs (dynamically)
# however, we need proper value type to "dump" preview an other meta.
# we will update it only for In memory targets only for now
except Exception as ex:
access_status = DbndTargetOperationStatus.NOK
raise friendly_error.task_execution.failed_to_save_value_to_target(
ex, self, parameter, original_value, current_value
)
finally:
if task_run:
try:
task_run.tracker.log_parameter_data(
parameter=parameter,
target=original_value,
value=current_value,
operation_type=DbndTargetOperationType.write,
operation_status=access_status,
)
except Exception as ex:
logger.warning("Failed to log target to tracking store. %s", ex)
@dbnd_handle_errors(exit_on_error=False)
def dbnd_run(self):
# type: (...)-> DatabandRun
"""Run task via Databand execution system."""
# this code should be executed under context!
from dbnd._core.current import get_databand_context
ctx = get_databand_context()
run = ctx.dbnd_run_task(self)
return run
import logging
from dbnd._core.constants import ApacheBeamClusterType, CloudType, EnvLabel
from dbnd._core.errors import friendly_error
from dbnd._core.parameter.parameter_builder import parameter
from dbnd._core.parameter.parameter_definition import ParameterScope
from dbnd._core.task.config import Config
from targets import DirTarget
logger = logging.getLogger(__name__)
task_env_param = parameter(scope=ParameterScope.children)
class EnvConfig(Config):
"""Databand's environment configuration"""
_conf__task_family = "env"
cloud_type = parameter(description="cloud type: gcp/aws/")[str]
env_label = parameter(
default=EnvLabel.dev,
description="environment type: dev/int/prod")[str] # label
production = parameter(
description="indicates that environment is production").value(False)
conn_id = parameter(default=None,
description="cloud connection settings")[str]
# MAIN OUTPUT FOLDER
class Task(_BaseTask, _TaskParamContainer):
"""
This is the base class of all dbnd Tasks, the base unit of work in databand.
A dbnd Task describes a unit or work.
The key methods of a Task, which must be implemented in a subclass are:
* :py:meth:`run` - the computation done by this task.
Each :py:class:`~dbnd.parameter` of the Task should be declared as members:
.. code:: python
class MyTask(dbnd.Task):
count = dbnd.parameter[int]
second_param = dbnd.parameter[str]
In addition to any declared properties and methods, there are a few
non-declared properties, which are created by the :py:class:`TaskMetaclass`
metaclass:
"""
"""
This value can be overriden to set the namespace that will be used.
(See :ref:`Task.namespaces_famlies_and_ids`)
If it's not specified and you try to read this value anyway, it will return
garbage. Please use :py:meth:`get_task_namespace` to read the namespace.
Note that setting this value with ``@property`` will not work, because this
is a class level value.
"""
_task_band_result = output(default=None, system=True)
_meta_output = output(
system=True,
output_name="meta",
output_ext="",
target_config=folder,
significant=False,
description="Location of all internal outputs (e.g. metrics)",
)
task_band = output.json(output_name="band")
task_enabled = parameter.system(scope=ParameterScope.children)[bool]
task_enabled_in_prod = parameter.system(
scope=ParameterScope.children)[bool]
# for permanent bump of task version use Task.task_class_version
task_version = parameter(
description="task version, directly affects task signature ",
scope=ParameterScope.children,
)[VersionStr]
task_class_version = parameter.value(
default=DEFAULT_CLASS_VERSION,
system=True,
description="task code version, "
"use while you want persistent change in your task version",
)
task_env = parameter.value(description="task environment name",
scope=ParameterScope.children)[EnvConfig]
task_target_date = parameter(description="task data target date",
scope=ParameterScope.children)[datetime.date]
task_airflow_op_kwargs = parameter.system(
default=None, description="airflow operator kwargs")[Dict[str, object]]
task_config = parameter.system(empty_default=True)[Dict]
task_is_system = parameter.system(default=False)[bool]
task_in_memory_outputs = parameter.system(
scope=ParameterScope.children,
description="Store all task outputs in memory")[bool]
task_is_dynamic = parameter.system(
scope=ParameterScope.children,
description="task was executed from within another task",
)[bool]
# for example: if task.run doesn't have access to databand, we can't run runtime tasks
task_supports_dynamic_tasks = parameter.system(
default=True,
description="indicates if task can run dynamic databand tasks")[bool]
task_retries = parameter.system(
description=
"Total number of attempts to run the task. So task_retries=3 -> task can fail 3 times before we give up"
)[int]
task_retry_delay = parameter.system(
description="timedelta to wait before retrying a task. Example: 5s")[
datetime.timedelta]
_dbnd_call_state = None # type: TaskCallState
def __init__(self, **kwargs):
super(Task, self).__init__(**kwargs)
self.ctrl = TaskCtrl(self)
def band(self):
"""
Please, do not override this function only in Pipeline/External tasks! we do all wiring work in Meta classes only
Our implementation should never be coupled to code!
:return:
"""
return
def run(self):
"""
The task run method, to be overridden in a subclass.
See :ref:`Task.run`
"""
pass # default impl
@property
def task_outputs(self):
"""
The output that this Task produces.
The output of the Task determines if the Task needs to be run--the task
is considered finished iff the outputs all exist.
See :ref:`Task.task_outputs`
"""
return self.ctrl.relations.task_outputs_user
@property
def task_dag(self):
# type: (...)->_TaskDagNode
return self.ctrl.task_dag
def _complete(self):
"""
If the task has any outputs, return ``True`` if all outputs exist.
Otherwise, return ``False``.
However, you may freely override this method with custom logic.
"""
# we check only user side task outputs
# all system tasks outputs are not important (if the exists or not)
# user don't see them
outputs = flatten(self.task_outputs)
if len(outputs) == 0:
warnings.warn(
"Task %r without outputs has no custom complete() method" %
self,
stacklevel=2,
)
return False
return all((o.exists() for o in outputs))
@property
def current_task_run(self):
# type: ()->TaskRun
return get_databand_run().get_task_run(self.task_id)
def _output(self):
"""
The default output that this Task produces. Use outputs! Override only if you are writing "base" class
"""
return NOTHING
def _requires(self):
"""
Override in "template" tasks which themselves are supposed to be
subclassed
Must return an iterable which among others contains the _requires() of
the superclass.
See :ref:`Task.requires`
"""
pass
def _task_submit(self):
"""
Task submission logic, by default we just call -> _task_run() -> run()
"""
return self._task_run()
def _task_run(self):
# bring all relevant files
self.current_task_run.sync_local.sync_pre_execute()
with self._auto_load_save_params(auto_read=self._conf_auto_read_params,
save_on_change=True):
result = self.run()
self.current_task_run.sync_local.sync_post_execute()
# publish all relevant files
return result
def set_upstream(self, task_or_task_list):
self.task_dag.set_upstream(task_or_task_list)
def set_downstream(self, task_or_task_list):
self.task_dag.set_downstream(task_or_task_list)
def __lshift__(self, other):
return self.set_upstream(other)
def __rshift__(self, other):
return self.set_downstream(other)
def set_global_upstream(self, task_or_task_list):
self.task_dag.set_global_upstream(task_or_task_list)
@property
def metrics(self):
# backward compatible code
return self.current_task_run.tracker
def log_dataframe(
self,
key,
df,
with_preview=True,
with_schema=True,
with_size=True,
with_stats=False,
):
meta_conf = ValueMetaConf(
log_preview=with_preview,
log_schema=with_schema,
log_size=with_size,
log_stats=with_stats,
)
self.metrics.log_dataframe(key, df, meta_conf=meta_conf)
def log_metric(self, key, value, source=None):
"""
Logs the passed-in parameter under the current run, creating a run if necessary.
:param key: Parameter name (string)
:param value: Parameter value (string)
"""
return self.metrics.log_metric(key, value, source=source)
def log_system_metric(self, key, value):
"""Shortcut for log_metric(..., source="system") """
return self.log_metric(key, value, source="system")
def log_artifact(self, name, artifact):
"""Log a local file or directory as an artifact of the currently active run."""
return self.metrics.log_artifact(name, artifact)
def get_template_vars(self):
# TODO: move to cached version, (after relations are built)
base = {
"task": self,
"task_family": self.task_meta.task_family,
"task_name": self.task_meta.task_name,
"task_signature": self.task_meta.task_signature,
"task_id": self.task_meta.task_id,
}
base.update(self._params.get_params_serialized(input_only=True))
if self.task_target_date is None:
base["task_target_date"] = "input"
return base
def on_kill(self):
"""
Override this method to cleanup subprocesses when a task instance
gets killed. Any use of the threading, subprocess or multiprocessing
module within an operator needs to be cleaned up or it will leave
ghost processes behind.
"""
pass
def _get_task_output_path_format(self, output_mode):
if self.task_env.production and output_mode == OutputMode.prod_immutable:
return self.settings.output.path_prod_immutable_task
return self._conf__base_output_path_fmt or self.settings.output.path_task
def get_target(self, name, config=None, output_ext=None, output_mode=None):
name = name or "tmp/dbnd-tmp-%09d" % random.randint(0, 999999999)
config = config or TargetConfig()
path_pattern = self._get_task_output_path_format(output_mode)
path = calculate_path(
task=self,
name=name,
output_ext=output_ext,
is_dir=config.folder,
path_pattern=path_pattern,
)
return target(path, config=config)
def get_root(self):
return self.task_env.root
def _initialize(self):
super(Task, self)._initialize()
self.ctrl._initialize_task()
def _should_run(self):
if not self.task_enabled:
return False
if self.task_env.production:
return self.task_enabled_in_prod or self.settings.run.enable_prod
return True
@dbnd_handle_errors(exit_on_error=False)
def dbnd_run(self):
# type: (...)-> DatabandRun
"""
Run task via Databand execution system
"""
# this code should be executed under context!
from dbnd._core.current import get_databand_context
ctx = get_databand_context()
result = ctx.dbnd_run_task(self)
return result
class LoggingConfig(config.Config):
"""Databand's logger configuration"""
_conf__task_family = "log"
disabled = parameter(description="Should logging be disabled").value(False)
capture_stdout_stderr = parameter(
description="Should logger retransmit all output wrtten to stdout/stderr"
).value(True)
capture_task_run_log = parameter.help(
"Capture task output into log").value(True)
override_airflow_logging_on_task_run = parameter(
description="Replace airflow logger with databand logger").value(True)
support_jupyter = parameter(
description="Support logging output to Jupiter UI").value(False)
level = parameter(
description="Logging level. DEBUG/INFO/WARN/ERROR").value("INFO")
formatter = parameter(
description="Log formatting string (logging library convention)")[str]
formatter_colorlog = parameter(
description="Log formatting string (logging library convention)")[str]
formatter_simple = parameter(
description="Log formatting string (logging library convention)")[str]
console_formatter_name = parameter(
description="The name of the formatter logging to console output")[str]
file_formatter_name = parameter(
description="The name of the formatter logging to file output")[str]
sentry_url = parameter(default=None,
description="URL for setting up sentry logger")[str]
sentry_env = parameter(default=None,
description="Envrionment for sentry logger")[str]
file_log = parameter(default=None,
description="Log to file (off by default)")[str]
stream_stdout = parameter(
description="Should databand'a logger stream stdout instead of stderr"
).value(False)
custom_dict_config = parameter(
default=None,
description="Advanced: Customized logging configuration")[Callable]
at_warn = parameter.help("name of loggers to put in WARNING mode").c[
List[str]]
at_debug = parameter.help("name of loggers to put in DEBUG mode").c[
List[str]]
exception_no_color = parameter(
default=False,
description="Do not use colors in exception handling")[bool]
exception_simple = parameter(
default=False, description="Simple mode of exception handling")[bool]
send_body_to_server = parameter(
default=True,
description="Enable or disable sending log file to server.")[bool]
send_body_to_server_max_size = parameter(
default=16 * 1024 * 1024, # 16MB
description="Max log file size in bytes to be sent to server.\n"
"\t* use -1 for unlimited;"
"\t* use negative (e.g. -1000) to get log's 'head' instead of 'tail'."
"Default: 16MB.",
)[int]
remote_logging_disabled = parameter.help(
"for tasks using a cloud environment, don't copy the task log to cloud storage"
).value(False)
targets_log_level = parameter(
default="DEBUG",
description=
"Should log the time it takes for marshalling and unmarshalling targets",
)[str]
sqlalchemy_print = parameter(
description="enable sqlalchemy logger").value(False)
sqlalchemy_trace = parameter(
description="trace sqlalchemy queries").value(False)
api_profile = parameter(description="profile api calls").value(False)
def _initialize(self):
super(LoggingConfig, self)._initialize()
self.task_log_file_formatter = None
def format_exception_as_str(self, exc_info, isolate=True):
if self.exception_simple:
return format_exception_as_str(exc_info)
try:
tbvaccine = TBVaccine(
no_colors=self.exception_no_color,
show_vars=False,
skip_non_user_on_isolate=True,
isolate=isolate,
)
return tbvaccine.format_tb(*exc_info)
except Exception as ex:
logger.info("Failed to format exception: %s", ex)
return format_exception_as_str(exc_info)
def get_dbnd_logging_config(self, filename=None):
if self.custom_dict_config:
if not in_quiet_mode():
logger.info("Using user provided logging config")
return self.settings.log.custom_dict_config()
return self.get_dbnd_logging_config_base(filename=filename)
def get_dbnd_logging_config_base(self, filename=None):
# type: (LoggingConfig, Optional[str]) -> Optional[dict]
log_settings = self
log_level = log_settings.level
# we want to have "real" output, so nothing can catch our handler
# in opposite to what airflow is doing
console_stream = (sys.__stdout__
if log_settings.stream_stdout else sys.__stderr__)
if "ipykernel" in sys.modules and self.support_jupyter:
# we can not use __stdout__ or __stderr__ as it will not be printed into jupyter web UI
# at the same time using sys.stdout when airflow is active is very dangerous
# as it can create dangerous loop from airflow redirection into root logger
console_stream = sys.stdout if log_settings.stream_stdout else sys.stderr
# dummy path, we will not write to this file
task_file_handler_file = databand_system_path("logs", "task.log")
setup_log_file(task_file_handler_file)
config = {
"version": 1,
"disable_existing_loggers": False,
"filters": {
"task_context_filter": {
"()": "dbnd._core.log.logging_utils.TaskContextFilter"
}
},
"formatters": {
"formatter": {
"format": log_settings.formatter
},
"formatter_simple": {
"format": log_settings.formatter_simple
},
"formatter_colorlog": {
"()": "dbnd._vendor.colorlog.ColoredFormatter",
"format": log_settings.formatter_colorlog,
"reset": True,
},
},
"handlers": {
"console": {
"class": "logging.StreamHandler",
"stream": console_stream,
"formatter": log_settings.console_formatter_name,
"filters": ["task_context_filter"],
}
},
"root": {
"handlers": ["console"],
"level": log_level
},
}
if filename:
setup_log_file(filename)
config["handlers"]["file"] = {
"class": "logging.FileHandler",
"formatter": log_settings.file_formatter_name,
"filename": filename,
"encoding": "utf-8",
}
config["root"]["handlers"].append("file")
loggers = config.setdefault("loggers", {})
for logger_warn in log_settings.at_warn:
loggers[logger_warn] = {
"level": logging.WARNING,
"propagate": True
}
for logger_debug in log_settings.at_debug:
loggers[logger_debug] = {"level": logging.DEBUG, "propagate": True}
if log_settings.sqlalchemy_print:
loggers["sqlalchemy.engine"] = {
"level": logging.INFO,
"propagate": True
}
if log_settings.sentry_url:
config["handlers"]["sentry"] = get_sentry_logging_config(
sentry_url=log_settings.sentry_url,
sentry_env=log_settings.sentry_env)
config["root"]["handlers"].append("sentry")
return config
def configure_dbnd_logging(self):
if self.disabled:
return
dict_config = self.get_dbnd_logging_config(filename=self.file_log)
airflow_task_log_handler = None
if self.override_airflow_logging_on_task_run:
airflow_task_log_handler = self.dbnd_override_airflow_logging_on_task_run(
)
configure_logging_dictConfig(dict_config=dict_config)
if airflow_task_log_handler:
logging.root.handlers.append(airflow_task_log_handler)
logger.debug("Databand logging is up!")
def dbnd_override_airflow_logging_on_task_run(self):
# EXISTING STATE:
# root logger use Console handler -> prints to current sys.stdout
# on `airflow run` without interactive -> we have `redirect_stderr` applied that will redirect sys.stdout
# into logger `airflow.task`, that will save everything into file.
# EVERY output of root logger will go through CONSOLE handler into AIRFLOW.TASK without being printed to screen
if not sys.stderr or not _safe_is_typeof(sys.stderr,
"StreamLogWriter"):
logger.debug(
"Airflow logging is already replaced by dbnd stream log writer!"
)
return
# NEW STATE
# we will move airflow.task file handler to root level
# we will set propogate
# we will stop redirect of airflow logging
# this will disable stdout ,stderr redirection
sys.stderr = sys.__stderr__
sys.stdout = sys.__stdout__
airflow_root_console_handler = find_handler(logging.root, "console")
if _safe_is_typeof(airflow_root_console_handler, "RedirectStdHandler"):
# we are removing this console logger
# this is the logger that capable to create self loop
# as it writes to "latest" sys.stdout,
# if you have stdout redirection into any of loggers, that will propogate into root
# you get very busy message loop that is really hard to debug
logging.root.handlers.remove(airflow_root_console_handler)
airflow_task_logger = logging.getLogger("airflow.task")
airflow_task_log_handler = find_handler(airflow_task_logger, "task")
if airflow_task_log_handler:
logging.root.handlers.append(airflow_task_log_handler)
airflow_task_logger.propagate = True
airflow_task_logger.handlers = []
return airflow_task_log_handler
def get_task_log_file_handler(self, log_file):
if not self.task_log_file_formatter:
config = self.get_dbnd_logging_config()
configurator = DictConfigurator(config)
file_formatter_config = configurator.config.get("formatters").get(
self.file_formatter_name)
self.task_log_file_formatter = configurator.configure_formatter(
file_formatter_config)
# "formatter": log_settings.file_formatter,
log_file = str(log_file)
setup_log_file(log_file)
handler = logging.FileHandler(filename=log_file, encoding="utf-8")
handler.setFormatter(self.task_log_file_formatter)
handler.setLevel(self.level)
return handler
class EnvConfig(Config):
"""Databand's environment configuration"""
_conf__task_family = "env"
cloud_type = parameter(description="cloud type: gcp/aws/")[str]
env_label = parameter(
default=EnvLabel.dev,
description="environment type: dev/int/prod")[str] # label
production = parameter(
description="indicates that environment is production").value(False)
conn_id = parameter(default=None,
description="cloud connection settings")[str]
# MAIN OUTPUT FOLDER
root = parameter.folder[DirTarget]
# DATABAND SYSTEM FOLDERS
dbnd_root = parameter(description="DBND rooted home folder").output.folder(
default=None)[DirTarget]
dbnd_local_root = parameter(
description="DBND home for the local engine environment"
).output.folder()[DirTarget]
dbnd_data_sync_root = parameter(
description="Rooted directory for target syncing against remote engine"
).output.folder()[DirTarget]
# execution
local_engine = parameter(default="local_machine_engine",
description="Engine for local execution")[str]
remote_engine = parameter(
description="Remote engine for driver/tasks execution").none[str]
submit_driver = parameter(
description="Submit driver to remote_engine").none[bool]
submit_tasks = parameter(
description="Submit tasks to remote engine one by one").none[bool]
# properties that will affect "task-env" section
spark_config = task_env_param.help("Spark Configuration").value("spark")
spark_engine = task_env_param.help(
"Cluster engine (local/emr(aws)/dataproc(gcp)/..").value("spark_local")
hdfs = task_env_param.help("Hdfs cluster config").value("hdfs_knox")
beam_config = task_env_param.help("Apache Beam configuration").value(
"beam")
beam_engine = task_env_param.help(
"Apache Beam cluster engine (local/dataflow)").value(
ApacheBeamClusterType.local)
docker_engine = task_env_param.help(
"Docker job engine (docker/aws_batch)").value("docker")
def _initialize(self):
super(EnvConfig, self)._initialize()
try:
self.dbnd_root = self.dbnd_root or self.root.folder("dbnd")
if not self.dbnd_local_root:
if not self.dbnd_root.is_local():
raise friendly_error.config.dbnd_root_local_not_defined(
self.name)
self.dbnd_local_root = self.dbnd_root
except Exception as e:
raise friendly_error.task_build.failed_to_access_dbnd_home(
self.dbnd_root, e)
if not self.dbnd_data_sync_root:
self.dbnd_data_sync_root = self.dbnd_root.folder("sync")
if self.submit_driver is None:
self.submit_driver = bool(self.remote_engine)
if self.submit_tasks is None:
self.submit_tasks = bool(self.remote_engine)
@property
def name(self):
return self.task_meta.task_name
@property
def cloud_type(self):
return self.task_meta.task_family
def prepare_env(self):
pass
class RunConfig(config.Config):
"""Databand's per run settings (e.g. execution date)"""
_conf__task_family = "run"
######
# on none generated at DatabandRun
name = parameter.value(default=None, description="Specify run name")[str]
description = parameter.value(default=None, description="Specify run description")[
Optional[str]
]
# Executor configuration
parallel = parameter(default=None)[bool]
task_executor_type = parameter(
default=None,
description="Alternate executor type: "
" local/airflow_inprocess/airflow_multiprocess_local/airflow_kubernetes,"
" see docs for more options",
)[str]
enable_airflow_kubernetes = parameter(
default=True,
description="Enable use of kubernetes executor for kubebernetes engine submission",
)[bool]
######
# Local/Remote control
interactive = parameter(
default=False,
description="When submitting driver to remote execution keep tracking of submitted process and wait for completion",
)[bool]
submit_driver = parameter(
description="override env.submit_driver for specific environment"
).none[bool]
submit_tasks = parameter(
description="override env.submit_tasks for specific environment"
).none[bool]
# What to do on run
open_web_tracker_in_browser = parameter(
description="If True, open web tracker in browser during task run."
).value(False)
is_archived = parameter(description="Save this run in the archive").value(False)
dry = parameter(default=False).help(
"Do not execute tasks, stop before sending them to the execution, and print their status"
)[bool]
run_result_json_path = parameter(default=None).help(
"The path to save the task band of the run"
)[str]
debug_pydevd_pycharm_port = parameter(default=None).help(
"Enable debugging with `pydevd_pycharm` by setting this to the port value expecting the debugger to connect.\n"
"This will start a new `settrace` connecting to `localhost` on the requested port, "
"right before starting the driver task_run."
)[int]
######
# AIRFLOW EXECUTOR CONFIG
execution_date = parameter(default=None, description="Override execution date")[
datetime
]
mark_success = parameter(
description="Mark jobs as succeeded without running them"
).value(False)
######
# Task Selectors (to schedule specific task from pipeline)
id = parameter(default=None, description="The list of task ids to run")[List[str]]
selected_tasks_regex = parameter(
default=None, description="Run only specified tasks (regular expresion)"
)[List[str]]
ignore_dependencies = parameter(
description="The regex to filter specific task_ids"
).value(False)
ignore_first_depends_on_past = parameter(
description="The regex to filter specific task_ids"
).value(False)
######
# Scheduler configuration
skip_completed = parameter(
description="Mark jobs as succeeded without running them"
).value(True)
fail_fast = parameter(
description="Skip all remaining tasks if a task has failed"
).value(True)
enable_prod = parameter(description="Enable production tasks").value(False)
skip_completed_on_run = parameter(default=True).help(
"Should dbnd task check that task is completed and mark it as re-used on task execution"
)[bool]
validate_task_inputs = parameter(default=True).help(
"Should dbnd task check that all input files exist"
)[bool]
validate_task_outputs = parameter(default=True).help(
"Should dbnd task check that all outputs exist after task has been executed"
)[bool]
validate_task_outputs_on_build = parameter(default=False).help(
"Should dbnd task check that there are no incomplete outputs before task executes"
)[bool]
pipeline_band_only_check = parameter(default=False).help(
"When checking if pipeline is completed, check only if the band file exist (skip the tasks)"
)[bool]
recheck_circle_dependencies = parameter(
description="Re check circle dependencies on every task creation,"
" use it if you need to find of circle in your graph "
).value(False)
task_complete_parallelism_level = parameter(default=1).help(
"Number of threads to use when checking if tasks are already complete"
)[int]
pool = parameter(default=None, description="Resource pool to use")[str]
######
# Advanced Run settings (debug/workarounds)
# run .pickle file
always_save_pipeline = parameter(
description="Boolean for always saving pipeline to pickle"
).value(False)
disable_save_pipeline = parameter(
description="Boolean for disabling pipeline pickling"
).value(False)
donot_pickle = parameter(
description="Do not attempt to pickle the DAG object to send over "
"to the workers, just tell the workers to run their version "
"of the code."
).value(False)
pickle_handler = parameter(
default=None,
description="Defines a python pickle handler to be used to pickle the "
"run's data",
)[str]
enable_concurent_sqlite = parameter(
description="Enable concurrent execution with sqlite db (use only for debug!)"
).value(False)
######
# HEARTBEAT (process that updates on driver status every `heartbeat_interval_s`
#
heartbeat_interval_s = parameter(
description="How often a run should send a heartbeat to the server. Set -1 to disable"
)[int]
heartbeat_timeout_s = parameter(
description="How old can a run's last heartbeat be before we consider it failed. Set -1 to disable"
)[int]
heartbeat_sender_log_to_file = parameter(
description="create a separate log file for the heartbeat sender and don't log the run process stdout"
)[bool]
hearbeat_disable_plugins = parameter(
default=False, description="disable dbnd plugins at heartbeat sub-process"
)[bool]
######
# Task/Pipeline in task Execution
task_run_at_execution_time_enabled = parameter(
default=True, description="Allow tasks calls during another task execution"
)[bool]
task_run_at_execution_time_in_memory_outputs = parameter(
default=False,
description="Store outputs for inline task at execution time in memory (do not use FileSystem)",
)[bool]
target_cache_on_access = parameter(
default=True, description="Cache targets values in memory during execution"
)[bool]
