def setup_scheduled_dags(sub_dir=None, unlink_first=True):
from airflow import settings
sub_dir = sub_dir or settings.DAGS_FOLDER
if not sub_dir:
raise Exception(
"Can't link scheduler: airflow's dag folder is undefined")
source_path = dbnd_airflow_path("scheduler", "dags",
"dbnd_dropin_scheduler.py")
target_path = os.path.join(sub_dir, "dbnd_dropin_scheduler.py")
if unlink_first and os.path.islink(target_path):
try:
logger.info("unlinking existing drop-in scheduler file at %s",
target_path)
os.unlink(target_path)
except Exception:
logger.error(
"failed to unlink drop-in scheduler file at %s: %s",
(target_path, format_exception_as_str()),
)
return
if not os.path.exists(target_path):
try:
logger.info("Linking %s to %s.", source_path, target_path)
if not os.path.exists(sub_dir):
os.makedirs(sub_dir)
os.symlink(source_path, target_path)
except Exception:
logger.error(
"failed to link drop-in scheduler in the airflow dags_folder: %s"
% format_exception_as_str())
def link_dropin_file(source_path, target_path, unlink_first=True, name="file"):
sub_dir = os.path.dirname(target_path)
if unlink_first and os.path.islink(target_path):
try:
logger.info("unlinking existing %s at %s", name, target_path)
os.unlink(target_path)
except Exception:
logger.error(
"failed to unlink %s at %s: %s",
(name, target_path, format_exception_as_str()),
)
return
if not os.path.exists(target_path):
try:
logger.info("Linking %s to %s.", source_path, target_path)
if not os.path.exists(sub_dir):
os.makedirs(sub_dir)
os.symlink(source_path, target_path)
except Exception:
logger.error(
"failed to link %s in the airflow dags_folder: %s"
% (name, format_exception_as_str())
)
def format_exception_as_str(self, exc_info, isolate=True):
if self.exception_simple:
return format_exception_as_str(exc_info)
try:
tbvaccine = TBVaccine(
no_colors=self.exception_no_color,
show_vars=False,
skip_non_user_on_isolate=True,
isolate=isolate,
)
return tbvaccine.format_tb(*exc_info)
except Exception as ex:
logger.info("Failed to format exception: %s", ex)
return format_exception_as_str(exc_info)
def failed_to_import_user_module(ex, module, description):
s = format_exception_as_str(sys.exc_info())
msg = "Module '%s' can not be loaded: %s" % (
module,
dbnd_module_not_found_tip(module),
)
return DatabandError(
"%s exception: %s." % (msg, s),
help_msg=
" Databand is trying to load user module '%s' as required by %s: \n "
"Probably, it has compile errors or not exists." %
(module, description),
show_exc_info=False,
)
def send_heartbeat_continuously(
run_uid, tracking_store, heartbeat_interval_s,
driver_pid): # type: (str, TrackingStore, int, int) -> None
logger.info(
"[heartbeat sender] starting heartbeat sender process (pid %s) with a send interval of %s seconds"
% (os.getpid(), heartbeat_interval_s))
try:
while True:
loop_start = time()
try:
if not pid_exists(
driver_pid
): # failsafe, in case the driver process died violently
logger.info(
"[heartbeat sender] driver process %s stopped, stopping heartbeat sender",
driver_pid,
)
return
run_state = tracking_store.heartbeat(run_uid=run_uid)
logger.debug("[heartbeat sender] sent heartbeat")
if run_state == RunState.SHUTDOWN.value:
logger.info(
"[heartbeat sender] received run state SHUTDOWN: killing driver process"
)
os.kill(driver_pid, signal.SIGTERM)
except KeyboardInterrupt:
logger.info(
"[heartbeat sender] stopping heartbeat sender process due to interrupt"
)
return
except Exception:
logger.error(
"[heartbeat sender] failed to send heartbeat: %s",
format_exception_as_str(),
)
time_to_sleep_s = max(0,
time() + heartbeat_interval_s - loop_start)
if time_to_sleep_s > 0:
sleep(time_to_sleep_s)
except KeyboardInterrupt:
return
except Exception:
logger.exception("[heartbeat sender] Failed to run heartbeat")
finally:
logger.info("[heartbeat sender] stopping heartbeat sender")
sys.exit(0)
def get_databand_error_message(ex, args=None, sys_exit=True):
args = args or sys.argv
please_report = False
print_source = True
if isinstance(ex, DatabandRunError):
# we already printed all information!
return (
"There is an error! Your run has failed!",
DatabandExitCodes.execution_failed,
)
if isinstance(ex, DatabandRuntimeError):
exit_code = DatabandExitCodes.execution_failed
elif isinstance(ex, DatabandConfigError):
exit_code = DatabandExitCodes.configuration_error
elif isinstance(ex, DatabandSystemError):
exit_code = DatabandExitCodes.error
please_report = True
elif isinstance(ex, DatabandError):
exit_code = DatabandExitCodes.error
elif ex.__class__.__name__ == "NoCredentialsError": # aws
exit_code = DatabandExitCodes.configuration_error
ex = friendly_error.config.no_credentials()
print_source = False
else:
please_report = True
exit_code = DatabandExitCodes.unknown_error
msg = str(ex)
extra_msg_lines = []
nested_exceptions = nested_exceptions_str(ex)
if nested_exceptions:
extra_msg_lines.append("Caused by: \n%s\n" %
indent(nested_exceptions, "\t"))
help_msg = get_help_msg(ex)
if help_msg:
extra_msg_lines.append(" Help: \n%s\n" % indent(help_msg, "\t"))
user_frame_info_str = get_user_frame_info_str(ex)
if user_frame_info_str and print_source:
extra_msg_lines.append("Source: \n%s\n" %
indent(user_frame_info_str, "\t"))
# if we crashed before finishing bootstrap we probably want to see the full trace, and we could have failed during config init so the verbose flag does nothing
if (show_exc_info(ex) or config.getboolean("databand", "verbose")
or not bootstrap._dbnd_bootstrap):
error_info = sys.exc_info()
extra_msg_lines.append(format_exception_as_str(error_info))
msg = truncate_msg(msg, ERROR_MESSAGE_HEAD_SIZE, ERROR_MESSAGE_TAIL_SIZE)
if please_report:
extra_msg_lines.append(
" Please report it to [email protected] or appropriate slack channel!"
)
msg = ("There is an error! Your run has failed with {exc_type}\n"
"{sep}\n"
" Command line: {command_line}\n"
" Failure:\n{msg}\n\n"
"{extra_msg}\n"
"{sep}\n"
"".format(
sep=console_utils.error_separator(),
command_line=subprocess.list2cmdline(args or []),
sep_small=console_utils.error_separator_small(),
msg=console_utils.bold(indent(msg, "\t")),
exc_type=ex.__class__.__name__,
extra_msg="\n ".join(extra_msg_lines),
))
return msg, exit_code
