def setup_scheduled_dags(sub_dir=None, unlink_first=True): from airflow import settings sub_dir = sub_dir or settings.DAGS_FOLDER if not sub_dir: raise Exception( "Can't link scheduler: airflow's dag folder is undefined") source_path = dbnd_airflow_path("scheduler", "dags", "dbnd_dropin_scheduler.py") target_path = os.path.join(sub_dir, "dbnd_dropin_scheduler.py") if unlink_first and os.path.islink(target_path): try: logger.info("unlinking existing drop-in scheduler file at %s", target_path) os.unlink(target_path) except Exception: logger.error( "failed to unlink drop-in scheduler file at %s: %s", (target_path, format_exception_as_str()), ) return if not os.path.exists(target_path): try: logger.info("Linking %s to %s.", source_path, target_path) if not os.path.exists(sub_dir): os.makedirs(sub_dir) os.symlink(source_path, target_path) except Exception: logger.error( "failed to link drop-in scheduler in the airflow dags_folder: %s" % format_exception_as_str())
def link_dropin_file(source_path, target_path, unlink_first=True, name="file"): sub_dir = os.path.dirname(target_path) if unlink_first and os.path.islink(target_path): try: logger.info("unlinking existing %s at %s", name, target_path) os.unlink(target_path) except Exception: logger.error( "failed to unlink %s at %s: %s", (name, target_path, format_exception_as_str()), ) return if not os.path.exists(target_path): try: logger.info("Linking %s to %s.", source_path, target_path) if not os.path.exists(sub_dir): os.makedirs(sub_dir) os.symlink(source_path, target_path) except Exception: logger.error( "failed to link %s in the airflow dags_folder: %s" % (name, format_exception_as_str()) )
def format_exception_as_str(self, exc_info, isolate=True): if self.exception_simple: return format_exception_as_str(exc_info) try: tbvaccine = TBVaccine( no_colors=self.exception_no_color, show_vars=False, skip_non_user_on_isolate=True, isolate=isolate, ) return tbvaccine.format_tb(*exc_info) except Exception as ex: logger.info("Failed to format exception: %s", ex) return format_exception_as_str(exc_info)
def failed_to_import_user_module(ex, module, description): s = format_exception_as_str(sys.exc_info()) msg = "Module '%s' can not be loaded: %s" % ( module, dbnd_module_not_found_tip(module), ) return DatabandError( "%s exception: %s." % (msg, s), help_msg= " Databand is trying to load user module '%s' as required by %s: \n " "Probably, it has compile errors or not exists." % (module, description), show_exc_info=False, )
def send_heartbeat_continuously( run_uid, tracking_store, heartbeat_interval_s, driver_pid): # type: (str, TrackingStore, int, int) -> None logger.info( "[heartbeat sender] starting heartbeat sender process (pid %s) with a send interval of %s seconds" % (os.getpid(), heartbeat_interval_s)) try: while True: loop_start = time() try: if not pid_exists( driver_pid ): # failsafe, in case the driver process died violently logger.info( "[heartbeat sender] driver process %s stopped, stopping heartbeat sender", driver_pid, ) return run_state = tracking_store.heartbeat(run_uid=run_uid) logger.debug("[heartbeat sender] sent heartbeat") if run_state == RunState.SHUTDOWN.value: logger.info( "[heartbeat sender] received run state SHUTDOWN: killing driver process" ) os.kill(driver_pid, signal.SIGTERM) except KeyboardInterrupt: logger.info( "[heartbeat sender] stopping heartbeat sender process due to interrupt" ) return except Exception: logger.error( "[heartbeat sender] failed to send heartbeat: %s", format_exception_as_str(), ) time_to_sleep_s = max(0, time() + heartbeat_interval_s - loop_start) if time_to_sleep_s > 0: sleep(time_to_sleep_s) except KeyboardInterrupt: return except Exception: logger.exception("[heartbeat sender] Failed to run heartbeat") finally: logger.info("[heartbeat sender] stopping heartbeat sender") sys.exit(0)
def get_databand_error_message(ex, args=None, sys_exit=True): args = args or sys.argv please_report = False print_source = True if isinstance(ex, DatabandRunError): # we already printed all information! return ( "There is an error! Your run has failed!", DatabandExitCodes.execution_failed, ) if isinstance(ex, DatabandRuntimeError): exit_code = DatabandExitCodes.execution_failed elif isinstance(ex, DatabandConfigError): exit_code = DatabandExitCodes.configuration_error elif isinstance(ex, DatabandSystemError): exit_code = DatabandExitCodes.error please_report = True elif isinstance(ex, DatabandError): exit_code = DatabandExitCodes.error elif ex.__class__.__name__ == "NoCredentialsError": # aws exit_code = DatabandExitCodes.configuration_error ex = friendly_error.config.no_credentials() print_source = False else: please_report = True exit_code = DatabandExitCodes.unknown_error msg = str(ex) extra_msg_lines = [] nested_exceptions = nested_exceptions_str(ex) if nested_exceptions: extra_msg_lines.append("Caused by: \n%s\n" % indent(nested_exceptions, "\t")) help_msg = get_help_msg(ex) if help_msg: extra_msg_lines.append(" Help: \n%s\n" % indent(help_msg, "\t")) user_frame_info_str = get_user_frame_info_str(ex) if user_frame_info_str and print_source: extra_msg_lines.append("Source: \n%s\n" % indent(user_frame_info_str, "\t")) # if we crashed before finishing bootstrap we probably want to see the full trace, and we could have failed during config init so the verbose flag does nothing if (show_exc_info(ex) or config.getboolean("databand", "verbose") or not bootstrap._dbnd_bootstrap): error_info = sys.exc_info() extra_msg_lines.append(format_exception_as_str(error_info)) msg = truncate_msg(msg, ERROR_MESSAGE_HEAD_SIZE, ERROR_MESSAGE_TAIL_SIZE) if please_report: extra_msg_lines.append( " Please report it to [email protected] or appropriate slack channel!" ) msg = ("There is an error! Your run has failed with {exc_type}\n" "{sep}\n" " Command line: {command_line}\n" " Failure:\n{msg}\n\n" "{extra_msg}\n" "{sep}\n" "".format( sep=console_utils.error_separator(), command_line=subprocess.list2cmdline(args or []), sep_small=console_utils.error_separator_small(), msg=console_utils.bold(indent(msg, "\t")), exc_type=ex.__class__.__name__, extra_msg="\n ".join(extra_msg_lines), )) return msg, exit_code