def _configure_loggers(self):
# Create a custom logging configuration
logging_config = copy.deepcopy(DEFAULT_LOGGING_CONFIG)
logging_config['handlers']['task']['base_log_folder'] = self.log_dir
logging_config['handlers']['task'][
'filename_template'] = '{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts | replace(":", ".") }}/{{ try_number }}.log'
# Write the custom logging configuration to a file
self.settings_folder = tempfile.mkdtemp()
settings_file = os.path.join(self.settings_folder,
"airflow_local_settings.py")
new_logging_file = "LOGGING_CONFIG = {}".format(logging_config)
with open(settings_file, 'w') as handle:
handle.writelines(new_logging_file)
sys.path.append(self.settings_folder)
with conf_vars({
('logging', 'logging_config_class'):
'airflow_local_settings.LOGGING_CONFIG',
("api", "auth_backend"):
"tests.test_utils.remote_user_api_auth_backend",
}):
self.app = app.create_app(testing=True)
self.client = self.app.test_client()
settings.configure_logging()
def assert_remote_logs(self, expected_message, ti):
with provide_gcp_context(GCP_STACKDDRIVER), conf_vars({
('logging', 'remote_logging'): 'True',
('logging', 'remote_base_log_folder'): f"stackdriver://{self.log_name}",
}):
from airflow.config_templates import airflow_local_settings
importlib.reload(airflow_local_settings)
settings.configure_logging()
task_log_reader = TaskLogReader()
logs = "\n".join(task_log_reader.read_log_stream(ti, try_number=None, metadata={}))
self.assertIn(expected_message, logs)
def assert_remote_logs(self, expected_message, ti):
with provide_gcp_context(GCP_GCS_KEY), conf_vars({
('logging', 'remote_logging'): 'True',
('logging', 'remote_base_log_folder'): f"gs://{self.bucket_name}/path/to/logs",
('logging', 'remote_log_conn_id'): "google_cloud_default",
}):
from airflow.config_templates import airflow_local_settings
importlib.reload(airflow_local_settings)
settings.configure_logging()
task_log_reader = TaskLogReader()
logs = "\n".join(task_log_reader.read_log_stream(ti, try_number=None, metadata={}))
self.assertIn(expected_message, logs)
def _configure_loggers(self):
logging_config = copy.deepcopy(DEFAULT_LOGGING_CONFIG)
logging_config["handlers"]["task"]["base_log_folder"] = self.log_dir
logging_config["handlers"]["task"][
"filename_template"
] = "{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts | replace(':', '.') }}/{{ try_number }}.log"
settings_file = os.path.join(self.settings_folder, "airflow_local_settings.py")
with open(settings_file, "w") as handle:
new_logging_file = f"LOGGING_CONFIG = {logging_config}"
handle.writelines(new_logging_file)
sys.path.append(self.settings_folder)
with conf_vars({("logging", "logging_config_class"): "airflow_local_settings.LOGGING_CONFIG"}):
settings.configure_logging()
def test(self):
self.logger.setLevel(logging.DEBUG)
self.logger.addHandler(self.handler)
self.assertEqual(self.logger.handlers, [self.handler])
settings.configure_logging()
# Make sure our log settings are preserved.
self.assertEqual(self.logger.level, logging.DEBUG)
self.assertIn(self.handler, self.logger.handlers)
# Make sure default StreamHandler got added.
self.assertIn(settings.handler, self.logger.handlers)
def test(self):
self.logger.setLevel(logging.DEBUG)
self.logger.addHandler(self.handler)
self.assertEqual(self.logger.handlers, [self.handler])
settings.configure_logging()
# Make sure our log settings are preserved.
self.assertEqual(self.logger.level, logging.DEBUG)
self.assertIn(self.handler, self.logger.handlers)
# Make sure default StreamHandler got added.
self.assertIn(settings.handler, self.logger.handlers)
def tearDown(self) -> None:
from airflow.config_templates import airflow_local_settings
importlib.reload(airflow_local_settings)
settings.configure_logging()
clear_db_runs()
