def get_task_logs(dag_id, task_id, pendulum_execution_date, session):
if not dag_id or not task_id or not pendulum_execution_date:
return None
dagbag = models.DagBag(
os.devnull, # to initialize an empty dag bag
store_serialized_dags=STORE_SERIALIZED_DAGS,
)
dag = dagbag.get_dag(dag_id)
ti = (
session.query(models.TaskInstance)
.filter(
models.TaskInstance.dag_id == dag_id,
models.TaskInstance.task_id == task_id,
models.TaskInstance.execution_date == pendulum_execution_date,
)
.first()
)
ti.task = dag.get_task(ti.task_id)
file_task_handler = FileTaskHandler(
base_log_folder=conf.get("core", "BASE_LOG_FOLDER"),
filename_template=conf.get("core", "LOG_FILENAME_TEMPLATE"),
)
logs, metadatas = file_task_handler.read(ti, None, None)
return logs, metadatas
def test_python_formatting(self):
expected_filename = 'dag_for_testing_filename_rendering/task_for_testing_filename_rendering/%s/42.log' % DEFAULT_DATE.isoformat(
)
fth = FileTaskHandler(
'', '{dag_id}/{task_id}/{execution_date}/{try_number}.log')
rendered_filename = fth._render_filename(self.ti, 42)
self.assertEqual(expected_filename, rendered_filename)
def test_jinja_rendering(self):
expected_filename = \
'dag_for_testing_filename_rendering/task_for_testing_filename_rendering/%s/42.log' \
% DEFAULT_DATE.isoformat()
fth = FileTaskHandler('', '{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log')
rendered_filename = fth._render_filename(self.ti, 42)
self.assertEqual(expected_filename, rendered_filename)
def test_jinja_rendering(self):
expected_filename = \
'dag_for_testing_filename_rendering/task_for_testing_filename_rendering/%s/42.log' \
% DEFAULT_DATE.isoformat()
fth = FileTaskHandler('', '{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log')
rendered_filename = fth._render_filename(self.ti, 42)
self.assertEqual(expected_filename, rendered_filename)
def test_python_formatting(self):
expected_filename = \
'dag_for_testing_filename_rendering/task_for_testing_filename_rendering/%s/42.log' \
% DEFAULT_DATE.isoformat()
fth = FileTaskHandler('', '{dag_id}/{task_id}/{execution_date}/{try_number}.log')
rendered_filename = fth._render_filename(self.ti, 42)
self.assertEqual(expected_filename, rendered_filename)
def _read(self, ti, try_number):
"""
Read logs of given task instance and try_number from GCS.
If failed, read the log from BigQuery.
:param ti: task instance object
:type ti: airflow.models.TypeInstance
:param try_number: task instance try_number to read logs from
:type try_number: int
"""
# Explicitly getting log relative path is necessary as the given
# task instance might be different than task instance passed in
# in set_context method.
log_relative_path = self._render_filename(ti, try_number + 1)
remote_loc = os.path.join(self.remote_base, log_relative_path)
if self.gcs_log_exists(remote_loc):
# If GCS remote file exists, we do not fetch logs from task instance
# local machine even if there are errors reading remote logs, as
# remote_log will contain error message.
remote_log = self.gcs_read(remote_loc, return_error=True)
log = '*** Reading remote log from {}.\n{}\n'.format(
remote_loc, remote_log)
return log
# everything above is copied from gcs_task_handler.py, the following elif statement includes changes
# specifically for bluecore_gcs_task_handler
elif ti.operator == "KubernetesJobOperator":
# If remote file is not available and the job is a Kubernetes Job,
# use the task instance attributes to query BigQuery logs and
# make the logs available in real time from the "Log" page of the UI.
# TODO: PROD-19910 job_name is always none on the first try, should fix this so an exception doesn't show each time
job_name = ti.xcom_pull(task_ids=ti.task_id,
key='kubernetes_job_name')
if job_name is None:
return "An XCOM kubernetes job_name was not found. This does not mean the job has failed. It is " \
"possible that a job_name has not yet been created. Try refreshing the page. "
pod_output = BQGCSTaskHandler.get_pods(job_name)
billing_project_id = configuration.get(
'core', 'kubernetes_bigquery_billing_project')
client = bigquery.Client(billing_project_id)
# get info for the BQ table name
data_project_name = configuration.get(
'core', 'kubernetes_bigquery_data_project')
start_date = ti.start_date.strftime("%Y%m%d")
end_date = datetime.utcnow().date().strftime("%Y%m%d")
log_lines = []
tables = []
for pod in pod_output['items']:
pod_id = pod['metadata']['name']
for container in pod['spec']['containers']:
container_name = string.replace(container['name'], '-',
'_')
bq_table_name = "{}.gke_logs.{}_*".format(
data_project_name, container_name)
tables.append((bq_table_name, pod_id))
if not tables:
message = "No pods were found running for this task, and the task's output has not been written to " \
"GCS. If refreshing this page doesn't present a log, it's possible that this task failed " \
"to spawn a kubernetes job.\n\n"
message += json.dumps(tables) + "\n\n"
try:
message += FileTaskHandler._read(self, ti, try_number)
except:
message += "Failed to read airflow worker log! Stack trace:\n{}".format(
traceback.format_exc())
return message
query = ("\n UNION ALL \n ".join([
BQGCSTaskHandler.generate_query(bq_table_name=bq_table_name,
pod_id=pod_id,
start_date=start_date,
end_date=end_date)
for bq_table_name, pod_id in tables
]) + "\n ORDER BY pod_id, logName, timestamp")
try:
query_job = client.query(query)
result = query_job.result()
except BadRequest as e:
return "BadRequest error from BigQuery. The query may be empty or the job finished. Try refreshing " \
"the page. \n {e}".format(e=e)
log_name = ""
for index, row in enumerate(result):
if row.logName.split("/")[-1:][0] != log_name:
log_name = row.logName.split("/")[-1:][0]
log_lines.append(
"LOGGING OUTPUT FROM {log_name} \n".format(
log_name=log_name))
log_lines.append("{} {}".format(
row.timestamp.strftime("%Y-%m-%d %H:%M:%S.%f")[:-4],
row.textPayload))
return "".join(log_lines)
# else statement taken from gcs_task_handler.py
else:
log = super(BQGCSTaskHandler, self)._read(ti, try_number)
return log
