def __init__(self,
base_log_folder,
filename_template,
log_id_template,
end_of_log_mark,
write_stdout,
json_format,
record_labels,
host='localhost:9200'):
"""
:param base_log_folder: base folder to store logs locally
:param log_id_template: log id template
:param host: Elasticsearch host name
"""
super(ElasticsearchTaskHandler, self).__init__(base_log_folder,
filename_template)
self.closed = False
self.log_id_template, self.log_id_jinja_template = \
parse_template_string(log_id_template)
self.client = elasticsearch.Elasticsearch([host])
self.mark_end_on_close = True
self.end_of_log_mark = end_of_log_mark
self.write_stdout = write_stdout
self.json_format = json_format
self.record_labels = [
label.strip() for label in record_labels.split(",")
]
self.handler = None
def __init__(self, level: Union[str, int] = None):
super().__init__(level=level)
self.filename_template, self.filename_jinja_template = parse_template_string(
TASK_LOG_FILENAME_TEMPLATE)
self._task_context_info: ExecutionLogTaskContextInfo = None
self._task_instance: TaskInstance = None
self._logfile_subpath: str = None
def __init__( # pylint: disable=too-many-arguments
self,
base_log_folder: str,
filename_template: str,
log_id_template: str,
end_of_log_mark: str,
write_stdout: bool,
json_format: bool,
json_fields: str,
host: str = "localhost:9200",
frontend: str = "localhost:5601",
es_kwargs: Optional[dict] = conf.getsection("elasticsearch_configs"),
):
"""
:param base_log_folder: base folder to store logs locally
:param log_id_template: log id template
:param host: Elasticsearch host name
"""
es_kwargs = es_kwargs or {}
super().__init__(base_log_folder, filename_template)
self.closed = False
self.log_id_template, self.log_id_jinja_template = parse_template_string(
log_id_template)
self.client = elasticsearch.Elasticsearch([host], **es_kwargs)
self.frontend = frontend
self.mark_end_on_close = True
self.end_of_log_mark = end_of_log_mark
self.write_stdout = write_stdout
self.json_format = json_format
self.json_fields = [label.strip() for label in json_fields.split(",")]
self.handler = None
self.context_set = False
def __init__(self,
base_log_folder,
filename_template,
log_id_template,
end_of_log_mark,
write_stdout,
json_format,
json_fields,
host='localhost:9200',
es_kwargs=conf.getsection("elasticsearch_configs") or {}):
"""
:param base_log_folder: base folder to store logs locally
:param log_id_template: log id template
:param host: Elasticsearch host name
"""
super().__init__(base_log_folder, filename_template)
self.closed = False
self.log_id_template, self.log_id_jinja_template = \
parse_template_string(log_id_template)
self.client = elasticsearch.Elasticsearch([host], **es_kwargs)
self.mark_end_on_close = True
self.end_of_log_mark = end_of_log_mark
self.write_stdout = write_stdout
self.json_format = json_format
self.json_fields = [label.strip() for label in json_fields.split(",")]
self.handler = None
def __init__(self, base_log_folder, filename_template):
"""
:param base_log_folder: Base log folder to place logs.
:param filename_template: template filename string
"""
super().__init__()
self.handler = None
self.local_base = base_log_folder
self.filename_template, self.filename_jinja_template = \
parse_template_string(filename_template)
def __init__(self, level: Union[str, int] = None):
"""
:param filename_template: template filename string
"""
super().__init__(level=level)
self.dag_dir: str = os.path.expanduser(DAGS_FOLDER)
self._log_filepath: str = os.path.join("process", "global.log")
self.filename_template, self.filename_jinja_template = parse_template_string(
PROCESS_LOG_FILENAME_TEMPLATE)
def __init__(self, base_log_folder, filename_template):
"""
:param base_log_folder: Base log folder to place logs.
:param filename_template: template filename string
"""
super().__init__()
self.handler = None
self.local_base = base_log_folder
self.filename_template, self.filename_jinja_template = \
parse_template_string(filename_template)
def __init__(self, base_log_folder, filename_template):
super().__init__()
self.handler = None
self.base_log_folder = base_log_folder
self.dag_dir = os.path.expanduser(settings.DAGS_FOLDER)
self.filename_template, self.filename_jinja_template = parse_template_string(filename_template)
self._cur_date = datetime.today()
Path(self._get_log_directory()).mkdir(parents=True, exist_ok=True)
self._symlink_latest_log_directory()
def set_context(self, ti):
if ti.raw:
return
prefix = conf.get('core', 'task_log_prefix_template')
rendered_prefix = ""
if prefix:
_, self.prefix_jinja_template = parse_template_string(prefix)
rendered_prefix = self._render_prefix(ti)
self.setFormatter(logging.Formatter(rendered_prefix + ":" + self.formatter._fmt))
self.setLevel(self.level)
def set_context(self, ti):
"""
Accept the run-time context (i.e. the current task) and configure the formatter accordingly.
:param ti:
:return:
"""
if ti.raw:
return
prefix = conf.get('logging', 'task_log_prefix_template')
rendered_prefix = ""
if prefix:
_, self.prefix_jinja_template = parse_template_string(prefix)
rendered_prefix = self._render_prefix(ti)
formatter = logging.Formatter(rendered_prefix + ":" + self.formatter._fmt) # pylint: disable=W0212
self.setFormatter(formatter)
self.setLevel(self.level)
def execute(self, context):
print(f'Querying PatentsView API for {self.entity} with parameters in {self.query_file_path}')
# read in query json file
with open(self.query_file_path, 'r') as f:
query = json.load(f)
# manually render airflow macro in json file
_, template = parse_template_string(json.dumps(query))
templated_query = template.render(**context)
# init hook and post to api
hook = PatentsViewHook()
response = hook.post(self.entity, json.loads(templated_query))
with open(self.response_file_path, 'w') as f:
json.dump(response, f)
print(f'Saved results to {self.response_file_path}')
def __init__(self, base_log_folder, filename_template,
log_id_template, end_of_log_mark,
host='localhost:9200'):
"""
:param base_log_folder: base folder to store logs locally
:param log_id_template: log id template
:param host: Elasticsearch host name
"""
super(ElasticsearchTaskHandler, self).__init__(
base_log_folder, filename_template)
self.closed = False
self.log_id_template, self.log_id_jinja_template = \
parse_template_string(log_id_template)
self.client = elasticsearch.Elasticsearch([host])
self.mark_end_on_close = True
self.end_of_log_mark = end_of_log_mark
def __init__(self, base_log_folder, filename_template,
log_id_template, end_of_log_mark,
host='localhost:9200'):
"""
:param base_log_folder: base folder to store logs locally
:param log_id_template: log id template
:param host: Elasticsearch host name
"""
super(ElasticsearchTaskHandler, self).__init__(
base_log_folder, filename_template)
self.closed = False
self.log_id_template, self.log_id_jinja_template = \
parse_template_string(log_id_template)
self.client = elasticsearch.Elasticsearch([host])
self.mark_end_on_close = True
self.end_of_log_mark = end_of_log_mark
def set_context(self, ti):
local_loc = self._init_file(ti)
self.handler = logging.FileHandler(local_loc)
suffix = conf.get('core', 'task_log_suffix_template')
encode = conf.get('core', 'base64_encoding')
rendered_suffix = ""
if suffix:
self.suffix_template, self.suffix_jinja_template = parse_template_string(
suffix)
rendered_suffix = self._render_suffix(ti, ti.try_number)
if encode:
rendered_suffix = base64.b64encode(
rendered_suffix.encode()).decode()
self.handler.setFormatter(
logging.Formatter(self.formatter._fmt + ":" + rendered_suffix))
self.handler.setLevel(self.level)
def _render_filename(self, ti: "TaskInstance", try_number: int) -> str:
with create_session() as session:
dag_run = ti.get_dagrun(session=session)
template = dag_run.get_log_template(session=session).filename
str_tpl, jinja_tpl = parse_template_string(template)
if jinja_tpl:
if hasattr(ti, "task"):
context = ti.get_template_context()
else:
context = Context(ti=ti, ts=dag_run.logical_date.isoformat())
context["try_number"] = try_number
return render_template_to_string(jinja_tpl, context)
elif str_tpl:
dag = ti.task.dag
assert dag is not None # For Mypy.
try:
data_interval: Tuple[
datetime, datetime] = dag.get_run_data_interval(dag_run)
except AttributeError: # ti.task is not always set.
data_interval = (dag_run.data_interval_start,
dag_run.data_interval_end)
if data_interval[0]:
data_interval_start = data_interval[0].isoformat()
else:
data_interval_start = ""
if data_interval[1]:
data_interval_end = data_interval[1].isoformat()
else:
data_interval_end = ""
return str_tpl.format(
dag_id=ti.dag_id,
task_id=ti.task_id,
run_id=ti.run_id,
data_interval_start=data_interval_start,
data_interval_end=data_interval_end,
execution_date=ti.get_dagrun().logical_date.isoformat(),
try_number=try_number,
)
else:
raise RuntimeError(
f"Unable to render log filename for {ti}. This should never happen"
)
def set_context(self, ti) -> None:
"""
Accept the run-time context (i.e. the current task) and configure the formatter accordingly.
:param ti:
:return:
"""
if ti.raw or self.formatter is None:
return
prefix = conf.get('logging', 'task_log_prefix_template')
if prefix:
_, self.prefix_jinja_template = parse_template_string(prefix)
rendered_prefix = self._render_prefix(ti)
else:
rendered_prefix = ""
formatter = logging.Formatter(
f"{rendered_prefix}:{self.formatter._fmt}")
self.setFormatter(formatter)
self.setLevel(self.level)
def set_context(self, ti, *, session: "Session" = NEW_SESSION) -> None:
"""
Accept the run-time context (i.e. the current task) and configure the formatter accordingly.
:param ti:
:return:
"""
if ti.raw or self.formatter is None:
return
prefix = ti.get_dagrun().get_task_prefix_template(session=session)
if prefix:
_, self.prefix_jinja_template = parse_template_string(prefix)
rendered_prefix = self._render_prefix(ti)
else:
rendered_prefix = ""
formatter = logging.Formatter(
f"{rendered_prefix}:{self.formatter._fmt}")
self.setFormatter(formatter)
self.setLevel(self.level)
def __init__(self, base_log_folder, filename_template):
super().__init__()
self.handler = None
self.base_log_folder = base_log_folder
self.dag_dir = os.path.expanduser(settings.DAGS_FOLDER)
self.filename_template, self.filename_jinja_template = \
parse_template_string(filename_template)
self._cur_date = datetime.today()
if not os.path.exists(self._get_log_directory()):
try:
os.makedirs(self._get_log_directory())
except OSError:
# only ignore case where the directory already exist
if not os.path.isdir(self._get_log_directory()):
raise
logging.warning("%s already exists", self._get_log_directory())
self._symlink_latest_log_directory()
def __init__(self, base_log_folder, filename_template):
"""
:param base_log_folder: Base log folder to place logs.
:param filename_template: template filename string
"""
super(FileProcessorHandler, self).__init__()
self.handler = None
self.base_log_folder = base_log_folder
self.dag_dir = os.path.expanduser(conf.get('core', 'DAGS_FOLDER'))
self.filename_template, self.filename_jinja_template = \
parse_template_string(filename_template)
self._cur_date = datetime.today()
if not os.path.exists(self._get_log_directory()):
try:
os.makedirs(self._get_log_directory())
except OSError as e:
# only ignore case where the directory already exist
if e.errno != errno.EEXIST:
raise
logging.warning("%s already exists", self._get_log_directory())
self._symlink_latest_log_directory()
def __init__(self, base_log_folder, filename_template):
"""
:param base_log_folder: Base log folder to place logs.
:param filename_template: template filename string
"""
super().__init__()
self.handler = None
self.base_log_folder = base_log_folder
self.dag_dir = os.path.expanduser(settings.DAGS_FOLDER)
self.filename_template, self.filename_jinja_template = \
parse_template_string(filename_template)
self._cur_date = datetime.today()
if not os.path.exists(self._get_log_directory()):
try:
os.makedirs(self._get_log_directory())
except OSError:
# only ignore case where the directory already exist
if not os.path.isdir(self._get_log_directory()):
raise
logging.warning("%s already exists", self._get_log_directory())
self._symlink_latest_log_directory()
def __init__(self, base_log_folder, filename_template):
"""
:param base_log_folder: Base log folder to place logs.
:param filename_template: template filename string
"""
super(FileProcessorHandler, self).__init__()
self.handler = None
self.base_log_folder = base_log_folder
self.dag_dir = os.path.expanduser(conf.get('core', 'DAGS_FOLDER'))
self.filename_template, self.filename_jinja_template = \
parse_template_string(filename_template)
self._cur_date = datetime.today()
if not os.path.exists(self._get_log_directory()):
try:
os.makedirs(self._get_log_directory())
except OSError as e:
# only ignore case where the directory already exist
if e.errno != errno.EEXIST:
raise
logging.warning("%s already exists", self._get_log_directory())
self._symlink_latest_log_directory()
def __init__(self, base_log_folder: str, filename_template: str):
super().__init__()
self.handler = None # type: Optional[logging.FileHandler]
self.local_base = base_log_folder
self.filename_template, self.filename_jinja_template = parse_template_string(filename_template)
