def __init__(self, base_log_folder, filename_template, log_id_template, end_of_log_mark, write_stdout, json_format, record_labels, host='localhost:9200'): """ :param base_log_folder: base folder to store logs locally :param log_id_template: log id template :param host: Elasticsearch host name """ super(ElasticsearchTaskHandler, self).__init__(base_log_folder, filename_template) self.closed = False self.log_id_template, self.log_id_jinja_template = \ parse_template_string(log_id_template) self.client = elasticsearch.Elasticsearch([host]) self.mark_end_on_close = True self.end_of_log_mark = end_of_log_mark self.write_stdout = write_stdout self.json_format = json_format self.record_labels = [ label.strip() for label in record_labels.split(",") ] self.handler = None
def __init__(self, level: Union[str, int] = None): super().__init__(level=level) self.filename_template, self.filename_jinja_template = parse_template_string( TASK_LOG_FILENAME_TEMPLATE) self._task_context_info: ExecutionLogTaskContextInfo = None self._task_instance: TaskInstance = None self._logfile_subpath: str = None
def __init__( # pylint: disable=too-many-arguments self, base_log_folder: str, filename_template: str, log_id_template: str, end_of_log_mark: str, write_stdout: bool, json_format: bool, json_fields: str, host: str = "localhost:9200", frontend: str = "localhost:5601", es_kwargs: Optional[dict] = conf.getsection("elasticsearch_configs"), ): """ :param base_log_folder: base folder to store logs locally :param log_id_template: log id template :param host: Elasticsearch host name """ es_kwargs = es_kwargs or {} super().__init__(base_log_folder, filename_template) self.closed = False self.log_id_template, self.log_id_jinja_template = parse_template_string( log_id_template) self.client = elasticsearch.Elasticsearch([host], **es_kwargs) self.frontend = frontend self.mark_end_on_close = True self.end_of_log_mark = end_of_log_mark self.write_stdout = write_stdout self.json_format = json_format self.json_fields = [label.strip() for label in json_fields.split(",")] self.handler = None self.context_set = False
def __init__(self, base_log_folder, filename_template, log_id_template, end_of_log_mark, write_stdout, json_format, json_fields, host='localhost:9200', es_kwargs=conf.getsection("elasticsearch_configs") or {}): """ :param base_log_folder: base folder to store logs locally :param log_id_template: log id template :param host: Elasticsearch host name """ super().__init__(base_log_folder, filename_template) self.closed = False self.log_id_template, self.log_id_jinja_template = \ parse_template_string(log_id_template) self.client = elasticsearch.Elasticsearch([host], **es_kwargs) self.mark_end_on_close = True self.end_of_log_mark = end_of_log_mark self.write_stdout = write_stdout self.json_format = json_format self.json_fields = [label.strip() for label in json_fields.split(",")] self.handler = None
def __init__(self, base_log_folder, filename_template): """ :param base_log_folder: Base log folder to place logs. :param filename_template: template filename string """ super().__init__() self.handler = None self.local_base = base_log_folder self.filename_template, self.filename_jinja_template = \ parse_template_string(filename_template)
def __init__(self, level: Union[str, int] = None): """ :param filename_template: template filename string """ super().__init__(level=level) self.dag_dir: str = os.path.expanduser(DAGS_FOLDER) self._log_filepath: str = os.path.join("process", "global.log") self.filename_template, self.filename_jinja_template = parse_template_string( PROCESS_LOG_FILENAME_TEMPLATE)
def __init__(self, base_log_folder, filename_template): super().__init__() self.handler = None self.base_log_folder = base_log_folder self.dag_dir = os.path.expanduser(settings.DAGS_FOLDER) self.filename_template, self.filename_jinja_template = parse_template_string(filename_template) self._cur_date = datetime.today() Path(self._get_log_directory()).mkdir(parents=True, exist_ok=True) self._symlink_latest_log_directory()
def set_context(self, ti): if ti.raw: return prefix = conf.get('core', 'task_log_prefix_template') rendered_prefix = "" if prefix: _, self.prefix_jinja_template = parse_template_string(prefix) rendered_prefix = self._render_prefix(ti) self.setFormatter(logging.Formatter(rendered_prefix + ":" + self.formatter._fmt)) self.setLevel(self.level)
def set_context(self, ti): """ Accept the run-time context (i.e. the current task) and configure the formatter accordingly. :param ti: :return: """ if ti.raw: return prefix = conf.get('logging', 'task_log_prefix_template') rendered_prefix = "" if prefix: _, self.prefix_jinja_template = parse_template_string(prefix) rendered_prefix = self._render_prefix(ti) formatter = logging.Formatter(rendered_prefix + ":" + self.formatter._fmt) # pylint: disable=W0212 self.setFormatter(formatter) self.setLevel(self.level)
def execute(self, context): print(f'Querying PatentsView API for {self.entity} with parameters in {self.query_file_path}') # read in query json file with open(self.query_file_path, 'r') as f: query = json.load(f) # manually render airflow macro in json file _, template = parse_template_string(json.dumps(query)) templated_query = template.render(**context) # init hook and post to api hook = PatentsViewHook() response = hook.post(self.entity, json.loads(templated_query)) with open(self.response_file_path, 'w') as f: json.dump(response, f) print(f'Saved results to {self.response_file_path}')
def __init__(self, base_log_folder, filename_template, log_id_template, end_of_log_mark, host='localhost:9200'): """ :param base_log_folder: base folder to store logs locally :param log_id_template: log id template :param host: Elasticsearch host name """ super(ElasticsearchTaskHandler, self).__init__( base_log_folder, filename_template) self.closed = False self.log_id_template, self.log_id_jinja_template = \ parse_template_string(log_id_template) self.client = elasticsearch.Elasticsearch([host]) self.mark_end_on_close = True self.end_of_log_mark = end_of_log_mark
def set_context(self, ti): local_loc = self._init_file(ti) self.handler = logging.FileHandler(local_loc) suffix = conf.get('core', 'task_log_suffix_template') encode = conf.get('core', 'base64_encoding') rendered_suffix = "" if suffix: self.suffix_template, self.suffix_jinja_template = parse_template_string( suffix) rendered_suffix = self._render_suffix(ti, ti.try_number) if encode: rendered_suffix = base64.b64encode( rendered_suffix.encode()).decode() self.handler.setFormatter( logging.Formatter(self.formatter._fmt + ":" + rendered_suffix)) self.handler.setLevel(self.level)
def _render_filename(self, ti: "TaskInstance", try_number: int) -> str: with create_session() as session: dag_run = ti.get_dagrun(session=session) template = dag_run.get_log_template(session=session).filename str_tpl, jinja_tpl = parse_template_string(template) if jinja_tpl: if hasattr(ti, "task"): context = ti.get_template_context() else: context = Context(ti=ti, ts=dag_run.logical_date.isoformat()) context["try_number"] = try_number return render_template_to_string(jinja_tpl, context) elif str_tpl: dag = ti.task.dag assert dag is not None # For Mypy. try: data_interval: Tuple[ datetime, datetime] = dag.get_run_data_interval(dag_run) except AttributeError: # ti.task is not always set. data_interval = (dag_run.data_interval_start, dag_run.data_interval_end) if data_interval[0]: data_interval_start = data_interval[0].isoformat() else: data_interval_start = "" if data_interval[1]: data_interval_end = data_interval[1].isoformat() else: data_interval_end = "" return str_tpl.format( dag_id=ti.dag_id, task_id=ti.task_id, run_id=ti.run_id, data_interval_start=data_interval_start, data_interval_end=data_interval_end, execution_date=ti.get_dagrun().logical_date.isoformat(), try_number=try_number, ) else: raise RuntimeError( f"Unable to render log filename for {ti}. This should never happen" )
def set_context(self, ti) -> None: """ Accept the run-time context (i.e. the current task) and configure the formatter accordingly. :param ti: :return: """ if ti.raw or self.formatter is None: return prefix = conf.get('logging', 'task_log_prefix_template') if prefix: _, self.prefix_jinja_template = parse_template_string(prefix) rendered_prefix = self._render_prefix(ti) else: rendered_prefix = "" formatter = logging.Formatter( f"{rendered_prefix}:{self.formatter._fmt}") self.setFormatter(formatter) self.setLevel(self.level)
def set_context(self, ti, *, session: "Session" = NEW_SESSION) -> None: """ Accept the run-time context (i.e. the current task) and configure the formatter accordingly. :param ti: :return: """ if ti.raw or self.formatter is None: return prefix = ti.get_dagrun().get_task_prefix_template(session=session) if prefix: _, self.prefix_jinja_template = parse_template_string(prefix) rendered_prefix = self._render_prefix(ti) else: rendered_prefix = "" formatter = logging.Formatter( f"{rendered_prefix}:{self.formatter._fmt}") self.setFormatter(formatter) self.setLevel(self.level)
def __init__(self, base_log_folder, filename_template): super().__init__() self.handler = None self.base_log_folder = base_log_folder self.dag_dir = os.path.expanduser(settings.DAGS_FOLDER) self.filename_template, self.filename_jinja_template = \ parse_template_string(filename_template) self._cur_date = datetime.today() if not os.path.exists(self._get_log_directory()): try: os.makedirs(self._get_log_directory()) except OSError: # only ignore case where the directory already exist if not os.path.isdir(self._get_log_directory()): raise logging.warning("%s already exists", self._get_log_directory()) self._symlink_latest_log_directory()
def __init__(self, base_log_folder, filename_template): """ :param base_log_folder: Base log folder to place logs. :param filename_template: template filename string """ super(FileProcessorHandler, self).__init__() self.handler = None self.base_log_folder = base_log_folder self.dag_dir = os.path.expanduser(conf.get('core', 'DAGS_FOLDER')) self.filename_template, self.filename_jinja_template = \ parse_template_string(filename_template) self._cur_date = datetime.today() if not os.path.exists(self._get_log_directory()): try: os.makedirs(self._get_log_directory()) except OSError as e: # only ignore case where the directory already exist if e.errno != errno.EEXIST: raise logging.warning("%s already exists", self._get_log_directory()) self._symlink_latest_log_directory()
def __init__(self, base_log_folder, filename_template): """ :param base_log_folder: Base log folder to place logs. :param filename_template: template filename string """ super().__init__() self.handler = None self.base_log_folder = base_log_folder self.dag_dir = os.path.expanduser(settings.DAGS_FOLDER) self.filename_template, self.filename_jinja_template = \ parse_template_string(filename_template) self._cur_date = datetime.today() if not os.path.exists(self._get_log_directory()): try: os.makedirs(self._get_log_directory()) except OSError: # only ignore case where the directory already exist if not os.path.isdir(self._get_log_directory()): raise logging.warning("%s already exists", self._get_log_directory()) self._symlink_latest_log_directory()
def __init__(self, base_log_folder: str, filename_template: str): super().__init__() self.handler = None # type: Optional[logging.FileHandler] self.local_base = base_log_folder self.filename_template, self.filename_jinja_template = parse_template_string(filename_template)