def _serialize(cls, var: Any) -> Any: # Unfortunately there is no support for recursive types in mypy """Helper function of depth first search for serialization. The serialization protocol is: (1) keeping JSON supported types: primitives, dict, list; (2) encoding other types as ``{TYPE: 'foo', VAR: 'bar'}``, the deserialization step decode VAR according to TYPE; (3) Operator has a special field CLASS to record the original class name for displaying in UI. """ try: if cls._is_primitive(var): # enum.IntEnum is an int instance, it causes json dumps error so we use its value. if isinstance(var, enum.Enum): return var.value return var elif isinstance(var, dict): return cls._encode( {str(k): cls._serialize(v) for k, v in var.items()}, type_=DAT.DICT ) elif isinstance(var, list): return [cls._serialize(v) for v in var] elif isinstance(var, DAG): return SerializedDAG.serialize_dag(var) elif isinstance(var, BaseOperator): return SerializedBaseOperator.serialize_operator(var) elif isinstance(var, cls._datetime_types): return cls._encode(var.timestamp(), type_=DAT.DATETIME) elif isinstance(var, datetime.timedelta): return cls._encode(var.total_seconds(), type_=DAT.TIMEDELTA) elif isinstance(var, (pendulum.tz.Timezone, pendulum.tz.timezone_info.TimezoneInfo)): return cls._encode(str(var.name), type_=DAT.TIMEZONE) elif isinstance(var, relativedelta.relativedelta): encoded = {k: v for k, v in var.__dict__.items() if not k.startswith("_") and v} if var.weekday and var.weekday.n: # Every n'th Friday for example encoded['weekday'] = [var.weekday.weekday, var.weekday.n] elif var.weekday: encoded['weekday'] = [var.weekday.weekday] return cls._encode(encoded, type_=DAT.RELATIVEDELTA) elif callable(var): return str(get_python_source(var)) elif isinstance(var, set): # FIXME: casts set to list in customized serialization in future. return cls._encode( [cls._serialize(v) for v in var], type_=DAT.SET) elif isinstance(var, tuple): # FIXME: casts tuple to list in customized serialization in future. return cls._encode( [cls._serialize(v) for v in var], type_=DAT.TUPLE) else: LOG.debug('Cast type %s to str in serialization.', type(var)) return str(var) except Exception: # pylint: disable=broad-except LOG.warning('Failed to stringify.', exc_info=True) return FAILED
def _serialize(cls, var, visited_dags): # pylint: disable=too-many-return-statements """Helper function of depth first search for serialization. visited_dags stores DAGs that are being serialized or have been serialized, for: (1) preventing deadlock loop caused by task.dag, task._dag, and dag.parent_dag; (2) replacing the fields in (1) with serialized counterparts. The serialization protocol is: (1) keeping JSON supported types: primitives, dict, list; (2) encoding other types as {TYPE: 'foo', VAR: 'bar'}, the deserialization step decode VAR according to TYPE; (3) Operator has a special field CLASS to record the original class name for displaying in UI. """ try: if cls._is_primitive(var): return var elif isinstance(var, dict): return cls._encode( { str(k): cls._serialize(v, visited_dags) for k, v in var.items() }, type_=DAT.DICT) elif isinstance(var, list): return [cls._serialize(v, visited_dags) for v in var] elif isinstance(var, DAG): return airflow.dag.serialization.SerializedDAG.serialize_dag( var, visited_dags) elif isinstance(var, BaseOperator): return airflow.dag.serialization.SerializedBaseOperator.serialize_operator( var, visited_dags) elif isinstance(var, cls._datetime_types): return cls._encode(var.isoformat(), type_=DAT.DATETIME) elif isinstance(var, datetime.timedelta): return cls._encode(var.total_seconds(), type_=DAT.TIMEDELTA) elif isinstance(var, pendulum.tz.Timezone): return cls._encode(str(var.name), type_=DAT.TIMEZONE) elif callable(var): return str(get_python_source(var)) elif isinstance(var, set): # FIXME: casts set to list in customized serilization in future. return cls._encode( [cls._serialize(v, visited_dags) for v in var], type_=DAT.SET) elif isinstance(var, tuple): # FIXME: casts tuple to list in customized serilization in future. return cls._encode( [cls._serialize(v, visited_dags) for v in var], type_=DAT.TUPLE) else: LOG.debug('Cast type %s to str in serialization.', type(var)) return str(var) except Exception: # pylint: disable=broad-except LOG.warning('Failed to stringify.', exc_info=True) return FAILED
def test_get_python_source_from_partial_func(self): def a_function(arg_x, arg_y): """ A function with two args """ pass partial_function = functools.partial(a_function, arg_x=1) result = utils.get_python_source(partial_function) self.assertIn('A function with two args', result)
def test_get_python_source_from_class(self): class AMockClass(object): def __call__(self): """ A __call__ method """ pass mocked_class = AMockClass() result = utils.get_python_source(mocked_class) self.assertIn('A __call__ method', result)
def test_get_python_source_from_method(self): class AMockClass(object): def a_method(self): """ A method """ pass mocked_class = AMockClass() result = utils.get_python_source(mocked_class.a_method) self.assertIn('A method', result)
def test_get_python_source_from_none(self): result = utils.get_python_source(None) self.assertIn('No source code available', result)
lambda x: render(x, lexers.SqlLexer), 'sql': lambda x: render(x, lexers.SqlLexer), 'doc': lambda x: render(x, lexers.TextLexer), 'doc_json': lambda x: render(x, lexers.JsonLexer), 'doc_rst': lambda x: render(x, lexers.RstLexer), 'doc_yaml': lambda x: render(x, lexers.YamlLexer), 'doc_md': wrapped_markdown, 'python_callable': lambda x: render( wwwutils.get_python_source(x), lexers.PythonLexer, ), } def data_profiling_required(f): """Decorator for views requiring data profiling access""" @wraps(f) def decorated_function(*args, **kwargs): if (current_app.config['LOGIN_DISABLED']): return f(*args, **kwargs) else: flash("This page requires data profiling privileges", "error") return redirect(url_for('admin.index'))