def _serialize(cls, var: Any) -> Any: # Unfortunately there is no support for recursive types in mypy
"""Helper function of depth first search for serialization.
The serialization protocol is:
(1) keeping JSON supported types: primitives, dict, list;
(2) encoding other types as ``{TYPE: 'foo', VAR: 'bar'}``, the deserialization
step decode VAR according to TYPE;
(3) Operator has a special field CLASS to record the original class
name for displaying in UI.
"""
try:
if cls._is_primitive(var):
# enum.IntEnum is an int instance, it causes json dumps error so we use its value.
if isinstance(var, enum.Enum):
return var.value
return var
elif isinstance(var, dict):
return cls._encode(
{str(k): cls._serialize(v) for k, v in var.items()},
type_=DAT.DICT
)
elif isinstance(var, list):
return [cls._serialize(v) for v in var]
elif isinstance(var, DAG):
return SerializedDAG.serialize_dag(var)
elif isinstance(var, BaseOperator):
return SerializedBaseOperator.serialize_operator(var)
elif isinstance(var, cls._datetime_types):
return cls._encode(var.timestamp(), type_=DAT.DATETIME)
elif isinstance(var, datetime.timedelta):
return cls._encode(var.total_seconds(), type_=DAT.TIMEDELTA)
elif isinstance(var, (pendulum.tz.Timezone, pendulum.tz.timezone_info.TimezoneInfo)):
return cls._encode(str(var.name), type_=DAT.TIMEZONE)
elif isinstance(var, relativedelta.relativedelta):
encoded = {k: v for k, v in var.__dict__.items() if not k.startswith("_") and v}
if var.weekday and var.weekday.n:
# Every n'th Friday for example
encoded['weekday'] = [var.weekday.weekday, var.weekday.n]
elif var.weekday:
encoded['weekday'] = [var.weekday.weekday]
return cls._encode(encoded, type_=DAT.RELATIVEDELTA)
elif callable(var):
return str(get_python_source(var))
elif isinstance(var, set):
# FIXME: casts set to list in customized serialization in future.
return cls._encode(
[cls._serialize(v) for v in var], type_=DAT.SET)
elif isinstance(var, tuple):
# FIXME: casts tuple to list in customized serialization in future.
return cls._encode(
[cls._serialize(v) for v in var], type_=DAT.TUPLE)
else:
LOG.debug('Cast type %s to str in serialization.', type(var))
return str(var)
except Exception: # pylint: disable=broad-except
LOG.warning('Failed to stringify.', exc_info=True)
return FAILED
def _serialize(cls, var, visited_dags): # pylint: disable=too-many-return-statements
"""Helper function of depth first search for serialization.
visited_dags stores DAGs that are being serialized or have been serialized,
for:
(1) preventing deadlock loop caused by task.dag, task._dag, and dag.parent_dag;
(2) replacing the fields in (1) with serialized counterparts.
The serialization protocol is:
(1) keeping JSON supported types: primitives, dict, list;
(2) encoding other types as {TYPE: 'foo', VAR: 'bar'}, the deserialization
step decode VAR according to TYPE;
(3) Operator has a special field CLASS to record the original class
name for displaying in UI.
"""
try:
if cls._is_primitive(var):
return var
elif isinstance(var, dict):
return cls._encode(
{
str(k): cls._serialize(v, visited_dags)
for k, v in var.items()
},
type_=DAT.DICT)
elif isinstance(var, list):
return [cls._serialize(v, visited_dags) for v in var]
elif isinstance(var, DAG):
return airflow.dag.serialization.SerializedDAG.serialize_dag(
var, visited_dags)
elif isinstance(var, BaseOperator):
return airflow.dag.serialization.SerializedBaseOperator.serialize_operator(
var, visited_dags)
elif isinstance(var, cls._datetime_types):
return cls._encode(var.isoformat(), type_=DAT.DATETIME)
elif isinstance(var, datetime.timedelta):
return cls._encode(var.total_seconds(), type_=DAT.TIMEDELTA)
elif isinstance(var, pendulum.tz.Timezone):
return cls._encode(str(var.name), type_=DAT.TIMEZONE)
elif callable(var):
return str(get_python_source(var))
elif isinstance(var, set):
# FIXME: casts set to list in customized serilization in future.
return cls._encode(
[cls._serialize(v, visited_dags) for v in var],
type_=DAT.SET)
elif isinstance(var, tuple):
# FIXME: casts tuple to list in customized serilization in future.
return cls._encode(
[cls._serialize(v, visited_dags) for v in var],
type_=DAT.TUPLE)
else:
LOG.debug('Cast type %s to str in serialization.', type(var))
return str(var)
except Exception: # pylint: disable=broad-except
LOG.warning('Failed to stringify.', exc_info=True)
return FAILED
def test_get_python_source_from_partial_func(self):
def a_function(arg_x, arg_y):
""" A function with two args """
pass
partial_function = functools.partial(a_function, arg_x=1)
result = utils.get_python_source(partial_function)
self.assertIn('A function with two args', result)
def test_get_python_source_from_partial_func(self):
def a_function(arg_x, arg_y):
""" A function with two args """
pass
partial_function = functools.partial(a_function, arg_x=1)
result = utils.get_python_source(partial_function)
self.assertIn('A function with two args', result)
def test_get_python_source_from_class(self):
class AMockClass(object):
def __call__(self):
""" A __call__ method """
pass
mocked_class = AMockClass()
result = utils.get_python_source(mocked_class)
self.assertIn('A __call__ method', result)
def test_get_python_source_from_method(self):
class AMockClass(object):
def a_method(self):
""" A method """
pass
mocked_class = AMockClass()
result = utils.get_python_source(mocked_class.a_method)
self.assertIn('A method', result)
def test_get_python_source_from_class(self):
class AMockClass(object):
def __call__(self):
""" A __call__ method """
pass
mocked_class = AMockClass()
result = utils.get_python_source(mocked_class)
self.assertIn('A __call__ method', result)
def test_get_python_source_from_method(self):
class AMockClass(object):
def a_method(self):
""" A method """
pass
mocked_class = AMockClass()
result = utils.get_python_source(mocked_class.a_method)
self.assertIn('A method', result)
def test_get_python_source_from_none(self):
result = utils.get_python_source(None)
self.assertIn('No source code available', result)
lambda x: render(x, lexers.SqlLexer),
'sql':
lambda x: render(x, lexers.SqlLexer),
'doc':
lambda x: render(x, lexers.TextLexer),
'doc_json':
lambda x: render(x, lexers.JsonLexer),
'doc_rst':
lambda x: render(x, lexers.RstLexer),
'doc_yaml':
lambda x: render(x, lexers.YamlLexer),
'doc_md':
wrapped_markdown,
'python_callable':
lambda x: render(
wwwutils.get_python_source(x),
lexers.PythonLexer,
),
}
def data_profiling_required(f):
"""Decorator for views requiring data profiling access"""
@wraps(f)
def decorated_function(*args, **kwargs):
if (current_app.config['LOGIN_DISABLED']):
return f(*args, **kwargs)
else:
flash("This page requires data profiling privileges", "error")
return redirect(url_for('admin.index'))
def test_get_python_source_from_none(self):
result = utils.get_python_source(None)
self.assertIn('No source code available', result)
