def get_many(cls,
execution_date,
key=None,
task_ids=None,
dag_ids=None,
include_prior_dates=False,
limit=100,
session=None):
"""
Retrieve an XCom value, optionally meeting certain criteria
TODO: "pickling" has been deprecated and JSON is preferred.
"pickling" will be removed in Airflow 2.0.
"""
filters = []
if key:
filters.append(cls.key == key)
if task_ids:
filters.append(cls.task_id.in_(as_tuple(task_ids)))
if dag_ids:
filters.append(cls.dag_id.in_(as_tuple(dag_ids)))
if include_prior_dates:
filters.append(cls.execution_date <= execution_date)
else:
filters.append(cls.execution_date == execution_date)
query = (
session.query(cls).filter(and_(*filters))
.order_by(cls.execution_date.desc(), cls.timestamp.desc())
.limit(limit))
results = query.all()
return results
def test_as_tuple(self):
self.assertEqual(helpers.as_tuple("a string is not a container"),
("a string is not a container", ))
self.assertEqual(
helpers.as_tuple(["a", "list", "is", "a", "container"]),
("a", "list", "is", "a", "container"))
def get_many(cls,
execution_date,
key=None,
task_ids=None,
dag_ids=None,
include_prior_dates=False,
limit=100,
session=None):
"""
Retrieve an XCom value, optionally meeting certain criteria
TODO: "pickling" has been deprecated and JSON is preferred.
"pickling" will be removed in Airflow 2.0.
"""
filters = []
if key:
filters.append(cls.key == key)
if task_ids:
filters.append(cls.task_id.in_(as_tuple(task_ids)))
if dag_ids:
filters.append(cls.dag_id.in_(as_tuple(dag_ids)))
if include_prior_dates:
filters.append(cls.execution_date <= execution_date)
else:
filters.append(cls.execution_date == execution_date)
query = (
session.query(cls).filter(and_(*filters))
.order_by(cls.execution_date.desc(), cls.timestamp.desc())
.limit(limit))
results = query.all()
return results
def test_as_tuple(self):
self.assertEquals(
helpers.as_tuple("a string is not a container"),
("a string is not a container",)
)
self.assertEquals(
helpers.as_tuple(["a", "list", "is", "a", "container"]),
("a", "list", "is", "a", "container")
)
def test_as_tuple(self):
assert helpers.as_tuple("a string is not a container") == ("a string is not a container",)
assert helpers.as_tuple(["a", "list", "is", "a", "container"]) == (
"a",
"list",
"is",
"a",
"container",
)
def test_as_tuple_no_iter(self):
test_str = 'test_str'
as_tup = helpers.as_tuple(test_str)
self.assertTupleEqual((test_str, ), as_tup)
def test_as_tuple_iter(self):
test_list = ['test_str']
as_tup = helpers.as_tuple(test_list)
self.assertTupleEqual(tuple(test_list), as_tup)
def test_as_tuple_no_iter(self):
test_str = 'test_str'
as_tup = helpers.as_tuple(test_str)
self.assertTupleEqual((test_str,), as_tup)
def test_as_tuple_iter(self):
test_list = ['test_str']
as_tup = helpers.as_tuple(test_list)
self.assertTupleEqual(tuple(test_list), as_tup)
def get_many_xcom_tags(
min_execution_date=None,
max_execution_date=None,
key=None,
values=None,
task_ids=None,
dag_ids=None,
limit=100,
session=None,
):
"""Retrieve XComs, optionally meeting certain criteria.
XComs are queried based on execution date, key, value, task_ids, and
dag_ids. The value is passed in as a dict and then serialized to a JSON
formatted string to be split up into a list containing the individual
key/value pairs.
Args:
min_execution_date (datetime, optional): The min date to filter on.
max_execution_date (datetime, optional): The max date to filter on.
key (string, optional): The key to filter on.
values (dict, optional): A dictionary of values to filter on.
task_ids (str or iterable of strings (representing task_ids),optional):
The tag_ids to filter on.
dag_ids (str or iterable of strings (representing dag_ids), optional):
The dag_ids to filter on.
limit (int, optional): The max number of results to be returned by XCom
query.
Returns:
A list of XCom objects that matched the filter criteria. If no matches
were found, then None is returned.
"""
filters = []
cleaned_values = []
# split the values into a list of individual key/value pairs
if values:
values = json.dumps(values)[1:-1].strip().split(",")
for item in values:
cleaned_values.append(
# XCom.serialize_value("%" + item.strip() + "%")
("%" + item.strip() + "%").encode("UTF-8"))
if key:
filters.append(XCom.key == key)
if values:
for filter_value in cleaned_values:
filters.append(XCom.value.like(filter_value))
if task_ids:
filters.append(XCom.task_id.in_(as_tuple(task_ids)))
if dag_ids:
filters.append(XCom.dag_id.in_(as_tuple(dag_ids)))
if min_execution_date:
filters.append(XCom.execution_date >= min_execution_date)
if max_execution_date:
filters.append(XCom.execution_date <= max_execution_date)
query = (session.query(XCom).filter(and_(*filters)).order_by(
XCom.execution_date.desc(), XCom.timestamp.desc()).limit(limit))
results = query.all()
return results
def test_as_tuple_no_iter(self):
test_str = 'test_str'
as_tup = helpers.as_tuple(test_str)
assert (test_str, ) == as_tup
def test_as_tuple_iter(self):
test_list = ['test_str']
as_tup = helpers.as_tuple(test_list)
assert tuple(test_list) == as_tup
def test_as_tuple_no_iter(self):
iter_obj = HelpersTest.TestObjNoIter()
as_tup = helpers.as_tuple(iter_obj)
self.assertTupleEqual(([iter_obj], ), as_tup)
