def test_stream_unprocessed_ineligible_already_input(self):
dfl = PipeLog(
graph_id=self.graph.hash,
node_key=self.node_source.key,
pipe_key=self.node_source.pipe.key,
runtime_url="test",
)
drl = DataBlockLog(
pipe_log=dfl,
data_block=self.dr1t1,
direction=Direction.OUTPUT,
)
dfl2 = PipeLog(
graph_id=self.graph.hash,
node_key=self.node1.key,
pipe_key=self.node1.pipe.key,
runtime_url="test",
)
drl2 = DataBlockLog(
pipe_log=dfl2,
data_block=self.dr1t1,
direction=Direction.INPUT,
)
self.sess.add_all([dfl, drl, dfl2, drl2])
s = StreamBuilder(nodes=self.node_source)
s = s.filter_unprocessed(self.node1)
assert s.get_query(self.ctx, self.sess).first() is None
def test_stream_unprocessed_eligible_schema(self):
dfl = PipeLog(
graph_id=self.graph.hash,
node_key=self.node_source.key,
pipe_key=self.node_source.pipe.key,
runtime_url="test",
)
drl = DataBlockLog(
pipe_log=dfl,
data_block=self.dr1t1,
direction=Direction.OUTPUT,
)
self.sess.add_all([dfl, drl])
s = StreamBuilder(nodes=self.node_source, schema="TestSchema1")
s = s.filter_unprocessed(self.node1)
assert s.get_query(self.ctx, self.sess).first() == self.dr1t1
s = StreamBuilder(nodes=self.node_source, schema="TestSchema2")
s = s.filter_unprocessed(self.node1)
assert s.get_query(self.ctx, self.sess).first() is None
def test_operators(self):
dfl = PipeLog(
graph_id=self.graph.hash,
node_key=self.node_source.key,
pipe_key=self.node_source.pipe.key,
runtime_url="test",
)
drl = DataBlockLog(
pipe_log=dfl,
data_block=self.dr1t1,
direction=Direction.OUTPUT,
)
drl2 = DataBlockLog(
pipe_log=dfl,
data_block=self.dr2t1,
direction=Direction.OUTPUT,
)
self.sess.add_all([dfl, drl, drl2])
self._cnt = 0
@operator
def count(stream: DataBlockStream) -> DataBlockStream:
for db in stream:
self._cnt += 1
yield db
sb = StreamBuilder(nodes=self.node_source)
expected_cnt = sb.get_query(self.ctx, self.sess).count()
assert expected_cnt == 2
list(count(sb).as_managed_stream(self.ctx, self.sess))
assert self._cnt == expected_cnt
# Test composed operators
self._cnt = 0
list(count(latest(sb)).as_managed_stream(self.ctx, self.sess))
assert self._cnt == 1
# Test kwargs
self._cnt = 0
list(
count(filter(sb, function=lambda db: False)).as_managed_stream(
self.ctx, self.sess))
assert self._cnt == 0
def test_stream_unprocessed_pristine(self):
s = StreamBuilder(nodes=self.node_source)
s = s.filter_unprocessed(self.node1)
assert s.get_query(self.ctx, self.sess).first() is None