def test_create_file():
source1 = Stream(stream_name='source1')
source2 = Stream(stream_name='source2')
n1 = source1.zip(source2)
n2 = n1.map(add).scan(mul).map(lambda x: x + 1)
n2.sink(source1.emit)
with tmpfile(extension='png') as fn:
visualize(n1, filename=fn)
assert os.path.exists(fn)
with tmpfile(extension='svg') as fn:
n1.visualize(filename=fn, rankdir="LR")
assert os.path.exists(fn)
with tmpfile(extension='dot') as fn:
n1.visualize(filename=fn, rankdir="LR")
with open(fn) as f:
text = f.read()
for word in [
'rankdir', 'source1', 'source2', 'zip', 'map', 'add',
'shape=box', 'shape=ellipse'
]:
assert word in text
def test_from_file():
with tmpfile() as fn:
with open(fn, 'wt') as f:
f.write('{"x": 1, "y": 2}\n')
f.write('{"x": 2, "y": 2}\n')
f.write('{"x": 3, "y": 2}\n')
f.flush()
source = Stream.from_textfile(fn, poll_interval=0.010,
asynchronous=True, start=False)
L = source.map(json.loads).pluck('x').sink_to_list()
assert L == []
source.start()
yield await_for(lambda: len(L) == 3, timeout=5)
assert L == [1, 2, 3]
f.write('{"x": 4, "y": 2}\n')
f.write('{"x": 5, "y": 2}\n')
f.flush()
start = time()
while L != [1, 2, 3, 4, 5]:
yield gen.sleep(0.01)
assert time() < start + 2 # reads within 2s
def test_from_file():
with tmpfile() as fn:
with open(fn, 'wt') as f:
f.write('{"x": 1, "y": 2}\n')
f.write('{"x": 2, "y": 2}\n')
f.write('{"x": 3, "y": 2}\n')
f.flush()
source = Stream.from_textfile(fn, poll_interval=0.010)
L = source.map(json.loads).pluck('x').sink_to_list()
assert L == []
source.start()
assert L == [1, 2, 3]
f.write('{"x": 4, "y": 2}\n')
f.write('{"x": 5, "y": 2}\n')
f.flush()
start = time()
while L != [1, 2, 3, 4, 5]:
yield gen.sleep(0.01)
assert time() < start + 2 # reads within 2s
def test_sink_to_textfile_named():
source = Stream()
with tmpfile() as filename:
_sink = source.map(str).sink_to_textfile(filename)
source.emit(0)
source.emit(1)
_sink._fp.flush()
assert open(filename, "r").read() == "0\n1\n"
def test_sink_to_textfile_fp():
source = Stream()
with tmpfile() as filename, open(filename, "w") as fp:
source.map(str).sink_to_textfile(fp)
source.emit(0)
source.emit(1)
fp.flush()
assert open(filename, "r").read() == "0\n1\n"
def test_sink_to_file():
with tmpfile() as fn:
source = Stream()
with sink_to_file(fn, source) as f:
source.emit('a')
source.emit('b')
with open(fn) as f:
data = f.read()
assert data == 'a\nb\n'
def test_sink_to_textfile_closes():
source = Stream()
with tmpfile() as filename:
sink = source.sink_to_textfile(filename)
fp = sink._fp
_global_sinks.remove(sink)
del sink
with pytest.raises(ValueError,
match=r"I/O operation on closed file\."):
fp.write(".")
def test_from_file_end():
with tmpfile() as fn:
with open(fn, 'wt') as f:
f.write('data1\n')
f.flush()
source = Stream.from_textfile(fn, poll_interval=0.010,
start=False, from_end=True)
out = source.sink_to_list()
source.start()
assert out == []
yield await_for(lambda: source.started, 2, period=0.02)
f.write('data2\n')
f.flush()
yield await_for(lambda: out == ['data2\n'], timeout=5, period=0.1)
def test_filenames():
with tmpfile() as fn:
os.mkdir(fn)
with open(os.path.join(fn, 'a'), 'w') as f:
pass
with open(os.path.join(fn, 'b'), 'w') as f:
pass
source = Stream.filenames(fn, asynchronous=True)
L = source.sink_to_list()
source.start()
while len(L) < 2:
yield gen.sleep(0.01)
assert L == [os.path.join(fn, x) for x in ['a', 'b']]
with open(os.path.join(fn, 'c'), 'w') as f:
pass
while len(L) < 3:
yield gen.sleep(0.01)
assert L == [os.path.join(fn, x) for x in ['a', 'b', 'c']]