def test_publish_bag(s, a, b):
db = pytest.importorskip('dask.bag')
c = Client((s.ip, s.port), start=False)
yield c._start()
f = Client((s.ip, s.port), start=False)
yield f._start()
bag = db.from_sequence([0, 1, 2])
bagp = c.persist(bag)
assert len(futures_of(bagp)) == 3
keys = {f.key for f in futures_of(bagp)}
assert keys == set(bag.dask)
yield c._publish_dataset(data=bagp)
# check that serialization didn't affect original bag's dask
assert len(futures_of(bagp)) == 3
result = yield f._get_dataset('data')
assert set(result.dask.keys()) == set(bagp.dask.keys())
assert {f.key
for f in result.dask.values()
} == {f.key
for f in bagp.dask.values()}
out = yield f.compute(result)._result()
assert out == [0, 1, 2]
yield c._shutdown()
yield f._shutdown()
def test_publish_bag(s, a, b):
db = pytest.importorskip('dask.bag')
c = Client((s.ip, s.port), start=False)
yield c._start()
f = Client((s.ip, s.port), start=False)
yield f._start()
bag = db.from_sequence([0, 1, 2])
bagp = c.persist(bag)
assert len(futures_of(bagp)) == 3
keys = {f.key for f in futures_of(bagp)}
assert keys == set(bag.dask)
yield c._publish_dataset(data=bagp)
# check that serialization didn't affect original bag's dask
assert len(futures_of(bagp)) == 3
result = yield f._get_dataset('data')
assert set(result.dask.keys()) == set(bagp.dask.keys())
assert {f.key for f in result.dask.values()} == {f.key for f in bagp.dask.values()}
out = yield f.compute(result)._result()
assert out == [0, 1, 2]
yield c._shutdown()
yield f._shutdown()
def test_publish_simple(s, a, b):
c = Client((s.ip, s.port), start=False)
yield c._start()
f = Client((s.ip, s.port), start=False)
yield f._start()
data = yield c._scatter(range(3))
out = yield c._publish_dataset(data=data)
assert 'data' in s.extensions['publish'].datasets
with pytest.raises(KeyError) as exc_info:
out = yield c._publish_dataset(data=data)
assert "exists" in str(exc_info.value)
assert "data" in str(exc_info.value)
result = yield c.scheduler.publish_list()
assert result == ['data']
result = yield f.scheduler.publish_list()
assert result == ['data']
yield c._shutdown()
yield f._shutdown()
def test_publish_simple(s, a, b):
c = Client((s.ip, s.port), start=False)
yield c._start()
f = Client((s.ip, s.port), start=False)
yield f._start()
data = yield c._scatter(range(3))
out = yield c._publish_dataset(data=data)
assert 'data' in s.extensions['publish'].datasets
with pytest.raises(KeyError) as exc_info:
out = yield c._publish_dataset(data=data)
assert "exists" in str(exc_info.value)
assert "data" in str(exc_info.value)
result = yield c.scheduler.publish_list()
assert result == ['data']
result = yield f.scheduler.publish_list()
assert result == ['data']
yield c._shutdown()
yield f._shutdown()
def test_publish_roundtrip(s, a, b):
c = Client((s.ip, s.port), start=False)
yield c._start()
f = Client((s.ip, s.port), start=False)
yield f._start()
data = yield c._scatter([0, 1, 2])
yield c._publish_dataset(data=data)
assert 'published-data' in s.who_wants[data[0].key]
result = yield f._get_dataset(name='data')
assert len(result) == len(data)
out = yield f._gather(result)
assert out == [0, 1, 2]
with pytest.raises(KeyError) as exc_info:
result = yield f._get_dataset(name='nonexistent')
assert "not found" in str(exc_info.value)
assert "nonexistent" in str(exc_info.value)
yield c._shutdown()
yield f._shutdown()
def test_publish_roundtrip(s, a, b):
c = Client((s.ip, s.port), start=False)
yield c._start()
f = Client((s.ip, s.port), start=False)
yield f._start()
data = yield c._scatter([0, 1, 2])
yield c._publish_dataset(data=data)
assert 'published-data' in s.who_wants[data[0].key]
result = yield f._get_dataset(name='data')
assert len(result) == len(data)
out = yield f._gather(result)
assert out == [0, 1, 2]
with pytest.raises(KeyError) as exc_info:
result = yield f._get_dataset(name='nonexistent')
assert "not found" in str(exc_info.value)
assert "nonexistent" in str(exc_info.value)
yield c._shutdown()
yield f._shutdown()
