def setUpClass(cls):
import tempfile
from mars import kvstore
options.worker.spill_directory = os.path.join(tempfile.gettempdir(),
'mars_test_spill')
cls._kv_store = kvstore.get(options.kv_store)
def setUpClass(cls):
from mars import kvstore
options.worker.spill_directory = os.path.join(tempfile.gettempdir(),
'mars_test_spill')
cls._kv_store = kvstore.get(options.kv_store)
cls.timeout = int(os.environ.get('CHECK_TIMEOUT', 120))
def testEtcdWatch(self):
with EtcdProcessHelper(port_range_start=51342).run():
kvstore = get('etcd://localhost:51342')
kvstore.write('/node/subnode/v1', 'value1')
kvstore.write('/node/v2', 'value2')
def watcher():
return kvstore.watch('/node/v2', timeout=10)
def writer():
gevent.sleep(1)
kvstore.write('/node/v2', 'value2\'')
g1 = gevent.spawn(writer)
g2 = gevent.spawn(watcher)
gevent.joinall([g1, g2])
self.assertEqual(g2.value.value, 'value2\'')
kvstore.delete('/node/v2')
def watcher():
return kvstore.watch('/node/subnode',
timeout=10,
recursive=True)
def writer():
gevent.sleep(1)
kvstore.write('/node/subnode/v1', 'value1\'')
g1 = gevent.spawn(writer)
g2 = gevent.spawn(watcher)
gevent.joinall([g1, g2])
self.assertEqual(g2.value.children[0].value, 'value1\'')
kvstore.write('/node/subnode/v3', '-1')
def watcher():
results = []
for idx, result in enumerate(
kvstore.eternal_watch('/node/subnode/v3')):
results.append(int(result.value))
if idx == 4:
break
return results
def writer():
gevent.sleep(0.1)
for v in range(5):
kvstore.write('/node/subnode/v3', str(v))
gevent.sleep(0.1)
g1 = gevent.spawn(writer)
g2 = gevent.spawn(watcher)
gevent.joinall([g1, g2])
self.assertEqual(g2.value, list(range(5)))
kvstore.delete('/node', dir=True, recursive=True)
def testLocalWatch(self):
kvstore = get(':inproc:')
kvstore.write('/node/subnode/v1', 'value1')
kvstore.write('/node/v2', 'value2')
def watcher():
return kvstore.watch('/node/v2', timeout=10)
def writer():
gevent.sleep(0.5)
kvstore.write('/node/v2', 'value2\'')
g1 = gevent.spawn(writer)
g2 = gevent.spawn(watcher)
gevent.joinall([g1, g2])
self.assertEqual(g2.value.value, 'value2\'')
kvstore.delete('/node/v2')
def watcher():
return kvstore.watch('/node/subnode', timeout=10, recursive=True)
def writer():
gevent.sleep(0.5)
kvstore.write('/node/subnode/v1', 'value1\'')
g1 = gevent.spawn(writer)
g2 = gevent.spawn(watcher)
gevent.joinall([g1, g2])
self.assertEqual(g2.value.children[0].value, 'value1\'')
kvstore.write('/node/subnode/v3', '-1')
def watcher():
results = []
for idx, result in enumerate(
kvstore.eternal_watch('/node/subnode/v3')):
results.append(int(result.value))
if idx == 4:
break
return results
def writer():
gevent.sleep(0.1)
for v in range(5):
kvstore.write('/node/subnode/v3', str(v))
gevent.sleep(0.1)
g1 = gevent.spawn(writer)
g2 = gevent.spawn(watcher)
gevent.joinall([g1, g2])
self.assertEqual(g2.value, list(range(5)))
kvstore.delete('/node', dir=True, recursive=True)
def setUpClass(cls):
import pyarrow.plasma as plasma
from mars import kvstore
cls._plasma_store = plasma.start_plasma_store(cls.plasma_storage_size)
cls.plasma_socket = options.worker.plasma_socket = cls._plasma_store.__enter__(
)[0]
options.worker.spill_directory = cls.spill_dir
cls._plasma_client = plasma.connect(options.worker.plasma_socket, '',
0)
cls._kv_store = kvstore.get(options.kv_store)
def setUpClass(cls):
import pyarrow.plasma as plasma
from mars import kvstore
from mars.utils import PlasmaProcessHelper
cls._plasma_helper = PlasmaProcessHelper(size=cls.plasma_storage_size,
socket=cls.plasma_socket)
cls._plasma_helper.run()
options.worker.spill_directory = cls.spill_dir
options.worker.plasma_socket = cls.plasma_socket
cls._plasma_client = plasma.connect(options.worker.plasma_socket, '',
0)
cls._kv_store = kvstore.get(options.kv_store)
def setUpClass(cls):
import pyarrow.plasma as plasma
from mars import kvstore
if sys.platform == 'darwin':
options.worker.plasma_dir = '/tmp'
else:
options.worker.plasma_dir = '/dev/shm'
cls._plasma_store = plasma.start_plasma_store(
cls.plasma_storage_size, plasma_directory=options.worker.plasma_dir)
cls.plasma_socket = options.worker.plasma_socket = cls._plasma_store.__enter__()[0]
options.worker.spill_directory = cls.spill_dir
try:
cls._plasma_client = plasma.connect(options.worker.plasma_socket)
except TypeError:
cls._plasma_client = plasma.connect(options.worker.plasma_socket, '', 0)
cls._kv_store = kvstore.get(options.kv_store)
def testLocalPathStore(self):
kvstore = get(':inproc:')
kvstore.write('/node/subnode/v1', 'value1')
kvstore.write('/node/v2', 'value2')
res = kvstore.read('/node', sort=True)
expected = PathResult(key='/node',
dir=True,
children=[
PathResult(key='/node/subnode', dir=True),
PathResult(key='/node/v2', value='value2'),
])
self.assertEqual(repr(res), repr(expected))
res = kvstore.read('/node', recursive=True, sort=True)
expected = PathResult(key='/node',
dir=True,
children=[
PathResult(key='/node/subnode/v1',
value='value1'),
PathResult(key='/node/v2', value='value2'),
])
self.assertEqual(repr(res), repr(expected))
kvstore.write('/node/v3', 'value3')
with self.assertRaises(KeyError):
kvstore.write('/node/v2/invalid_value', value='invalid')
res = kvstore.read('/', recursive=False, sort=True)
expected = PathResult(key='/',
dir=True,
children=[
PathResult(key='/node', dir=True),
])
self.assertEqual(repr(res), repr(expected))
res = kvstore.read('/', recursive=True, sort=True)
expected = PathResult(key='/',
dir=True,
children=[
PathResult(key='/node/subnode/v1',
value='value1'),
PathResult(key='/node/v2', value='value2'),
PathResult(key='/node/v3', value='value3'),
])
self.assertEqual(repr(res), repr(expected))
kvstore.write('/node/subnode2/v4', 'value4')
with self.assertRaises(KeyError):
kvstore.delete('/node/subnode', dir=True)
kvstore.delete('/node/subnode/v1')
res = kvstore.read('/', recursive=True, sort=True)
expected = PathResult(key='/',
dir=True,
children=[
PathResult(key='/node/subnode', dir=True),
PathResult(key='/node/subnode2/v4',
value='value4'),
PathResult(key='/node/v2', value='value2'),
PathResult(key='/node/v3', value='value3'),
])
self.assertEqual(repr(res), repr(expected))
kvstore.delete('/node/subnode2', dir=True, recursive=True)
res = kvstore.read('/', recursive=True, sort=True)
expected = PathResult(key='/',
dir=True,
children=[
PathResult(key='/node/subnode', dir=True),
PathResult(key='/node/v2', value='value2'),
PathResult(key='/node/v3', value='value3')
])
self.assertEqual(repr(res), repr(expected))
def testEtcdPathStore(self):
with EtcdProcessHelper(port_range_start=51342).run():
kvstore = get(u'etcd://localhost:51342')
kvstore.write(u'/node/subnode/v1', u'value1')
kvstore.write(u'/node/v2', u'value2')
res = kvstore.read(u'/node', sort=True)
expected = PathResult(key=u'/node',
dir=True,
children=[
PathResult(key=u'/node/subnode',
dir=True),
PathResult(key=u'/node/v2',
value=u'value2'),
])
self.assertEqual(repr(res), repr(expected))
res = kvstore.read(u'/node', recursive=True, sort=True)
expected = PathResult(key=u'/node',
dir=True,
children=[
PathResult(key=u'/node/subnode/v1',
value=u'value1'),
PathResult(key=u'/node/v2',
value=u'value2'),
])
self.assertEqual(repr(res), repr(expected))
kvstore.write(u'/node/v3', u'value3')
with self.assertRaises(KeyError):
kvstore.write(u'/node/v2/invalid_value', value=u'invalid')
res = kvstore.read('/', recursive=False, sort=True)
expected = PathResult(key='/',
dir=True,
children=[
PathResult(key=u'/node', dir=True),
])
self.assertEqual(repr(res), repr(expected))
res = kvstore.read('/', recursive=True, sort=True)
expected = PathResult(key='/',
dir=True,
children=[
PathResult(key=u'/node/subnode/v1',
value=u'value1'),
PathResult(key=u'/node/v2',
value=u'value2'),
PathResult(key=u'/node/v3',
value=u'value3'),
])
self.assertEqual(repr(res), repr(expected))
kvstore.write(u'/node/subnode2/v4', u'value4')
with self.assertRaises(KeyError):
kvstore.delete(u'/node/subnode', dir=True)
kvstore.delete(u'/node/subnode/v1')
res = kvstore.read('/', recursive=True, sort=True)
expected = PathResult(key='/',
dir=True,
children=[
PathResult(key=u'/node/subnode',
dir=True),
PathResult(key=u'/node/subnode2/v4',
value=u'value4'),
PathResult(key=u'/node/v2',
value=u'value2'),
PathResult(key=u'/node/v3',
value=u'value3'),
])
self.assertEqual(repr(res), repr(expected))
kvstore.delete(u'/node', recursive=True, dir=True)