def testWrite(passIx):
try:
testKeyspaces = [
SharedState.Keyspace("TakeHighestIdKeyType",
NativeJson.Json("TestSpace%s" % ix), 3) \
for ix in range(3)
]
harness = self.getHarness(inMemory=True,
cachePathOverride=tempDir)
v1 = harness.newView()
for space in testKeyspaces:
rng = SharedState.KeyRange(space, 0, None, None, True,
False)
v1.subscribe(rng)
for ix in range(9):
keyspaceIx = (ix + passIx) % len(testKeyspaces)
space = testKeyspaces[
keyspaceIx] #random.choice(testKeyspaces)
with SharedState.Transaction(v1):
key = SharedState.Key(
space,
(NativeJson.Json("key %s" % ix),
NativeJson.Json(''), NativeJson.Json('')))
value = JsonNative.Json(
'<value is %s in space %s on pass %s: %s>' %
(ix, keyspaceIx, passIx, uuid.uuid4().hex))
v1[key] = value
testValues[key] = value
v1.flush()
time.sleep(0.01)
harness.teardown()
harness = self.getHarness(inMemory=True,
cachePathOverride=tempDir)
v1 = harness.newView()
for space in testKeyspaces:
rng = SharedState.KeyRange(space, 0, None, None, True,
False)
v1.subscribe(rng)
for key, value in testValues.iteritems():
self.assertSharedStateKeyResolvesToValue(
key, v1, testValues[key])
except:
logging.warn("Exception: %s", traceback.format_exc())
raise
finally:
time.sleep(0.01)
harness.teardown()
def test_two_keyspaces(self):
try:
toWrite = Queue.Queue()
space1 = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace1"), 1)
space2 = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace2"), 1)
def callback1(items):
for key in items.iterkeys():
self.assertEqual(key.keyspace, NativeJson.Json("TestSpace1"))
toWrite.task_done()
def callback2(items):
for key in items.iterkeys():
self.assertEqual(key.keyspace, NativeJson.Json("TestSpace2"))
toWrite.task_done()
def onConnected(_):
simpleView.subscribeToKeyspace(space1, 0, callback1)
simpleView.subscribeToKeyspace(space2, 0, callback2)
simpleView = AsyncView.AsyncView(self.harness.viewFactory,
onConnected)
simpleView.startService()
v1 = self.harness.newView()
v1.subscribe(SharedState.KeyRange(space1, 0, None, None, True, False),
True)
v1.subscribe(SharedState.KeyRange(space2, 0, None, None, True, False),
True)
for x in range(100):
space = space1
if random.random() < .5:
space = space2
toWrite.put((str(x), space))
with SharedState.Transaction(v1):
v1[SharedState.Key(space, (NativeJson.Json(str(x)),))] = space.name
toWrite.join()
finally:
if simpleView:
simpleView.stopService()
self.harness.teardown()
def test_basic_listener(self):
v1 = self.harness.newView()
v2 = self.harness.newView()
while not v1.waitConnectTimeout(.1):
pass
while not v2.waitConnectTimeout(.1):
pass
l = SharedState.Listener(v1)
v1.subscribe(
SharedState.KeyRange(_testKeyspace, 0, None, None, True, True),
True)
v2.subscribe(
SharedState.KeyRange(_testKeyspace, 0, None, None, True, True),
True)
self.assertIsInstance(l.get()[0][1],
SharedState.SharedStateNative.KeyRange)
self.assertTrue(len(l.get(.1)) == 0)
self.assertTrue(len(l.getNonblock()) == 0)
testKey = SharedState.Key(_testKeyspace, (NativeJson.Json('test'), ))
with SharedState.Transaction(v2):
v2[testKey] = NativeJson.Json('test2')
v2.flush()
v1.flush()
updates = l.get(1)
self.assertTrue(len(updates) == 1)
self.assertEqual(updates[0][0], "KeyUpdates")
self.assertIsInstance(updates[0][1], list)
self.assertTrue(updates[0][1][0] == testKey)
with SharedState.Transaction(v2):
v2[testKey] = NativeJson.Json('newer test')
v2.flush()
v1.flush()
updates = l.get(1)
self.assertTrue(len(updates) == 1)
self.assertEqual(updates[0][0], "KeyUpdates")
self.assertIsInstance(updates[0][1], list)
self.assertTrue(updates[0][1][0] == testKey)
def setUp(self):
self.harness = SharedStateTestHarness.SharedStateTestHarness(True)
self.manager = SharedStateService.KeyspaceManager(10001,1, pingInterval=1)
self.introKeyspace = SharedState.getClientInfoKeyspace()
self.introRange = SharedState.KeyRange(self.introKeyspace, 1, None, None, True, False)
self.views = []
self.maxDiff = None
def test_flush(self):
harness = self.getHarness(inMemory=True)
try:
v1 = harness.newView()
space = SharedState.Keyspace("ComparisonKeyType",
NativeJson.Json("TestSpace"), 1)
rng = SharedState.KeyRange(space, 0, None, None, True, False)
v1.subscribe(rng)
keysToWrite = [str(x) for x in range(40)]
value = "value" * 100 * 1024
with SharedState.Transaction(v1):
self.assertRaises(UserWarning, v1.flush, True)
with SharedState.Transaction(v1):
for keyName in keysToWrite:
key = SharedState.Key(space, (NativeJson.Json(keyName), ))
v1[key] = NativeJson.Json(value)
v1.flush()
finally:
time.sleep(0.01)
harness.teardown()
def subscribeToKeyspace(view, keyspaceNames, keyspaceType):
keyspaces = [SharedState.Keyspace(keyspaceType, json(keyspaceName), 1)
for keyspaceName in keyspaceNames]
keyranges = [SharedState.KeyRange(keyspace, 0, None, None, True, True)
for keyspace in keyspaces]
view.subscribe(keyranges)
return keyspaces, keyranges
def createLogFileDir(self):
self.tempdir = tempfile.mkdtemp()
self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType",
NativeJson.Json("test-space"), 1)
self.keyrange = SharedState.KeyRange(self.keyspace, 0, None, None,
True, False)
self.logFileDir = StorageNative.LogFileDirectory(
self.tempdir, self.keyspace, self.keyrange)
self.baseDir = os.path.split(self.logFileDir.getCurrentLogPath())[0]
def setUp(self):
self.sharedState = SharedStateTestHarness(inMemory = True)
self.keyspacePath = (0, 1)
self.keyspaceName = Node.keyspacePathToKeyspaceName(self.keyspacePath)
self.keyName = 'key'
self.keyValue = 'value'
self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType", self.keyspaceName, 1)
self.keyRange = SharedState.KeyRange(self.keyspace, 0, None, None, True, False)
def addKeyspaceListener(self,
keyspaceName,
keyspaceListener,
keyPrefix,
blockUntilLoaded=False,
assertAlreadyLoaded=False):
"""Add a keyspace listener to the synchronizer."""
assert isinstance(keyPrefix, NativeJson.Json)
assert keyPrefix.isArray()
assert isinstance(keyspaceName, NativeJson.Json)
lookupKey = (keyspaceName, keyPrefix)
if keyspaceName not in self.keyspaceNameToPrefixes_:
self.keyspaceNameToPrefixes_[keyspaceName] = set()
for otherPrefix in self.keyspaceNameToPrefixes_[keyspaceName]:
assert not (
otherPrefix.arrayStartswith(keyPrefix)
and otherPrefix != keyPrefix
), ("Can't subscribe to %s:%s since we're already subscribed to %s:%s"
% (keyspaceName, keyPrefix, keyspaceName, otherPrefix))
self.keyspaceNameToPrefixes_[keyspaceName].add(keyPrefix)
if lookupKey not in self.keyspaceListenersByKeyspaceAndPrefix_:
if assertAlreadyLoaded:
assert False, "Keyspace %s was not already loaded." % keyspaceName
self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey] = set()
self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey].add(
keyspaceListener)
self.valuesByKeyspaceAndPrefix_[lookupKey] = {}
logging.info("shared state subscribing to %s:%s", keyspaceName,
keyPrefix)
keyspace = SharedState.Keyspace("TakeHighestIdKeyType",
keyspaceName, 1)
self.view_.subscribe(
SharedState.KeyRange(keyspace, 0, keyPrefix,
keyPrefix + NativeJson.Json([None]), True,
False), blockUntilLoaded)
if blockUntilLoaded:
self.update()
self.ensureMarkedLoaded(lookupKey)
else:
self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey].add(
keyspaceListener)
keyspaceListener.keysLoaded(
self.valuesByKeyspaceAndPrefix_[lookupKey], True)
def test_keyspace_cache(self):
numKeys = 1024 * 256
before = TCMalloc.getBytesUsed()
view = createViewWithNoChannel()
keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1)
keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True)
cache = SharedStateNative.KeyspaceCache(keyrange, None)
for event in producePartialEvents(view, [keyspace], 'test', numKeys, 1, 8):
cache.addEvent(event)
cache.newMinimumId(numKeys)
view = None
gc.collect()
bytesUsed = TCMalloc.getBytesUsed() - before
self.assertTrue(bytesUsed < 1024 * 16)
def test_listener(self):
harness = self.getHarness(inMemory=True)
try:
v1 = harness.newView()
v2 = harness.newView()
l = SharedState.Listener(v2)
space = SharedState.Keyspace("ComparisonKeyType",
NativeJson.Json("TestSpace"), 1)
rng = SharedState.KeyRange(space, 0, None, None, True, False)
v1.subscribe(rng)
v2.subscribe(rng)
keysToWrite = [JsonNative.Json(str(x)) for x in range(40)]
def writer():
with SharedState.Transaction(v1):
for keyName in keysToWrite:
key = SharedState.Key(space, (keyName, ))
v1[key] = JsonNative.Json("value")
changedKeys = []
def reader():
while len(changedKeys) < len(keysToWrite):
updates = l.get()
for updateType, update in updates:
if updateType == "KeyUpdates":
for key in update:
changedKeys.append(key[0])
writerThread = threading.Thread(target=writer)
readerThread = threading.Thread(target=reader)
writerThread.start()
readerThread.start()
writerThread.join()
readerThread.join()
self.assertEqual(set(keysToWrite), set(changedKeys))
finally:
time.sleep(0.01)
harness.teardown()
def subscribeToKey(self, keyspace, keyname, dimension, callback=None):
'''
Subscribe to a single key in a keyspace and optionally receive notifications
of changes via a callback.
callback signature:
callback(items) : where items is a dictionary from (SharedState.Key -> string)
'''
self.assertSingleThreaded()
keyrange = SharedState.KeyRange(keyspace, dimension, keyname, keyname,
True, True)
return self.subscribeToKeyspace(keyspace,
dimension,
callback,
keyrange=keyrange)
def test_null_value_is_deleted(self):
numEvents = 1024 * 64
valueSize = 128
numKeys = numEvents
view = createViewWithNoChannel()
keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1)
keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True)
cache = SharedStateNative.KeyspaceCache(keyrange, None)
gc.collect()
m0 = TCMalloc.getBytesUsed()
for event in producePartialEvents(view, [keyspace], 'test', numKeys, numKeys, valueSize):
cache.addEvent(event)
cache.addEvent(produceNullifyingEvent(view, event))
cache.newMinimumId(numKeys * 2)
gc.collect()
self.assertLess(TCMalloc.getBytesUsed() - m0, 1024 * 4)
def test(threadIx):
for subPassIx in range(subPasses):
logging.info("Thread %s starting pass %s", threadIx, subPassIx)
testKeyspace = SharedState.Keyspace(
"TakeHighestIdKeyType", NativeJson.Json("TestSpace"), 1)
view = viewFactory.createView()
rng = SharedState.KeyRange(testKeyspace, 0, None, None, True,
False)
view.subscribe(rng)
for ix in range(keysToWrite):
with SharedState.Transaction(view):
ix = random.randint(0, keyspaceSize)
key = SharedState.Key(
testKeyspace, (NativeJson.Json("key %s" % ix), ))
value = uuid.uuid4().hex
view[key] = NativeJson.Json(value)
worked[threadIx] = True
def test_file_management(self):
tempDir = tempfile.mkdtemp()
curOpenFiles = len(os.listdir('/proc/%s/fd' % os.getpid()))
OPEN_FILE_LIMIT = 200
if curOpenFiles >= OPEN_FILE_LIMIT:
os.system("ls -alh /proc/%s/fd" % os.getpid())
self.assertTrue(curOpenFiles < OPEN_FILE_LIMIT,
"Too many open files: %s" % curOpenFiles)
soft, hard = resource.getrlimit(resource.RLIMIT_OFILE)
harness = self.getHarness(inMemory=True,
cachePathOverride=tempDir,
maxOpenFiles=15)
try:
v1 = harness.newView()
resource.setrlimit(resource.RLIMIT_OFILE,
(curOpenFiles + 30, hard))
for ix in range(128):
space = SharedState.Keyspace(
"TakeHighestIdKeyType",
NativeJson.Json("TestSpace%s" % ix), 1)
rng = SharedState.KeyRange(space, 0, None, None, True, False)
v1.subscribe(rng)
key = SharedState.Key(space, (NativeJson.Json('key%s' % ix), ))
with SharedState.Transaction(v1):
v1[key] = NativeJson.Json('value %s' % ix)
finally:
time.sleep(0.01)
harness.teardown()
resource.setrlimit(resource.RLIMIT_OFILE, (soft, hard))
try:
shutil.rmtree(tempDir)
except:
pass
def test_require_subscription_manager(self):
tempDir = tempfile.mkdtemp()
try:
testKeyspace = SharedState.Keyspace("TakeHighestIdKeyType",
NativeJson.Json("TestSpace"),
1)
harness = self.getHarness(inMemory=True, cachePathOverride=tempDir)
v1 = harness.newView()
v1.subscribe(
SharedState.KeyRange(testKeyspace, 0, None, None, True, False))
for ix in range(1000):
with SharedState.Transaction(v1):
key = SharedState.Key(testKeyspace,
(NativeJson.Json("key"), ))
v1[key] = NativeJson.Json(str(ix))
time.sleep(2)
v2 = harness.newView()
v2.setMustSubscribe(False)
with SharedState.Transaction(v2):
key = SharedState.Key(testKeyspace, (NativeJson.Json("key"), ))
value = 'value is' + uuid.uuid4().hex
v2[key] = NativeJson.Json(value)
time.sleep(2)
def toTry():
with SharedState.Transaction(v2):
key = SharedState.Key(testKeyspace,
(NativeJson.Json("key"), ))
value = 'value is' + uuid.uuid4().hex
v2[key] = NativeJson.Json(value)
self.assertRaises(UserWarning, toTry)
finally:
time.sleep(0.01)
harness.teardown()
shutil.rmtree(tempDir)
def subscribeToTestKeyspace(self, view, block=True):
view.subscribe(
SharedState.KeyRange(self.testKeyspace, 0, None, None, True, False),
block
)
def subscribeToKeyspace(self, view, spacename):
keyspace = SharedState.Keyspace("TakeHighestIdKeyType", spacename, 1)
view.subscribe(
SharedState.KeyRange(keyspace, 0, None, None, True, False), True)
def createAllKeysRange(keyspace, dimension):
return SharedState.KeyRange(keyspace, dimension, None, None, True, False)
def setUp(self):
print
self.keyspace = SharedState.Keyspace("TakeHighestIdType",
json('test_keyspace'), 1)
self.keyrange = SharedState.KeyRange(self.keyspace, 0, None, None,
True, True)
