def test_events_in_subscription(self):
asynView = None
try:
v1 = self.harness.newView()
self.subscribeToTestKeyspace(v1)
for ix in range(100):
key = SharedState.Key(self.testKeyspace, (NativeJson.Json('test%s' % ix),))
with SharedState.Transaction(v1):
v1[key] = NativeJson.Json('value1')
done = threading.Event()
numKeyEvents = [0]
def onKeyEvents(eventDict):
numKeyEvents[0] += len(eventDict)
def onSubscriptionLoaded(_):
done.set()
def onConnected(_):
deferred = asynView.subscribeToKeyspace(self.testKeyspace, 0, onKeyEvents)
deferred.addCallbacks(onSubscriptionLoaded, lambda error: None)
asynView = AsyncView.AsyncView(self.harness.viewFactory,
onConnectCallback=onConnected)
asynView.startService()
while not done.wait(.1):
pass
self.assertEqual(numKeyEvents[0], 100)
finally:
asynView.stopService()
def setUp(self):
self.harness = SharedStateTestHarness.SharedStateTestHarness(True)
self.manager = SharedStateService.KeyspaceManager(10001,1, pingInterval=1)
self.introKeyspace = SharedState.getClientInfoKeyspace()
self.introRange = SharedState.KeyRange(self.introKeyspace, 1, None, None, True, False)
self.views = []
self.maxDiff = None
def runInTransaction(self, keyPathElement, toRun, waitForKeyToLoad = True):
"""
Runs a specified function in a shared state transaction.
toRun is a function that takes the following argumenst:
self - a self reference
key - the shared state key used by the test
view - the a shared state view subscribed to the key range
that contains key.
waitForKeyToLoad - wait for SharedState to actually load the
key
"""
view = self.sharedState.newView()
view.subscribe(self.keyRange)
passes = 0
while passes < 100:
with SharedState.Transaction(view):
key = SharedState.Key(self.keyspace, (Node.keyPathToKeyName((keyPathElement,)),))
if not waitForKeyToLoad or view[key] is not None:
toRun(self, key, view)
return
passes += 1
time.sleep(.1)
assert False, "Test timed out"
def test_flush(self):
harness = self.getHarness(inMemory=True)
try:
v1 = harness.newView()
space = SharedState.Keyspace("ComparisonKeyType",
NativeJson.Json("TestSpace"), 1)
rng = SharedState.KeyRange(space, 0, None, None, True, False)
v1.subscribe(rng)
keysToWrite = [str(x) for x in range(40)]
value = "value" * 100 * 1024
with SharedState.Transaction(v1):
self.assertRaises(UserWarning, v1.flush, True)
with SharedState.Transaction(v1):
for keyName in keysToWrite:
key = SharedState.Key(space, (NativeJson.Json(keyName), ))
v1[key] = NativeJson.Json(value)
v1.flush()
finally:
time.sleep(0.01)
harness.teardown()
def createLogFileDir(self):
self.tempdir = tempfile.mkdtemp()
self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType",
NativeJson.Json("test-space"), 1)
self.keyrange = SharedState.KeyRange(self.keyspace, 0, None, None,
True, False)
self.logFileDir = StorageNative.LogFileDirectory(
self.tempdir, self.keyspace, self.keyrange)
self.baseDir = os.path.split(self.logFileDir.getCurrentLogPath())[0]
def subscribeToKeyspace(view, keyspaceNames, keyspaceType):
keyspaces = [SharedState.Keyspace(keyspaceType, json(keyspaceName), 1)
for keyspaceName in keyspaceNames]
keyranges = [SharedState.KeyRange(keyspace, 0, None, None, True, True)
for keyspace in keyspaces]
view.subscribe(keyranges)
return keyspaces, keyranges
def setUp(self):
self.sharedState = SharedStateTestHarness(inMemory = True)
self.keyspacePath = (0, 1)
self.keyspaceName = Node.keyspacePathToKeyspaceName(self.keyspacePath)
self.keyName = 'key'
self.keyValue = 'value'
self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType", self.keyspaceName, 1)
self.keyRange = SharedState.KeyRange(self.keyspace, 0, None, None, True, False)
def keyspaceIterItems(self, keyspace):
self.assertSingleThreaded()
startKey = tuple(
[NativeJson.lowestValue() for x in range(keyspace.dimension)])
with SharedState.Transaction(self._view):
k = self._view.nextKey(SharedState.Key(keyspace, startKey))
while k is not None and str(k.keyspace) == str(keyspace.name):
if self._view[k] is not None:
yield k, self._view[k].value()
k = self._view.nextKey(k)
def addKeyspaceListener(self,
keyspaceName,
keyspaceListener,
keyPrefix,
blockUntilLoaded=False,
assertAlreadyLoaded=False):
"""Add a keyspace listener to the synchronizer."""
assert isinstance(keyPrefix, NativeJson.Json)
assert keyPrefix.isArray()
assert isinstance(keyspaceName, NativeJson.Json)
lookupKey = (keyspaceName, keyPrefix)
if keyspaceName not in self.keyspaceNameToPrefixes_:
self.keyspaceNameToPrefixes_[keyspaceName] = set()
for otherPrefix in self.keyspaceNameToPrefixes_[keyspaceName]:
assert not (
otherPrefix.arrayStartswith(keyPrefix)
and otherPrefix != keyPrefix
), ("Can't subscribe to %s:%s since we're already subscribed to %s:%s"
% (keyspaceName, keyPrefix, keyspaceName, otherPrefix))
self.keyspaceNameToPrefixes_[keyspaceName].add(keyPrefix)
if lookupKey not in self.keyspaceListenersByKeyspaceAndPrefix_:
if assertAlreadyLoaded:
assert False, "Keyspace %s was not already loaded." % keyspaceName
self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey] = set()
self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey].add(
keyspaceListener)
self.valuesByKeyspaceAndPrefix_[lookupKey] = {}
logging.info("shared state subscribing to %s:%s", keyspaceName,
keyPrefix)
keyspace = SharedState.Keyspace("TakeHighestIdKeyType",
keyspaceName, 1)
self.view_.subscribe(
SharedState.KeyRange(keyspace, 0, keyPrefix,
keyPrefix + NativeJson.Json([None]), True,
False), blockUntilLoaded)
if blockUntilLoaded:
self.update()
self.ensureMarkedLoaded(lookupKey)
else:
self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey].add(
keyspaceListener)
keyspaceListener.keysLoaded(
self.valuesByKeyspaceAndPrefix_[lookupKey], True)
def testWrite(passIx):
try:
testKeyspaces = [
SharedState.Keyspace("TakeHighestIdKeyType",
NativeJson.Json("TestSpace%s" % ix), 3) \
for ix in range(3)
]
harness = self.getHarness(inMemory=True,
cachePathOverride=tempDir)
v1 = harness.newView()
for space in testKeyspaces:
rng = SharedState.KeyRange(space, 0, None, None, True,
False)
v1.subscribe(rng)
for ix in range(9):
keyspaceIx = (ix + passIx) % len(testKeyspaces)
space = testKeyspaces[
keyspaceIx] #random.choice(testKeyspaces)
with SharedState.Transaction(v1):
key = SharedState.Key(
space,
(NativeJson.Json("key %s" % ix),
NativeJson.Json(''), NativeJson.Json('')))
value = JsonNative.Json(
'<value is %s in space %s on pass %s: %s>' %
(ix, keyspaceIx, passIx, uuid.uuid4().hex))
v1[key] = value
testValues[key] = value
v1.flush()
time.sleep(0.01)
harness.teardown()
harness = self.getHarness(inMemory=True,
cachePathOverride=tempDir)
v1 = harness.newView()
for space in testKeyspaces:
rng = SharedState.KeyRange(space, 0, None, None, True,
False)
v1.subscribe(rng)
for key, value in testValues.iteritems():
self.assertSharedStateKeyResolvesToValue(
key, v1, testValues[key])
except:
logging.warn("Exception: %s", traceback.format_exc())
raise
finally:
time.sleep(0.01)
harness.teardown()
def test_async_view(self):
try:
simpleView = None
toWrite = {
NativeJson.Json(str(ix)): NativeJson.Json('value-%s' % ix)
for ix in range(100)
}
written = {}
v1 = self.harness.newView()
self.subscribeToTestKeyspace(v1)
for key, value in toWrite.iteritems():
with SharedState.Transaction(v1):
key = SharedState.Key(self.testKeyspace, (key,))
v1[key] = value
written[key[0]] = value
v1.flush()
received = {}
newKeysWritten = threading.Condition()
def onConnected(_):
def callback(keysDict):
for key, value in keysDict.iteritems():
received[key[0]] = value
with newKeysWritten:
newKeysWritten.notify()
simpleView.subscribeToKeyspace(self.testKeyspace, 0, callback)
simpleView = AsyncView.AsyncView(self.harness.viewFactory,
onConnectCallback=onConnected)
simpleView.startService()
while set(received.values()) != set(written.values()):
with newKeysWritten:
newKeysWritten.wait()
self.assertEqual(set(received.values()), set(written.values()))
finally:
if simpleView:
simpleView.stopService()
self.harness.teardown()
def test_keyspace_cache(self):
numKeys = 1024 * 256
before = TCMalloc.getBytesUsed()
view = createViewWithNoChannel()
keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1)
keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True)
cache = SharedStateNative.KeyspaceCache(keyrange, None)
for event in producePartialEvents(view, [keyspace], 'test', numKeys, 1, 8):
cache.addEvent(event)
cache.newMinimumId(numKeys)
view = None
gc.collect()
bytesUsed = TCMalloc.getBytesUsed() - before
self.assertTrue(bytesUsed < 1024 * 16)
def __init__(self, viewFactory):
Stoppable.Stoppable.__init__(self)
self.viewFactory = viewFactory
self.activeMachineIds = set()
self._outgoingNotifications = Queue.Queue()
self._lock = threading.RLock()
self._lastDisconnectTime = None
self._lastReconnectTime = None
self._registeredIpPortAndMachineId = None
logging.debug("Connecting to shared state: %s", viewFactory)
self.listeners_ = set()
self.clientID = None
self.isConnected = False
self.isConnecting = False
self.disconnectedWhileConnecting = False
self._triggerUpdateAfterDisconnectThreads = []
self.clientIdToIpPortAndMachineIdAsString = {}
self.machineIdToClientId = {}
self.workerStatusKeyspace = SharedState.Keyspace(
"ComparisonKeyType", NativeJson.Json(
(('P', 'CumulusNodeStatus'), )), 1)
self.asyncView = None
self.eventLoopThread = ManagedThread.ManagedThread(
target=self.eventDispatchLoop)
def writeOwnRegistrationInformationIntoView_(self):
if self._registeredIpPortAndMachineId is None:
return
address, ports, machineIdAsString = self._registeredIpPortAndMachineId
logging.info(
"CumulusActiveMachine registering worker %s at %s:%s with id %s",
self.clientID,
address,
ports,
machineIdAsString
)
with self._lock:
self.activeMachineIds.add(machineIdAsString)
key = SharedState.Key(self.workerStatusKeyspace, (NativeJson.Json(str(self.clientID)),))
value = NativeJson.Json( (str(address), [str(x) for x in ports], machineIdAsString) )
doneWithCall = threading.Event()
self.asyncView.pushTransaction(key, value, lambda result: doneWithCall.set())
while not doneWithCall.wait(.1) and not self._stopFlag.is_set():
logging.debug('CumulusActiveMachines waiting on registerSelfAsActive')
def test_listener(self):
harness = self.getHarness(inMemory=True)
try:
v1 = harness.newView()
v2 = harness.newView()
l = SharedState.Listener(v2)
space = SharedState.Keyspace("ComparisonKeyType",
NativeJson.Json("TestSpace"), 1)
rng = SharedState.KeyRange(space, 0, None, None, True, False)
v1.subscribe(rng)
v2.subscribe(rng)
keysToWrite = [JsonNative.Json(str(x)) for x in range(40)]
def writer():
with SharedState.Transaction(v1):
for keyName in keysToWrite:
key = SharedState.Key(space, (keyName, ))
v1[key] = JsonNative.Json("value")
changedKeys = []
def reader():
while len(changedKeys) < len(keysToWrite):
updates = l.get()
for updateType, update in updates:
if updateType == "KeyUpdates":
for key in update:
changedKeys.append(key[0])
writerThread = threading.Thread(target=writer)
readerThread = threading.Thread(target=reader)
writerThread.start()
readerThread.start()
writerThread.join()
readerThread.join()
self.assertEqual(set(keysToWrite), set(changedKeys))
finally:
time.sleep(0.01)
harness.teardown()
def test_null_value_is_deleted(self):
numEvents = 1024 * 64
valueSize = 128
numKeys = numEvents
view = createViewWithNoChannel()
keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1)
keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True)
cache = SharedStateNative.KeyspaceCache(keyrange, None)
gc.collect()
m0 = TCMalloc.getBytesUsed()
for event in producePartialEvents(view, [keyspace], 'test', numKeys, numKeys, valueSize):
cache.addEvent(event)
cache.addEvent(produceNullifyingEvent(view, event))
cache.newMinimumId(numKeys * 2)
gc.collect()
self.assertLess(TCMalloc.getBytesUsed() - m0, 1024 * 4)
def test_exception(self):
v = self.newView()
v.waitConnect()
v.subscribe(self.introRange)
key = SharedState.Key(self.introKeyspace, (NativeJson.Json("0"),NativeJson.Json(str(v.id))))
# this should raise because the view is not frozen
def shouldRaise():
v[key] = NativeJson.Json("some data")
self.assertRaises(UserWarning, shouldRaise)
def test_add_transaction(self):
try:
toWrite = {
NativeJson.Json(str(x)): NativeJson.Json('value-%s' % x)
for x in range(50)
}
simpleView = None
def onConnect(_):
def callback(value):
for key, value in toWrite.iteritems():
key = SharedState.Key(self.testKeyspace, (key,))
simpleView.pushTransaction(key, value)
deferred = simpleView.subscribeToKeyspace(self.testKeyspace, 0, callback)
deferred.addCallbacks(callback, lambda value: None)
simpleView = AsyncView.AsyncView(self.harness.viewFactory,
onConnectCallback=onConnect)
simpleView.startService()
v1 = self.harness.newView()
listener = SharedState.Listener(v1)
self.subscribeToTestKeyspace(v1)
toCheck = dict(toWrite)
while len(toCheck):
updates = listener.get()
for updateType, update in updates:
if updateType == "KeyUpdates":
for key in update:
if key[0] in toCheck:
with SharedState.Transaction(v1):
self.assertEqual(toCheck[key[0]], v1[key].value())
del toCheck[key[0]]
finally:
if simpleView:
simpleView.stopService()
self.harness.teardown()
def test_keyspace_iteration(self):
try:
toWrite = {
NativeJson.Json(str(x)): NativeJson.Json('value-%s' % x)
for x in range(50)
}
simpleView = None
v1 = self.harness.newView()
self.subscribeToTestKeyspace(v1)
for key, value in toWrite.iteritems():
key = SharedState.Key(self.testKeyspace, (key,))
with SharedState.Transaction(v1):
v1[key] = value
v1.flush()
def onConnect(_):
def checkIteration(value):
for key, value in simpleView.keyspaceItems(self.testKeyspace):
assert key[0] in toWrite
self.assertEqual(toWrite[key[0]], value)
deferred = simpleView.subscribeToKeyspace(self.testKeyspace,
0,
lambda keys: None)
deferred.addCallbacks(checkIteration, lambda value: None)
simpleView = AsyncView.AsyncView(self.harness.viewFactory,
onConnectCallback=onConnect)
simpleView.startService()
finally:
if simpleView:
simpleView.stopService()
self.harness.teardown()
def test_simple_manager(self):
view = createViewWithNoChannel()
before = TCMalloc.getBytesUsed()
keyspace = SharedState.Keyspace("TakeHighestIdKeyType", json('test'), 1)
cache = SharedStateNative.KeyspaceManager(0, 1, 0x7fffffff, 0x7fffffff, None)
for event in producePartialEvents(view, [keyspace], 'test', 1024 * 32, 1, 8):
cache.addEvent(event)
view = None
gc.collect()
bytesUsed = TCMalloc.getBytesUsed() - before
self.assertTrue(bytesUsed < 1024 * 128)
def test(threadIx):
for subPassIx in range(subPasses):
logging.info("Thread %s starting pass %s", threadIx, subPassIx)
testKeyspace = SharedState.Keyspace(
"TakeHighestIdKeyType", NativeJson.Json("TestSpace"), 1)
view = viewFactory.createView()
rng = SharedState.KeyRange(testKeyspace, 0, None, None, True,
False)
view.subscribe(rng)
for ix in range(keysToWrite):
with SharedState.Transaction(view):
ix = random.randint(0, keyspaceSize)
key = SharedState.Key(
testKeyspace, (NativeJson.Json("key %s" % ix), ))
value = uuid.uuid4().hex
view[key] = NativeJson.Json(value)
worked[threadIx] = True
def compressOrphandLogFiles(self):
for keyspaceDir in os.listdir(self.cachePath):
keyspaceType, dimensions, keyspaceName = keyspaceDir.split('::')
dimensions = int(dimensions)
keyspace = SharedState.Keyspace(
keyspaceType, NativeJson.Json.parse(keyspaceName), dimensions)
for i in range(dimensions):
logging.info("Compressing keyspace: %s", keyspaceName)
keyspaceStorage = self.keyspaceManager.storage.storageForKeyspace(
keyspace, i)
keyspaceStorage.compress()
def test_file_management(self):
tempDir = tempfile.mkdtemp()
curOpenFiles = len(os.listdir('/proc/%s/fd' % os.getpid()))
OPEN_FILE_LIMIT = 200
if curOpenFiles >= OPEN_FILE_LIMIT:
os.system("ls -alh /proc/%s/fd" % os.getpid())
self.assertTrue(curOpenFiles < OPEN_FILE_LIMIT,
"Too many open files: %s" % curOpenFiles)
soft, hard = resource.getrlimit(resource.RLIMIT_OFILE)
harness = self.getHarness(inMemory=True,
cachePathOverride=tempDir,
maxOpenFiles=15)
try:
v1 = harness.newView()
resource.setrlimit(resource.RLIMIT_OFILE,
(curOpenFiles + 30, hard))
for ix in range(128):
space = SharedState.Keyspace(
"TakeHighestIdKeyType",
NativeJson.Json("TestSpace%s" % ix), 1)
rng = SharedState.KeyRange(space, 0, None, None, True, False)
v1.subscribe(rng)
key = SharedState.Key(space, (NativeJson.Json('key%s' % ix), ))
with SharedState.Transaction(v1):
v1[key] = NativeJson.Json('value %s' % ix)
finally:
time.sleep(0.01)
harness.teardown()
resource.setrlimit(resource.RLIMIT_OFILE, (soft, hard))
try:
shutil.rmtree(tempDir)
except:
pass
def __init__(self, ChannelType, numKeys, keySize):
self.view = createViewWithNoChannel()
self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType", json('test'), 1)
self.viewChannel, self.managerChannel = ChannelType(callbackScheduler)
self.eventsRead = 0
self.numKeys = numKeys
self.keySize = keySize
self.iterator = enumerate(producePartialEvents(
self.view,
[self.keyspace],
'test',
self.numKeys,
1,
self.keySize))
def getValue(self, key):
self.assertSingleThreaded()
assert not self._stopFlag.is_set()
if key.keyspace not in self._subscribedKeyspaces:
raise UserWarning(
"Must be subscribed to a keyspace before pushing a transaction"
)
with SharedState.Transaction(self._view):
valueHolder = self._view[key]
if valueHolder is None:
raise KeyError()
return valueHolder.value()
def assertSharedStateKeyResolvesToValue(self, key, view, value):
passes = 0
lastVal = None
while passes < 100:
with SharedState.Transaction(view):
if view[key] is not None:
lastVal = view[key].value()
self.assertEqual(value, lastVal)
return
passes += 1
time.sleep(.1)
assert False, "key %s never resolved to value %s. stayed as %s" % (
key, value, lastVal)
def disabled_read_write_kv_maps_using_storage(self):
for ix in range(1000):
self.harness.writeToKeyspace(
self.view,
self.keyspaceName,
NativeJson.Json('key%s' % ix),
NativeJson.Json('test%s' % ix)
)
logging.warn("tempDir: %s, tempDir2: %s", self.tempDir, self.tempDir2)
allItemsFromViewInitial = self.harness.getAllItemsFromView(self.view, self.keyspaceName)
keyspaceList = [x for x in self.harness.manager.getAllKeyspaces()
if x != SharedState.getClientInfoKeyspace()]
self.assertTrue(len(allItemsFromViewInitial) == 1000)
self.view.flush()
self.harness.sendPingAndCompact()
self.harness.teardown()
try:
storageA = SharedStateNative.Storage.FileStorage(self.tempDir, 100, 10)
storageB = SharedStateNative.Storage.FileStorage(self.tempDir2, 100, 10)
self.assertTrue(len(keyspaceList) == 1)
for keyspace in keyspaceList:
storageForKeyspaceA = storageA.storageForKeyspace(keyspace, 0)
storageForKeyspaceB = storageB.storageForKeyspace(keyspace, 0)
data = storageForKeyspaceA.readKeyValueMap()
self.assertTrue(len(data) > 0)
storageForKeyspaceB.writeKeyValueMap(data)
self.harness = self.newHarness(self.tempDir2)
self.view = self.harness.newView()
self.harness.subscribeToKeyspace(self.view, self.keyspaceName)
allItemsFromViewFinal = self.harness.getAllItemsFromView(self.view, self.keyspaceName)
self.assertEqual(allItemsFromViewInitial, allItemsFromViewFinal)
finally:
if storageA:
storageA.shutdown()
if storageB:
storageB.shutdown()
def test_require_subscription_view(self):
try:
testKeyspace = SharedState.Keyspace("TakeHighestIdKeyType",
NativeJson.Json("TestSpace"),
1)
harness = self.getHarness(inMemory=True)
v1 = harness.newView()
def toTry():
with SharedState.Transaction(v1):
key = SharedState.Key(testKeyspace, ("key", ))
v1[key] = NativeJson.Json('this is a test value')
self.assertRaises(UserWarning, toTry)
finally:
time.sleep(0.01)
harness.teardown()
def subscribeToKey(self, keyspace, keyname, dimension, callback=None):
'''
Subscribe to a single key in a keyspace and optionally receive notifications
of changes via a callback.
callback signature:
callback(items) : where items is a dictionary from (SharedState.Key -> string)
'''
self.assertSingleThreaded()
keyrange = SharedState.KeyRange(keyspace, dimension, keyname, keyname,
True, True)
return self.subscribeToKeyspace(keyspace,
dimension,
callback,
keyrange=keyrange)
def _handleKeyUpdates(self, keys):
assert len(keys)
toCallback = {}
with SharedState.Transaction(self._view):
for key in keys:
if key.keyspace in self._keyspaceCallbackFunctions:
if not key.keyspace in toCallback:
toCallback[key.keyspace] = []
toCallback[key.keyspace].append(
(key, self._view[key].value()))
for keyspaceName, changedPairs in toCallback.iteritems():
try:
self._keyspaceCallbackFunctions[keyspaceName](
dict(changedPairs))
except Exception as e:
logging.error('exception calling callback\n%s',
''.join(traceback.format_exc()))
self._onError(CallbackException(e))
def getAllKeysFromView(self, view, spacename):
keyspace = SharedState.Keyspace("TakeHighestIdKeyType", spacename, 1)
with SharedState.Transaction(view):
return [x for x in SharedState.iterKeys(view, keyspace)]
def getIntrospectionContentsForView(self, view):
for k, v in SharedState.iterItems(view, SharedState.getClientInfoKeyspace()):
if v.value() != NativeJson.Json('disconnected'):
yield int(k[1].toSimple())
def getAllItemsFromView(self, view, spacename):
keyspace = SharedState.Keyspace("TakeHighestIdKeyType", spacename, 1)
with SharedState.Transaction(view):
return [(x[0], x[1].value()) for x in SharedState.iterItems(view, keyspace)]
