def test_events_in_subscription(self): asynView = None try: v1 = self.harness.newView() self.subscribeToTestKeyspace(v1) for ix in range(100): key = SharedState.Key(self.testKeyspace, (NativeJson.Json('test%s' % ix),)) with SharedState.Transaction(v1): v1[key] = NativeJson.Json('value1') done = threading.Event() numKeyEvents = [0] def onKeyEvents(eventDict): numKeyEvents[0] += len(eventDict) def onSubscriptionLoaded(_): done.set() def onConnected(_): deferred = asynView.subscribeToKeyspace(self.testKeyspace, 0, onKeyEvents) deferred.addCallbacks(onSubscriptionLoaded, lambda error: None) asynView = AsyncView.AsyncView(self.harness.viewFactory, onConnectCallback=onConnected) asynView.startService() while not done.wait(.1): pass self.assertEqual(numKeyEvents[0], 100) finally: asynView.stopService()
def setUp(self): self.harness = SharedStateTestHarness.SharedStateTestHarness(True) self.manager = SharedStateService.KeyspaceManager(10001,1, pingInterval=1) self.introKeyspace = SharedState.getClientInfoKeyspace() self.introRange = SharedState.KeyRange(self.introKeyspace, 1, None, None, True, False) self.views = [] self.maxDiff = None
def runInTransaction(self, keyPathElement, toRun, waitForKeyToLoad = True): """ Runs a specified function in a shared state transaction. toRun is a function that takes the following argumenst: self - a self reference key - the shared state key used by the test view - the a shared state view subscribed to the key range that contains key. waitForKeyToLoad - wait for SharedState to actually load the key """ view = self.sharedState.newView() view.subscribe(self.keyRange) passes = 0 while passes < 100: with SharedState.Transaction(view): key = SharedState.Key(self.keyspace, (Node.keyPathToKeyName((keyPathElement,)),)) if not waitForKeyToLoad or view[key] is not None: toRun(self, key, view) return passes += 1 time.sleep(.1) assert False, "Test timed out"
def test_flush(self): harness = self.getHarness(inMemory=True) try: v1 = harness.newView() space = SharedState.Keyspace("ComparisonKeyType", NativeJson.Json("TestSpace"), 1) rng = SharedState.KeyRange(space, 0, None, None, True, False) v1.subscribe(rng) keysToWrite = [str(x) for x in range(40)] value = "value" * 100 * 1024 with SharedState.Transaction(v1): self.assertRaises(UserWarning, v1.flush, True) with SharedState.Transaction(v1): for keyName in keysToWrite: key = SharedState.Key(space, (NativeJson.Json(keyName), )) v1[key] = NativeJson.Json(value) v1.flush() finally: time.sleep(0.01) harness.teardown()
def createLogFileDir(self): self.tempdir = tempfile.mkdtemp() self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("test-space"), 1) self.keyrange = SharedState.KeyRange(self.keyspace, 0, None, None, True, False) self.logFileDir = StorageNative.LogFileDirectory( self.tempdir, self.keyspace, self.keyrange) self.baseDir = os.path.split(self.logFileDir.getCurrentLogPath())[0]
def subscribeToKeyspace(view, keyspaceNames, keyspaceType): keyspaces = [SharedState.Keyspace(keyspaceType, json(keyspaceName), 1) for keyspaceName in keyspaceNames] keyranges = [SharedState.KeyRange(keyspace, 0, None, None, True, True) for keyspace in keyspaces] view.subscribe(keyranges) return keyspaces, keyranges
def setUp(self): self.sharedState = SharedStateTestHarness(inMemory = True) self.keyspacePath = (0, 1) self.keyspaceName = Node.keyspacePathToKeyspaceName(self.keyspacePath) self.keyName = 'key' self.keyValue = 'value' self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType", self.keyspaceName, 1) self.keyRange = SharedState.KeyRange(self.keyspace, 0, None, None, True, False)
def keyspaceIterItems(self, keyspace): self.assertSingleThreaded() startKey = tuple( [NativeJson.lowestValue() for x in range(keyspace.dimension)]) with SharedState.Transaction(self._view): k = self._view.nextKey(SharedState.Key(keyspace, startKey)) while k is not None and str(k.keyspace) == str(keyspace.name): if self._view[k] is not None: yield k, self._view[k].value() k = self._view.nextKey(k)
def addKeyspaceListener(self, keyspaceName, keyspaceListener, keyPrefix, blockUntilLoaded=False, assertAlreadyLoaded=False): """Add a keyspace listener to the synchronizer.""" assert isinstance(keyPrefix, NativeJson.Json) assert keyPrefix.isArray() assert isinstance(keyspaceName, NativeJson.Json) lookupKey = (keyspaceName, keyPrefix) if keyspaceName not in self.keyspaceNameToPrefixes_: self.keyspaceNameToPrefixes_[keyspaceName] = set() for otherPrefix in self.keyspaceNameToPrefixes_[keyspaceName]: assert not ( otherPrefix.arrayStartswith(keyPrefix) and otherPrefix != keyPrefix ), ("Can't subscribe to %s:%s since we're already subscribed to %s:%s" % (keyspaceName, keyPrefix, keyspaceName, otherPrefix)) self.keyspaceNameToPrefixes_[keyspaceName].add(keyPrefix) if lookupKey not in self.keyspaceListenersByKeyspaceAndPrefix_: if assertAlreadyLoaded: assert False, "Keyspace %s was not already loaded." % keyspaceName self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey] = set() self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey].add( keyspaceListener) self.valuesByKeyspaceAndPrefix_[lookupKey] = {} logging.info("shared state subscribing to %s:%s", keyspaceName, keyPrefix) keyspace = SharedState.Keyspace("TakeHighestIdKeyType", keyspaceName, 1) self.view_.subscribe( SharedState.KeyRange(keyspace, 0, keyPrefix, keyPrefix + NativeJson.Json([None]), True, False), blockUntilLoaded) if blockUntilLoaded: self.update() self.ensureMarkedLoaded(lookupKey) else: self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey].add( keyspaceListener) keyspaceListener.keysLoaded( self.valuesByKeyspaceAndPrefix_[lookupKey], True)
def testWrite(passIx): try: testKeyspaces = [ SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace%s" % ix), 3) \ for ix in range(3) ] harness = self.getHarness(inMemory=True, cachePathOverride=tempDir) v1 = harness.newView() for space in testKeyspaces: rng = SharedState.KeyRange(space, 0, None, None, True, False) v1.subscribe(rng) for ix in range(9): keyspaceIx = (ix + passIx) % len(testKeyspaces) space = testKeyspaces[ keyspaceIx] #random.choice(testKeyspaces) with SharedState.Transaction(v1): key = SharedState.Key( space, (NativeJson.Json("key %s" % ix), NativeJson.Json(''), NativeJson.Json(''))) value = JsonNative.Json( '<value is %s in space %s on pass %s: %s>' % (ix, keyspaceIx, passIx, uuid.uuid4().hex)) v1[key] = value testValues[key] = value v1.flush() time.sleep(0.01) harness.teardown() harness = self.getHarness(inMemory=True, cachePathOverride=tempDir) v1 = harness.newView() for space in testKeyspaces: rng = SharedState.KeyRange(space, 0, None, None, True, False) v1.subscribe(rng) for key, value in testValues.iteritems(): self.assertSharedStateKeyResolvesToValue( key, v1, testValues[key]) except: logging.warn("Exception: %s", traceback.format_exc()) raise finally: time.sleep(0.01) harness.teardown()
def test_async_view(self): try: simpleView = None toWrite = { NativeJson.Json(str(ix)): NativeJson.Json('value-%s' % ix) for ix in range(100) } written = {} v1 = self.harness.newView() self.subscribeToTestKeyspace(v1) for key, value in toWrite.iteritems(): with SharedState.Transaction(v1): key = SharedState.Key(self.testKeyspace, (key,)) v1[key] = value written[key[0]] = value v1.flush() received = {} newKeysWritten = threading.Condition() def onConnected(_): def callback(keysDict): for key, value in keysDict.iteritems(): received[key[0]] = value with newKeysWritten: newKeysWritten.notify() simpleView.subscribeToKeyspace(self.testKeyspace, 0, callback) simpleView = AsyncView.AsyncView(self.harness.viewFactory, onConnectCallback=onConnected) simpleView.startService() while set(received.values()) != set(written.values()): with newKeysWritten: newKeysWritten.wait() self.assertEqual(set(received.values()), set(written.values())) finally: if simpleView: simpleView.stopService() self.harness.teardown()
def test_keyspace_cache(self): numKeys = 1024 * 256 before = TCMalloc.getBytesUsed() view = createViewWithNoChannel() keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1) keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True) cache = SharedStateNative.KeyspaceCache(keyrange, None) for event in producePartialEvents(view, [keyspace], 'test', numKeys, 1, 8): cache.addEvent(event) cache.newMinimumId(numKeys) view = None gc.collect() bytesUsed = TCMalloc.getBytesUsed() - before self.assertTrue(bytesUsed < 1024 * 16)
def __init__(self, viewFactory): Stoppable.Stoppable.__init__(self) self.viewFactory = viewFactory self.activeMachineIds = set() self._outgoingNotifications = Queue.Queue() self._lock = threading.RLock() self._lastDisconnectTime = None self._lastReconnectTime = None self._registeredIpPortAndMachineId = None logging.debug("Connecting to shared state: %s", viewFactory) self.listeners_ = set() self.clientID = None self.isConnected = False self.isConnecting = False self.disconnectedWhileConnecting = False self._triggerUpdateAfterDisconnectThreads = [] self.clientIdToIpPortAndMachineIdAsString = {} self.machineIdToClientId = {} self.workerStatusKeyspace = SharedState.Keyspace( "ComparisonKeyType", NativeJson.Json( (('P', 'CumulusNodeStatus'), )), 1) self.asyncView = None self.eventLoopThread = ManagedThread.ManagedThread( target=self.eventDispatchLoop)
def writeOwnRegistrationInformationIntoView_(self): if self._registeredIpPortAndMachineId is None: return address, ports, machineIdAsString = self._registeredIpPortAndMachineId logging.info( "CumulusActiveMachine registering worker %s at %s:%s with id %s", self.clientID, address, ports, machineIdAsString ) with self._lock: self.activeMachineIds.add(machineIdAsString) key = SharedState.Key(self.workerStatusKeyspace, (NativeJson.Json(str(self.clientID)),)) value = NativeJson.Json( (str(address), [str(x) for x in ports], machineIdAsString) ) doneWithCall = threading.Event() self.asyncView.pushTransaction(key, value, lambda result: doneWithCall.set()) while not doneWithCall.wait(.1) and not self._stopFlag.is_set(): logging.debug('CumulusActiveMachines waiting on registerSelfAsActive')
def test_listener(self): harness = self.getHarness(inMemory=True) try: v1 = harness.newView() v2 = harness.newView() l = SharedState.Listener(v2) space = SharedState.Keyspace("ComparisonKeyType", NativeJson.Json("TestSpace"), 1) rng = SharedState.KeyRange(space, 0, None, None, True, False) v1.subscribe(rng) v2.subscribe(rng) keysToWrite = [JsonNative.Json(str(x)) for x in range(40)] def writer(): with SharedState.Transaction(v1): for keyName in keysToWrite: key = SharedState.Key(space, (keyName, )) v1[key] = JsonNative.Json("value") changedKeys = [] def reader(): while len(changedKeys) < len(keysToWrite): updates = l.get() for updateType, update in updates: if updateType == "KeyUpdates": for key in update: changedKeys.append(key[0]) writerThread = threading.Thread(target=writer) readerThread = threading.Thread(target=reader) writerThread.start() readerThread.start() writerThread.join() readerThread.join() self.assertEqual(set(keysToWrite), set(changedKeys)) finally: time.sleep(0.01) harness.teardown()
def test_null_value_is_deleted(self): numEvents = 1024 * 64 valueSize = 128 numKeys = numEvents view = createViewWithNoChannel() keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1) keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True) cache = SharedStateNative.KeyspaceCache(keyrange, None) gc.collect() m0 = TCMalloc.getBytesUsed() for event in producePartialEvents(view, [keyspace], 'test', numKeys, numKeys, valueSize): cache.addEvent(event) cache.addEvent(produceNullifyingEvent(view, event)) cache.newMinimumId(numKeys * 2) gc.collect() self.assertLess(TCMalloc.getBytesUsed() - m0, 1024 * 4)
def test_exception(self): v = self.newView() v.waitConnect() v.subscribe(self.introRange) key = SharedState.Key(self.introKeyspace, (NativeJson.Json("0"),NativeJson.Json(str(v.id)))) # this should raise because the view is not frozen def shouldRaise(): v[key] = NativeJson.Json("some data") self.assertRaises(UserWarning, shouldRaise)
def test_add_transaction(self): try: toWrite = { NativeJson.Json(str(x)): NativeJson.Json('value-%s' % x) for x in range(50) } simpleView = None def onConnect(_): def callback(value): for key, value in toWrite.iteritems(): key = SharedState.Key(self.testKeyspace, (key,)) simpleView.pushTransaction(key, value) deferred = simpleView.subscribeToKeyspace(self.testKeyspace, 0, callback) deferred.addCallbacks(callback, lambda value: None) simpleView = AsyncView.AsyncView(self.harness.viewFactory, onConnectCallback=onConnect) simpleView.startService() v1 = self.harness.newView() listener = SharedState.Listener(v1) self.subscribeToTestKeyspace(v1) toCheck = dict(toWrite) while len(toCheck): updates = listener.get() for updateType, update in updates: if updateType == "KeyUpdates": for key in update: if key[0] in toCheck: with SharedState.Transaction(v1): self.assertEqual(toCheck[key[0]], v1[key].value()) del toCheck[key[0]] finally: if simpleView: simpleView.stopService() self.harness.teardown()
def test_keyspace_iteration(self): try: toWrite = { NativeJson.Json(str(x)): NativeJson.Json('value-%s' % x) for x in range(50) } simpleView = None v1 = self.harness.newView() self.subscribeToTestKeyspace(v1) for key, value in toWrite.iteritems(): key = SharedState.Key(self.testKeyspace, (key,)) with SharedState.Transaction(v1): v1[key] = value v1.flush() def onConnect(_): def checkIteration(value): for key, value in simpleView.keyspaceItems(self.testKeyspace): assert key[0] in toWrite self.assertEqual(toWrite[key[0]], value) deferred = simpleView.subscribeToKeyspace(self.testKeyspace, 0, lambda keys: None) deferred.addCallbacks(checkIteration, lambda value: None) simpleView = AsyncView.AsyncView(self.harness.viewFactory, onConnectCallback=onConnect) simpleView.startService() finally: if simpleView: simpleView.stopService() self.harness.teardown()
def test_simple_manager(self): view = createViewWithNoChannel() before = TCMalloc.getBytesUsed() keyspace = SharedState.Keyspace("TakeHighestIdKeyType", json('test'), 1) cache = SharedStateNative.KeyspaceManager(0, 1, 0x7fffffff, 0x7fffffff, None) for event in producePartialEvents(view, [keyspace], 'test', 1024 * 32, 1, 8): cache.addEvent(event) view = None gc.collect() bytesUsed = TCMalloc.getBytesUsed() - before self.assertTrue(bytesUsed < 1024 * 128)
def test(threadIx): for subPassIx in range(subPasses): logging.info("Thread %s starting pass %s", threadIx, subPassIx) testKeyspace = SharedState.Keyspace( "TakeHighestIdKeyType", NativeJson.Json("TestSpace"), 1) view = viewFactory.createView() rng = SharedState.KeyRange(testKeyspace, 0, None, None, True, False) view.subscribe(rng) for ix in range(keysToWrite): with SharedState.Transaction(view): ix = random.randint(0, keyspaceSize) key = SharedState.Key( testKeyspace, (NativeJson.Json("key %s" % ix), )) value = uuid.uuid4().hex view[key] = NativeJson.Json(value) worked[threadIx] = True
def compressOrphandLogFiles(self): for keyspaceDir in os.listdir(self.cachePath): keyspaceType, dimensions, keyspaceName = keyspaceDir.split('::') dimensions = int(dimensions) keyspace = SharedState.Keyspace( keyspaceType, NativeJson.Json.parse(keyspaceName), dimensions) for i in range(dimensions): logging.info("Compressing keyspace: %s", keyspaceName) keyspaceStorage = self.keyspaceManager.storage.storageForKeyspace( keyspace, i) keyspaceStorage.compress()
def test_file_management(self): tempDir = tempfile.mkdtemp() curOpenFiles = len(os.listdir('/proc/%s/fd' % os.getpid())) OPEN_FILE_LIMIT = 200 if curOpenFiles >= OPEN_FILE_LIMIT: os.system("ls -alh /proc/%s/fd" % os.getpid()) self.assertTrue(curOpenFiles < OPEN_FILE_LIMIT, "Too many open files: %s" % curOpenFiles) soft, hard = resource.getrlimit(resource.RLIMIT_OFILE) harness = self.getHarness(inMemory=True, cachePathOverride=tempDir, maxOpenFiles=15) try: v1 = harness.newView() resource.setrlimit(resource.RLIMIT_OFILE, (curOpenFiles + 30, hard)) for ix in range(128): space = SharedState.Keyspace( "TakeHighestIdKeyType", NativeJson.Json("TestSpace%s" % ix), 1) rng = SharedState.KeyRange(space, 0, None, None, True, False) v1.subscribe(rng) key = SharedState.Key(space, (NativeJson.Json('key%s' % ix), )) with SharedState.Transaction(v1): v1[key] = NativeJson.Json('value %s' % ix) finally: time.sleep(0.01) harness.teardown() resource.setrlimit(resource.RLIMIT_OFILE, (soft, hard)) try: shutil.rmtree(tempDir) except: pass
def __init__(self, ChannelType, numKeys, keySize): self.view = createViewWithNoChannel() self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType", json('test'), 1) self.viewChannel, self.managerChannel = ChannelType(callbackScheduler) self.eventsRead = 0 self.numKeys = numKeys self.keySize = keySize self.iterator = enumerate(producePartialEvents( self.view, [self.keyspace], 'test', self.numKeys, 1, self.keySize))
def getValue(self, key): self.assertSingleThreaded() assert not self._stopFlag.is_set() if key.keyspace not in self._subscribedKeyspaces: raise UserWarning( "Must be subscribed to a keyspace before pushing a transaction" ) with SharedState.Transaction(self._view): valueHolder = self._view[key] if valueHolder is None: raise KeyError() return valueHolder.value()
def assertSharedStateKeyResolvesToValue(self, key, view, value): passes = 0 lastVal = None while passes < 100: with SharedState.Transaction(view): if view[key] is not None: lastVal = view[key].value() self.assertEqual(value, lastVal) return passes += 1 time.sleep(.1) assert False, "key %s never resolved to value %s. stayed as %s" % ( key, value, lastVal)
def disabled_read_write_kv_maps_using_storage(self): for ix in range(1000): self.harness.writeToKeyspace( self.view, self.keyspaceName, NativeJson.Json('key%s' % ix), NativeJson.Json('test%s' % ix) ) logging.warn("tempDir: %s, tempDir2: %s", self.tempDir, self.tempDir2) allItemsFromViewInitial = self.harness.getAllItemsFromView(self.view, self.keyspaceName) keyspaceList = [x for x in self.harness.manager.getAllKeyspaces() if x != SharedState.getClientInfoKeyspace()] self.assertTrue(len(allItemsFromViewInitial) == 1000) self.view.flush() self.harness.sendPingAndCompact() self.harness.teardown() try: storageA = SharedStateNative.Storage.FileStorage(self.tempDir, 100, 10) storageB = SharedStateNative.Storage.FileStorage(self.tempDir2, 100, 10) self.assertTrue(len(keyspaceList) == 1) for keyspace in keyspaceList: storageForKeyspaceA = storageA.storageForKeyspace(keyspace, 0) storageForKeyspaceB = storageB.storageForKeyspace(keyspace, 0) data = storageForKeyspaceA.readKeyValueMap() self.assertTrue(len(data) > 0) storageForKeyspaceB.writeKeyValueMap(data) self.harness = self.newHarness(self.tempDir2) self.view = self.harness.newView() self.harness.subscribeToKeyspace(self.view, self.keyspaceName) allItemsFromViewFinal = self.harness.getAllItemsFromView(self.view, self.keyspaceName) self.assertEqual(allItemsFromViewInitial, allItemsFromViewFinal) finally: if storageA: storageA.shutdown() if storageB: storageB.shutdown()
def test_require_subscription_view(self): try: testKeyspace = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace"), 1) harness = self.getHarness(inMemory=True) v1 = harness.newView() def toTry(): with SharedState.Transaction(v1): key = SharedState.Key(testKeyspace, ("key", )) v1[key] = NativeJson.Json('this is a test value') self.assertRaises(UserWarning, toTry) finally: time.sleep(0.01) harness.teardown()
def subscribeToKey(self, keyspace, keyname, dimension, callback=None): ''' Subscribe to a single key in a keyspace and optionally receive notifications of changes via a callback. callback signature: callback(items) : where items is a dictionary from (SharedState.Key -> string) ''' self.assertSingleThreaded() keyrange = SharedState.KeyRange(keyspace, dimension, keyname, keyname, True, True) return self.subscribeToKeyspace(keyspace, dimension, callback, keyrange=keyrange)
def _handleKeyUpdates(self, keys): assert len(keys) toCallback = {} with SharedState.Transaction(self._view): for key in keys: if key.keyspace in self._keyspaceCallbackFunctions: if not key.keyspace in toCallback: toCallback[key.keyspace] = [] toCallback[key.keyspace].append( (key, self._view[key].value())) for keyspaceName, changedPairs in toCallback.iteritems(): try: self._keyspaceCallbackFunctions[keyspaceName]( dict(changedPairs)) except Exception as e: logging.error('exception calling callback\n%s', ''.join(traceback.format_exc())) self._onError(CallbackException(e))
def getAllKeysFromView(self, view, spacename): keyspace = SharedState.Keyspace("TakeHighestIdKeyType", spacename, 1) with SharedState.Transaction(view): return [x for x in SharedState.iterKeys(view, keyspace)]
def getIntrospectionContentsForView(self, view): for k, v in SharedState.iterItems(view, SharedState.getClientInfoKeyspace()): if v.value() != NativeJson.Json('disconnected'): yield int(k[1].toSimple())
def getAllItemsFromView(self, view, spacename): keyspace = SharedState.Keyspace("TakeHighestIdKeyType", spacename, 1) with SharedState.Transaction(view): return [(x[0], x[1].value()) for x in SharedState.iterItems(view, keyspace)]