def testWrite(passIx): try: testKeyspaces = [ SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace%s" % ix), 3) \ for ix in range(3) ] harness = self.getHarness(inMemory=True, cachePathOverride=tempDir) v1 = harness.newView() for space in testKeyspaces: rng = SharedState.KeyRange(space, 0, None, None, True, False) v1.subscribe(rng) for ix in range(9): keyspaceIx = (ix + passIx) % len(testKeyspaces) space = testKeyspaces[ keyspaceIx] #random.choice(testKeyspaces) with SharedState.Transaction(v1): key = SharedState.Key( space, (NativeJson.Json("key %s" % ix), NativeJson.Json(''), NativeJson.Json(''))) value = JsonNative.Json( '<value is %s in space %s on pass %s: %s>' % (ix, keyspaceIx, passIx, uuid.uuid4().hex)) v1[key] = value testValues[key] = value v1.flush() time.sleep(0.01) harness.teardown() harness = self.getHarness(inMemory=True, cachePathOverride=tempDir) v1 = harness.newView() for space in testKeyspaces: rng = SharedState.KeyRange(space, 0, None, None, True, False) v1.subscribe(rng) for key, value in testValues.iteritems(): self.assertSharedStateKeyResolvesToValue( key, v1, testValues[key]) except: logging.warn("Exception: %s", traceback.format_exc()) raise finally: time.sleep(0.01) harness.teardown()
def test_two_keyspaces(self): try: toWrite = Queue.Queue() space1 = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace1"), 1) space2 = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace2"), 1) def callback1(items): for key in items.iterkeys(): self.assertEqual(key.keyspace, NativeJson.Json("TestSpace1")) toWrite.task_done() def callback2(items): for key in items.iterkeys(): self.assertEqual(key.keyspace, NativeJson.Json("TestSpace2")) toWrite.task_done() def onConnected(_): simpleView.subscribeToKeyspace(space1, 0, callback1) simpleView.subscribeToKeyspace(space2, 0, callback2) simpleView = AsyncView.AsyncView(self.harness.viewFactory, onConnected) simpleView.startService() v1 = self.harness.newView() v1.subscribe(SharedState.KeyRange(space1, 0, None, None, True, False), True) v1.subscribe(SharedState.KeyRange(space2, 0, None, None, True, False), True) for x in range(100): space = space1 if random.random() < .5: space = space2 toWrite.put((str(x), space)) with SharedState.Transaction(v1): v1[SharedState.Key(space, (NativeJson.Json(str(x)),))] = space.name toWrite.join() finally: if simpleView: simpleView.stopService() self.harness.teardown()
def test_basic_listener(self): v1 = self.harness.newView() v2 = self.harness.newView() while not v1.waitConnectTimeout(.1): pass while not v2.waitConnectTimeout(.1): pass l = SharedState.Listener(v1) v1.subscribe( SharedState.KeyRange(_testKeyspace, 0, None, None, True, True), True) v2.subscribe( SharedState.KeyRange(_testKeyspace, 0, None, None, True, True), True) self.assertIsInstance(l.get()[0][1], SharedState.SharedStateNative.KeyRange) self.assertTrue(len(l.get(.1)) == 0) self.assertTrue(len(l.getNonblock()) == 0) testKey = SharedState.Key(_testKeyspace, (NativeJson.Json('test'), )) with SharedState.Transaction(v2): v2[testKey] = NativeJson.Json('test2') v2.flush() v1.flush() updates = l.get(1) self.assertTrue(len(updates) == 1) self.assertEqual(updates[0][0], "KeyUpdates") self.assertIsInstance(updates[0][1], list) self.assertTrue(updates[0][1][0] == testKey) with SharedState.Transaction(v2): v2[testKey] = NativeJson.Json('newer test') v2.flush() v1.flush() updates = l.get(1) self.assertTrue(len(updates) == 1) self.assertEqual(updates[0][0], "KeyUpdates") self.assertIsInstance(updates[0][1], list) self.assertTrue(updates[0][1][0] == testKey)
def setUp(self): self.harness = SharedStateTestHarness.SharedStateTestHarness(True) self.manager = SharedStateService.KeyspaceManager(10001,1, pingInterval=1) self.introKeyspace = SharedState.getClientInfoKeyspace() self.introRange = SharedState.KeyRange(self.introKeyspace, 1, None, None, True, False) self.views = [] self.maxDiff = None
def test_flush(self): harness = self.getHarness(inMemory=True) try: v1 = harness.newView() space = SharedState.Keyspace("ComparisonKeyType", NativeJson.Json("TestSpace"), 1) rng = SharedState.KeyRange(space, 0, None, None, True, False) v1.subscribe(rng) keysToWrite = [str(x) for x in range(40)] value = "value" * 100 * 1024 with SharedState.Transaction(v1): self.assertRaises(UserWarning, v1.flush, True) with SharedState.Transaction(v1): for keyName in keysToWrite: key = SharedState.Key(space, (NativeJson.Json(keyName), )) v1[key] = NativeJson.Json(value) v1.flush() finally: time.sleep(0.01) harness.teardown()
def subscribeToKeyspace(view, keyspaceNames, keyspaceType): keyspaces = [SharedState.Keyspace(keyspaceType, json(keyspaceName), 1) for keyspaceName in keyspaceNames] keyranges = [SharedState.KeyRange(keyspace, 0, None, None, True, True) for keyspace in keyspaces] view.subscribe(keyranges) return keyspaces, keyranges
def createLogFileDir(self): self.tempdir = tempfile.mkdtemp() self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("test-space"), 1) self.keyrange = SharedState.KeyRange(self.keyspace, 0, None, None, True, False) self.logFileDir = StorageNative.LogFileDirectory( self.tempdir, self.keyspace, self.keyrange) self.baseDir = os.path.split(self.logFileDir.getCurrentLogPath())[0]
def setUp(self): self.sharedState = SharedStateTestHarness(inMemory = True) self.keyspacePath = (0, 1) self.keyspaceName = Node.keyspacePathToKeyspaceName(self.keyspacePath) self.keyName = 'key' self.keyValue = 'value' self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType", self.keyspaceName, 1) self.keyRange = SharedState.KeyRange(self.keyspace, 0, None, None, True, False)
def addKeyspaceListener(self, keyspaceName, keyspaceListener, keyPrefix, blockUntilLoaded=False, assertAlreadyLoaded=False): """Add a keyspace listener to the synchronizer.""" assert isinstance(keyPrefix, NativeJson.Json) assert keyPrefix.isArray() assert isinstance(keyspaceName, NativeJson.Json) lookupKey = (keyspaceName, keyPrefix) if keyspaceName not in self.keyspaceNameToPrefixes_: self.keyspaceNameToPrefixes_[keyspaceName] = set() for otherPrefix in self.keyspaceNameToPrefixes_[keyspaceName]: assert not ( otherPrefix.arrayStartswith(keyPrefix) and otherPrefix != keyPrefix ), ("Can't subscribe to %s:%s since we're already subscribed to %s:%s" % (keyspaceName, keyPrefix, keyspaceName, otherPrefix)) self.keyspaceNameToPrefixes_[keyspaceName].add(keyPrefix) if lookupKey not in self.keyspaceListenersByKeyspaceAndPrefix_: if assertAlreadyLoaded: assert False, "Keyspace %s was not already loaded." % keyspaceName self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey] = set() self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey].add( keyspaceListener) self.valuesByKeyspaceAndPrefix_[lookupKey] = {} logging.info("shared state subscribing to %s:%s", keyspaceName, keyPrefix) keyspace = SharedState.Keyspace("TakeHighestIdKeyType", keyspaceName, 1) self.view_.subscribe( SharedState.KeyRange(keyspace, 0, keyPrefix, keyPrefix + NativeJson.Json([None]), True, False), blockUntilLoaded) if blockUntilLoaded: self.update() self.ensureMarkedLoaded(lookupKey) else: self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey].add( keyspaceListener) keyspaceListener.keysLoaded( self.valuesByKeyspaceAndPrefix_[lookupKey], True)
def test_keyspace_cache(self): numKeys = 1024 * 256 before = TCMalloc.getBytesUsed() view = createViewWithNoChannel() keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1) keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True) cache = SharedStateNative.KeyspaceCache(keyrange, None) for event in producePartialEvents(view, [keyspace], 'test', numKeys, 1, 8): cache.addEvent(event) cache.newMinimumId(numKeys) view = None gc.collect() bytesUsed = TCMalloc.getBytesUsed() - before self.assertTrue(bytesUsed < 1024 * 16)
def test_listener(self): harness = self.getHarness(inMemory=True) try: v1 = harness.newView() v2 = harness.newView() l = SharedState.Listener(v2) space = SharedState.Keyspace("ComparisonKeyType", NativeJson.Json("TestSpace"), 1) rng = SharedState.KeyRange(space, 0, None, None, True, False) v1.subscribe(rng) v2.subscribe(rng) keysToWrite = [JsonNative.Json(str(x)) for x in range(40)] def writer(): with SharedState.Transaction(v1): for keyName in keysToWrite: key = SharedState.Key(space, (keyName, )) v1[key] = JsonNative.Json("value") changedKeys = [] def reader(): while len(changedKeys) < len(keysToWrite): updates = l.get() for updateType, update in updates: if updateType == "KeyUpdates": for key in update: changedKeys.append(key[0]) writerThread = threading.Thread(target=writer) readerThread = threading.Thread(target=reader) writerThread.start() readerThread.start() writerThread.join() readerThread.join() self.assertEqual(set(keysToWrite), set(changedKeys)) finally: time.sleep(0.01) harness.teardown()
def subscribeToKey(self, keyspace, keyname, dimension, callback=None): ''' Subscribe to a single key in a keyspace and optionally receive notifications of changes via a callback. callback signature: callback(items) : where items is a dictionary from (SharedState.Key -> string) ''' self.assertSingleThreaded() keyrange = SharedState.KeyRange(keyspace, dimension, keyname, keyname, True, True) return self.subscribeToKeyspace(keyspace, dimension, callback, keyrange=keyrange)
def test_null_value_is_deleted(self): numEvents = 1024 * 64 valueSize = 128 numKeys = numEvents view = createViewWithNoChannel() keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1) keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True) cache = SharedStateNative.KeyspaceCache(keyrange, None) gc.collect() m0 = TCMalloc.getBytesUsed() for event in producePartialEvents(view, [keyspace], 'test', numKeys, numKeys, valueSize): cache.addEvent(event) cache.addEvent(produceNullifyingEvent(view, event)) cache.newMinimumId(numKeys * 2) gc.collect() self.assertLess(TCMalloc.getBytesUsed() - m0, 1024 * 4)
def test(threadIx): for subPassIx in range(subPasses): logging.info("Thread %s starting pass %s", threadIx, subPassIx) testKeyspace = SharedState.Keyspace( "TakeHighestIdKeyType", NativeJson.Json("TestSpace"), 1) view = viewFactory.createView() rng = SharedState.KeyRange(testKeyspace, 0, None, None, True, False) view.subscribe(rng) for ix in range(keysToWrite): with SharedState.Transaction(view): ix = random.randint(0, keyspaceSize) key = SharedState.Key( testKeyspace, (NativeJson.Json("key %s" % ix), )) value = uuid.uuid4().hex view[key] = NativeJson.Json(value) worked[threadIx] = True
def test_file_management(self): tempDir = tempfile.mkdtemp() curOpenFiles = len(os.listdir('/proc/%s/fd' % os.getpid())) OPEN_FILE_LIMIT = 200 if curOpenFiles >= OPEN_FILE_LIMIT: os.system("ls -alh /proc/%s/fd" % os.getpid()) self.assertTrue(curOpenFiles < OPEN_FILE_LIMIT, "Too many open files: %s" % curOpenFiles) soft, hard = resource.getrlimit(resource.RLIMIT_OFILE) harness = self.getHarness(inMemory=True, cachePathOverride=tempDir, maxOpenFiles=15) try: v1 = harness.newView() resource.setrlimit(resource.RLIMIT_OFILE, (curOpenFiles + 30, hard)) for ix in range(128): space = SharedState.Keyspace( "TakeHighestIdKeyType", NativeJson.Json("TestSpace%s" % ix), 1) rng = SharedState.KeyRange(space, 0, None, None, True, False) v1.subscribe(rng) key = SharedState.Key(space, (NativeJson.Json('key%s' % ix), )) with SharedState.Transaction(v1): v1[key] = NativeJson.Json('value %s' % ix) finally: time.sleep(0.01) harness.teardown() resource.setrlimit(resource.RLIMIT_OFILE, (soft, hard)) try: shutil.rmtree(tempDir) except: pass
def test_require_subscription_manager(self): tempDir = tempfile.mkdtemp() try: testKeyspace = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace"), 1) harness = self.getHarness(inMemory=True, cachePathOverride=tempDir) v1 = harness.newView() v1.subscribe( SharedState.KeyRange(testKeyspace, 0, None, None, True, False)) for ix in range(1000): with SharedState.Transaction(v1): key = SharedState.Key(testKeyspace, (NativeJson.Json("key"), )) v1[key] = NativeJson.Json(str(ix)) time.sleep(2) v2 = harness.newView() v2.setMustSubscribe(False) with SharedState.Transaction(v2): key = SharedState.Key(testKeyspace, (NativeJson.Json("key"), )) value = 'value is' + uuid.uuid4().hex v2[key] = NativeJson.Json(value) time.sleep(2) def toTry(): with SharedState.Transaction(v2): key = SharedState.Key(testKeyspace, (NativeJson.Json("key"), )) value = 'value is' + uuid.uuid4().hex v2[key] = NativeJson.Json(value) self.assertRaises(UserWarning, toTry) finally: time.sleep(0.01) harness.teardown() shutil.rmtree(tempDir)
def subscribeToTestKeyspace(self, view, block=True): view.subscribe( SharedState.KeyRange(self.testKeyspace, 0, None, None, True, False), block )
def subscribeToKeyspace(self, view, spacename): keyspace = SharedState.Keyspace("TakeHighestIdKeyType", spacename, 1) view.subscribe( SharedState.KeyRange(keyspace, 0, None, None, True, False), True)
def createAllKeysRange(keyspace, dimension): return SharedState.KeyRange(keyspace, dimension, None, None, True, False)
def setUp(self): print self.keyspace = SharedState.Keyspace("TakeHighestIdType", json('test_keyspace'), 1) self.keyrange = SharedState.KeyRange(self.keyspace, 0, None, None, True, True)