def test_two_keyspaces(self): try: toWrite = Queue.Queue() space1 = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace1"), 1) space2 = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace2"), 1) def callback1(items): for key in items.iterkeys(): self.assertEqual(key.keyspace, NativeJson.Json("TestSpace1")) toWrite.task_done() def callback2(items): for key in items.iterkeys(): self.assertEqual(key.keyspace, NativeJson.Json("TestSpace2")) toWrite.task_done() def onConnected(_): simpleView.subscribeToKeyspace(space1, 0, callback1) simpleView.subscribeToKeyspace(space2, 0, callback2) simpleView = AsyncView.AsyncView(self.harness.viewFactory, onConnected) simpleView.startService() v1 = self.harness.newView() v1.subscribe(SharedState.KeyRange(space1, 0, None, None, True, False), True) v1.subscribe(SharedState.KeyRange(space2, 0, None, None, True, False), True) for x in range(100): space = space1 if random.random() < .5: space = space2 toWrite.put((str(x), space)) with SharedState.Transaction(v1): v1[SharedState.Key(space, (NativeJson.Json(str(x)),))] = space.name toWrite.join() finally: if simpleView: simpleView.stopService() self.harness.teardown()
def __init__(self, viewFactory): Stoppable.Stoppable.__init__(self) self.viewFactory = viewFactory self.activeMachineIds = set() self._outgoingNotifications = Queue.Queue() self._lock = threading.RLock() self._lastDisconnectTime = None self._lastReconnectTime = None self._registeredIpPortAndMachineId = None logging.debug("Connecting to shared state: %s", viewFactory) self.listeners_ = set() self.clientID = None self.isConnected = False self.isConnecting = False self.disconnectedWhileConnecting = False self._triggerUpdateAfterDisconnectThreads = [] self.clientIdToIpPortAndMachineIdAsString = {} self.machineIdToClientId = {} self.workerStatusKeyspace = SharedState.Keyspace( "ComparisonKeyType", NativeJson.Json( (('P', 'CumulusNodeStatus'), )), 1) self.asyncView = None self.eventLoopThread = ManagedThread.ManagedThread( target=self.eventDispatchLoop)
def test_flush(self): harness = self.getHarness(inMemory=True) try: v1 = harness.newView() space = SharedState.Keyspace("ComparisonKeyType", NativeJson.Json("TestSpace"), 1) rng = SharedState.KeyRange(space, 0, None, None, True, False) v1.subscribe(rng) keysToWrite = [str(x) for x in range(40)] value = "value" * 100 * 1024 with SharedState.Transaction(v1): self.assertRaises(UserWarning, v1.flush, True) with SharedState.Transaction(v1): for keyName in keysToWrite: key = SharedState.Key(space, (NativeJson.Json(keyName), )) v1[key] = NativeJson.Json(value) v1.flush() finally: time.sleep(0.01) harness.teardown()
def subscribeToKeyspace(view, keyspaceNames, keyspaceType): keyspaces = [SharedState.Keyspace(keyspaceType, json(keyspaceName), 1) for keyspaceName in keyspaceNames] keyranges = [SharedState.KeyRange(keyspace, 0, None, None, True, True) for keyspace in keyspaces] view.subscribe(keyranges) return keyspaces, keyranges
def createLogFileDir(self): self.tempdir = tempfile.mkdtemp() self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("test-space"), 1) self.keyrange = SharedState.KeyRange(self.keyspace, 0, None, None, True, False) self.logFileDir = StorageNative.LogFileDirectory( self.tempdir, self.keyspace, self.keyrange) self.baseDir = os.path.split(self.logFileDir.getCurrentLogPath())[0]
def setUp(self): self.sharedState = SharedStateTestHarness(inMemory = True) self.keyspacePath = (0, 1) self.keyspaceName = Node.keyspacePathToKeyspaceName(self.keyspacePath) self.keyName = 'key' self.keyValue = 'value' self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType", self.keyspaceName, 1) self.keyRange = SharedState.KeyRange(self.keyspace, 0, None, None, True, False)
def addKeyspaceListener(self, keyspaceName, keyspaceListener, keyPrefix, blockUntilLoaded=False, assertAlreadyLoaded=False): """Add a keyspace listener to the synchronizer.""" assert isinstance(keyPrefix, NativeJson.Json) assert keyPrefix.isArray() assert isinstance(keyspaceName, NativeJson.Json) lookupKey = (keyspaceName, keyPrefix) if keyspaceName not in self.keyspaceNameToPrefixes_: self.keyspaceNameToPrefixes_[keyspaceName] = set() for otherPrefix in self.keyspaceNameToPrefixes_[keyspaceName]: assert not ( otherPrefix.arrayStartswith(keyPrefix) and otherPrefix != keyPrefix ), ("Can't subscribe to %s:%s since we're already subscribed to %s:%s" % (keyspaceName, keyPrefix, keyspaceName, otherPrefix)) self.keyspaceNameToPrefixes_[keyspaceName].add(keyPrefix) if lookupKey not in self.keyspaceListenersByKeyspaceAndPrefix_: if assertAlreadyLoaded: assert False, "Keyspace %s was not already loaded." % keyspaceName self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey] = set() self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey].add( keyspaceListener) self.valuesByKeyspaceAndPrefix_[lookupKey] = {} logging.info("shared state subscribing to %s:%s", keyspaceName, keyPrefix) keyspace = SharedState.Keyspace("TakeHighestIdKeyType", keyspaceName, 1) self.view_.subscribe( SharedState.KeyRange(keyspace, 0, keyPrefix, keyPrefix + NativeJson.Json([None]), True, False), blockUntilLoaded) if blockUntilLoaded: self.update() self.ensureMarkedLoaded(lookupKey) else: self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey].add( keyspaceListener) keyspaceListener.keysLoaded( self.valuesByKeyspaceAndPrefix_[lookupKey], True)
def testWrite(passIx): try: testKeyspaces = [ SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace%s" % ix), 3) \ for ix in range(3) ] harness = self.getHarness(inMemory=True, cachePathOverride=tempDir) v1 = harness.newView() for space in testKeyspaces: rng = SharedState.KeyRange(space, 0, None, None, True, False) v1.subscribe(rng) for ix in range(9): keyspaceIx = (ix + passIx) % len(testKeyspaces) space = testKeyspaces[ keyspaceIx] #random.choice(testKeyspaces) with SharedState.Transaction(v1): key = SharedState.Key( space, (NativeJson.Json("key %s" % ix), NativeJson.Json(''), NativeJson.Json(''))) value = JsonNative.Json( '<value is %s in space %s on pass %s: %s>' % (ix, keyspaceIx, passIx, uuid.uuid4().hex)) v1[key] = value testValues[key] = value v1.flush() time.sleep(0.01) harness.teardown() harness = self.getHarness(inMemory=True, cachePathOverride=tempDir) v1 = harness.newView() for space in testKeyspaces: rng = SharedState.KeyRange(space, 0, None, None, True, False) v1.subscribe(rng) for key, value in testValues.iteritems(): self.assertSharedStateKeyResolvesToValue( key, v1, testValues[key]) except: logging.warn("Exception: %s", traceback.format_exc()) raise finally: time.sleep(0.01) harness.teardown()
def test_simple_manager(self): view = createViewWithNoChannel() before = TCMalloc.getBytesUsed() keyspace = SharedState.Keyspace("TakeHighestIdKeyType", json('test'), 1) cache = SharedStateNative.KeyspaceManager(0, 1, 0x7fffffff, 0x7fffffff, None) for event in producePartialEvents(view, [keyspace], 'test', 1024 * 32, 1, 8): cache.addEvent(event) view = None gc.collect() bytesUsed = TCMalloc.getBytesUsed() - before self.assertTrue(bytesUsed < 1024 * 128)
def compressOrphandLogFiles(self): for keyspaceDir in os.listdir(self.cachePath): keyspaceType, dimensions, keyspaceName = keyspaceDir.split('::') dimensions = int(dimensions) keyspace = SharedState.Keyspace( keyspaceType, NativeJson.Json.parse(keyspaceName), dimensions) for i in range(dimensions): logging.info("Compressing keyspace: %s", keyspaceName) keyspaceStorage = self.keyspaceManager.storage.storageForKeyspace( keyspace, i) keyspaceStorage.compress()
def __init__(self, ChannelType, numKeys, keySize): self.view = createViewWithNoChannel() self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType", json('test'), 1) self.viewChannel, self.managerChannel = ChannelType(callbackScheduler) self.eventsRead = 0 self.numKeys = numKeys self.keySize = keySize self.iterator = enumerate(producePartialEvents( self.view, [self.keyspace], 'test', self.numKeys, 1, self.keySize))
def test_keyspace_cache(self): numKeys = 1024 * 256 before = TCMalloc.getBytesUsed() view = createViewWithNoChannel() keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1) keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True) cache = SharedStateNative.KeyspaceCache(keyrange, None) for event in producePartialEvents(view, [keyspace], 'test', numKeys, 1, 8): cache.addEvent(event) cache.newMinimumId(numKeys) view = None gc.collect() bytesUsed = TCMalloc.getBytesUsed() - before self.assertTrue(bytesUsed < 1024 * 16)
def test_listener(self): harness = self.getHarness(inMemory=True) try: v1 = harness.newView() v2 = harness.newView() l = SharedState.Listener(v2) space = SharedState.Keyspace("ComparisonKeyType", NativeJson.Json("TestSpace"), 1) rng = SharedState.KeyRange(space, 0, None, None, True, False) v1.subscribe(rng) v2.subscribe(rng) keysToWrite = [JsonNative.Json(str(x)) for x in range(40)] def writer(): with SharedState.Transaction(v1): for keyName in keysToWrite: key = SharedState.Key(space, (keyName, )) v1[key] = JsonNative.Json("value") changedKeys = [] def reader(): while len(changedKeys) < len(keysToWrite): updates = l.get() for updateType, update in updates: if updateType == "KeyUpdates": for key in update: changedKeys.append(key[0]) writerThread = threading.Thread(target=writer) readerThread = threading.Thread(target=reader) writerThread.start() readerThread.start() writerThread.join() readerThread.join() self.assertEqual(set(keysToWrite), set(changedKeys)) finally: time.sleep(0.01) harness.teardown()
def test_require_subscription_view(self): try: testKeyspace = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace"), 1) harness = self.getHarness(inMemory=True) v1 = harness.newView() def toTry(): with SharedState.Transaction(v1): key = SharedState.Key(testKeyspace, ("key", )) v1[key] = NativeJson.Json('this is a test value') self.assertRaises(UserWarning, toTry) finally: time.sleep(0.01) harness.teardown()
def test_null_value_is_deleted(self): numEvents = 1024 * 64 valueSize = 128 numKeys = numEvents view = createViewWithNoChannel() keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1) keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True) cache = SharedStateNative.KeyspaceCache(keyrange, None) gc.collect() m0 = TCMalloc.getBytesUsed() for event in producePartialEvents(view, [keyspace], 'test', numKeys, numKeys, valueSize): cache.addEvent(event) cache.addEvent(produceNullifyingEvent(view, event)) cache.newMinimumId(numKeys * 2) gc.collect() self.assertLess(TCMalloc.getBytesUsed() - m0, 1024 * 4)
def test(threadIx): for subPassIx in range(subPasses): logging.info("Thread %s starting pass %s", threadIx, subPassIx) testKeyspace = SharedState.Keyspace( "TakeHighestIdKeyType", NativeJson.Json("TestSpace"), 1) view = viewFactory.createView() rng = SharedState.KeyRange(testKeyspace, 0, None, None, True, False) view.subscribe(rng) for ix in range(keysToWrite): with SharedState.Transaction(view): ix = random.randint(0, keyspaceSize) key = SharedState.Key( testKeyspace, (NativeJson.Json("key %s" % ix), )) value = uuid.uuid4().hex view[key] = NativeJson.Json(value) worked[threadIx] = True
def __init__(self, numIters, valueSize=128, useMemoChannel=True, numKeyspaces=1000, keyType="ComparisonKeyType"): self.cacheDir = tempfile.mkdtemp() self.fileStore = SharedStateNative.Storage.FileStorage( self.cacheDir, 100, .1, useMemoChannel) self.view = createViewWithNoChannel() self.keyspaces = [SharedState.Keyspace(keyType, json('test-%s' % ix), 1) for ix in range(numKeyspaces)] self._numIters = numIters self.keyspaceStorage = {} self.iterator = enumerate( producePartialEvents( self.view, self.keyspaces, 'test', self._numIters, 1, valueSize))
def test_file_management(self): tempDir = tempfile.mkdtemp() curOpenFiles = len(os.listdir('/proc/%s/fd' % os.getpid())) OPEN_FILE_LIMIT = 200 if curOpenFiles >= OPEN_FILE_LIMIT: os.system("ls -alh /proc/%s/fd" % os.getpid()) self.assertTrue(curOpenFiles < OPEN_FILE_LIMIT, "Too many open files: %s" % curOpenFiles) soft, hard = resource.getrlimit(resource.RLIMIT_OFILE) harness = self.getHarness(inMemory=True, cachePathOverride=tempDir, maxOpenFiles=15) try: v1 = harness.newView() resource.setrlimit(resource.RLIMIT_OFILE, (curOpenFiles + 30, hard)) for ix in range(128): space = SharedState.Keyspace( "TakeHighestIdKeyType", NativeJson.Json("TestSpace%s" % ix), 1) rng = SharedState.KeyRange(space, 0, None, None, True, False) v1.subscribe(rng) key = SharedState.Key(space, (NativeJson.Json('key%s' % ix), )) with SharedState.Transaction(v1): v1[key] = NativeJson.Json('value %s' % ix) finally: time.sleep(0.01) harness.teardown() resource.setrlimit(resource.RLIMIT_OFILE, (soft, hard)) try: shutil.rmtree(tempDir) except: pass
def test_require_subscription_manager(self): tempDir = tempfile.mkdtemp() try: testKeyspace = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace"), 1) harness = self.getHarness(inMemory=True, cachePathOverride=tempDir) v1 = harness.newView() v1.subscribe( SharedState.KeyRange(testKeyspace, 0, None, None, True, False)) for ix in range(1000): with SharedState.Transaction(v1): key = SharedState.Key(testKeyspace, (NativeJson.Json("key"), )) v1[key] = NativeJson.Json(str(ix)) time.sleep(2) v2 = harness.newView() v2.setMustSubscribe(False) with SharedState.Transaction(v2): key = SharedState.Key(testKeyspace, (NativeJson.Json("key"), )) value = 'value is' + uuid.uuid4().hex v2[key] = NativeJson.Json(value) time.sleep(2) def toTry(): with SharedState.Transaction(v2): key = SharedState.Key(testKeyspace, (NativeJson.Json("key"), )) value = 'value is' + uuid.uuid4().hex v2[key] = NativeJson.Json(value) self.assertRaises(UserWarning, toTry) finally: time.sleep(0.01) harness.teardown() shutil.rmtree(tempDir)
def getAllItemsFromView(self, view, spacename): keyspace = SharedState.Keyspace("TakeHighestIdKeyType", spacename, 1) with SharedState.Transaction(view): return [(x[0], x[1].value()) for x in SharedState.iterItems(view, keyspace)]
def writeValue_(self, keyspaceName, keyName, value): self.view_[SharedState.Key( SharedState.Keyspace("TakeHighestIdKeyType", keyspaceName, 1), (keyName, ))] = value
def setUp(self): print self.keyspace = SharedState.Keyspace("TakeHighestIdType", json('test_keyspace'), 1) self.keyrange = SharedState.KeyRange(self.keyspace, 0, None, None, True, True)
# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest import ufora.distributed.SharedState.SharedState as SharedState import ufora.distributed.SharedState.tests.SharedStateTestHarness as SharedStateTestHarness import ufora.native.Json as NativeJson _testKeyspace = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace"), 1) class ListenerTest(unittest.TestCase): def setUp(self): self.harness = SharedStateTestHarness.SharedStateTestHarness(True) def test_basic_listener(self): v1 = self.harness.newView() v2 = self.harness.newView() while not v1.waitConnectTimeout(.1): pass while not v2.waitConnectTimeout(.1): pass
def subscribeToKeyspace(self, view, spacename): keyspace = SharedState.Keyspace("TakeHighestIdKeyType", spacename, 1) view.subscribe( SharedState.KeyRange(keyspace, 0, None, None, True, False), True)
def setUp(self): self.testKeyspace = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace"), 1) self.harness = SharedStateTestHarness.SharedStateTestHarness(inMemory=True)
def getAllKeysFromView(self, view, spacename): keyspace = SharedState.Keyspace("TakeHighestIdKeyType", spacename, 1) with SharedState.Transaction(view): return [x for x in SharedState.iterKeys(view, keyspace)]
def writeToKeyspace(self, view, spacename, key, val): keyspace = SharedState.Keyspace("TakeHighestIdKeyType", spacename, 1) with SharedState.Transaction(view): view[SharedState.Key(keyspace, (key, ))] = val