def disable_disk_space_error(self):
# root priveledges are required to execute this test
if os.geteuid() == 0:
ramdiskPath = tempfile.mkdtemp()
os.system('mount -t tmpfs -o size=1M tmpfs %s' % ramdiskPath)
keyspace = NativeJson.Json('test-keyspace-name')
try:
self.harness = self.newHarness(cachePath=ramdiskPath)
self.view = self.harness.newView()
self.harness.subscribeToKeyspace(self.view, NativeJson.Json(keyspace))
with open('/dev/urandom') as randBytes:
for ix in range(1024):
self.harness.writeToKeyspace(
self.view,
keyspace,
NativeJson.Json('key%s' % ix),
NativeJson.Json(randBytes.read(1024))
)
except UserWarning:
# The view will be disconnected and throw this
pass
finally:
try:
assert False
except:
pass
self.harness.manager.shutdown()
self.harness = None
self.assertEqual(os.system('umount -f %s' % ramdiskPath), 0)
else:
logging.warning("Test can only be run as superuser")
def test_json_hashing(self):
j1 = Json.Json([1,2,3,4])
j2 = Json.Json([1,2,3,4])
j3 = Json.Json([1])
self.assertEqual(j1.hash, j2.hash)
self.assertTrue(j1.hash != j3.hash)
def test_compact_and_prune(self):
'''
Replicates an instance where a statefile and another logfile with a lower iteration number
can interact given a certain prune / compact cycle
'''
def createNewHarnessAndView():
self.harness = self.newHarness(maxLogFileSizeMb=.08)
self.view = self.harness.newView()
self.harness.subscribeToKeyspace(self.view, self.keyspaceName)
return self.harness, self.view
self.harness.teardown()
self.harness, self.view = createNewHarnessAndView()
totalSize = 0
self.harness.sendPingAndCompact()
# ensure we have a logfile started here...
while getLogFileLen(self.tempDir, self.keyspaceName) < 1024 * 100:
for ix in range(200):
self.harness.writeToKeyspace(self.view, self.keyspaceName, NativeJson.Json(self.randomString()), NativeJson.Json(self.randomString()))
self.harness.manager.check()
LogFilePruner.pruneLogFiles(self.tempDir)
self.harness.teardown()
self.harness, self.view = createNewHarnessAndView()
loopAgain = True
while loopAgain:
for ix in range(200):
self.harness.writeToKeyspace(self.view, self.keyspaceName, NativeJson.Json(self.randomString()), NativeJson.Json(self.randomString()))
loopAgain = getLogFileLen(self.tempDir, self.keyspaceName) < 1024 * 100
self.harness.manager.check()
def test_long_keyspace(self):
longKeyspace = ''.join(chr(x + ord('a')) for x in range(26)) * 9
self.harness.subscribeToKeyspace(self.view, NativeJson.Json(longKeyspace))
self.harness.writeToKeyspace(self.view,
NativeJson.Json(longKeyspace),
NativeJson.Json('key0'),
NativeJson.Json('test0'))
def test_corrupt_logfile(self):
self.harness.sendPingAndCompact()
# ensure we have a logfile started here...
while getLogFileLen(self.tempDir, self.keyspaceName) < 1000:
self.harness.writeToKeyspace(
self.view,
self.keyspaceName,
NativeJson.Json('key0'),
NativeJson.Json('test0')
)
self.harness.sendPingAndCompact()
self.harness.teardown()
self.fileToCorrupt = getCurLogFile(self.tempDir, self.keyspaceName)
with open(self.fileToCorrupt, 'a') as f:
size = os.stat(self.fileToCorrupt).st_size
f.truncate(int(size * .9))
self.harness = self.newHarness()
view = self.harness.newView()
self.harness.subscribeToKeyspace(view, self.keyspaceName)
while getLogFileLen(self.tempDir, self.keyspaceName) < 10000:
self.harness.writeToKeyspace(view, self.keyspaceName, NativeJson.Json('key0'), NativeJson.Json('test0'))
# this segfaults in the failing case, so there's nothing to assert
self.harness.sendPingAndCompact()
def test_flush(self):
harness = self.getHarness(inMemory=True)
try:
v1 = harness.newView()
space = SharedState.Keyspace("ComparisonKeyType",
NativeJson.Json("TestSpace"), 1)
rng = SharedState.KeyRange(space, 0, None, None, True, False)
v1.subscribe(rng)
keysToWrite = [str(x) for x in range(40)]
value = "value" * 100 * 1024
with SharedState.Transaction(v1):
self.assertRaises(UserWarning, v1.flush, True)
with SharedState.Transaction(v1):
for keyName in keysToWrite:
key = SharedState.Key(space, (NativeJson.Json(keyName), ))
v1[key] = NativeJson.Json(value)
v1.flush()
finally:
time.sleep(0.01)
harness.teardown()
def writeOwnRegistrationInformationIntoView_(self):
if self._registeredIpPortAndMachineId is None:
return
address, ports, machineIdAsString = self._registeredIpPortAndMachineId
logging.info(
"CumulusActiveMachine registering worker %s at %s:%s with id %s",
self.clientID,
address,
ports,
machineIdAsString
)
with self._lock:
self.activeMachineIds.add(machineIdAsString)
key = SharedState.Key(self.workerStatusKeyspace, (NativeJson.Json(str(self.clientID)),))
value = NativeJson.Json( (str(address), [str(x) for x in ports], machineIdAsString) )
doneWithCall = threading.Event()
self.asyncView.pushTransaction(key, value, lambda result: doneWithCall.set())
while not doneWithCall.wait(.1) and not self._stopFlag.is_set():
logging.debug('CumulusActiveMachines waiting on registerSelfAsActive')
def test_events_in_subscription(self):
asynView = None
try:
v1 = self.harness.newView()
self.subscribeToTestKeyspace(v1)
for ix in range(100):
key = SharedState.Key(self.testKeyspace, (NativeJson.Json('test%s' % ix),))
with SharedState.Transaction(v1):
v1[key] = NativeJson.Json('value1')
done = threading.Event()
numKeyEvents = [0]
def onKeyEvents(eventDict):
numKeyEvents[0] += len(eventDict)
def onSubscriptionLoaded(_):
done.set()
def onConnected(_):
deferred = asynView.subscribeToKeyspace(self.testKeyspace, 0, onKeyEvents)
deferred.addCallbacks(onSubscriptionLoaded, lambda error: None)
asynView = AsyncView.AsyncView(self.harness.viewFactory,
onConnectCallback=onConnected)
asynView.startService()
while not done.wait(.1):
pass
self.assertEqual(numKeyEvents[0], 100)
finally:
asynView.stopService()
def test_exception(self):
v = self.newView()
v.waitConnect()
v.subscribe(self.introRange)
key = SharedState.Key(self.introKeyspace, (NativeJson.Json("0"),NativeJson.Json(str(v.id))))
# this should raise because the view is not frozen
def shouldRaise():
v[key] = NativeJson.Json("some data")
self.assertRaises(UserWarning, shouldRaise)
def testWrite(passIx):
try:
testKeyspaces = [
SharedState.Keyspace("TakeHighestIdKeyType",
NativeJson.Json("TestSpace%s" % ix), 3) \
for ix in range(3)
]
harness = self.getHarness(inMemory=True,
cachePathOverride=tempDir)
v1 = harness.newView()
for space in testKeyspaces:
rng = SharedState.KeyRange(space, 0, None, None, True,
False)
v1.subscribe(rng)
for ix in range(9):
keyspaceIx = (ix + passIx) % len(testKeyspaces)
space = testKeyspaces[
keyspaceIx] #random.choice(testKeyspaces)
with SharedState.Transaction(v1):
key = SharedState.Key(
space,
(NativeJson.Json("key %s" % ix),
NativeJson.Json(''), NativeJson.Json('')))
value = JsonNative.Json(
'<value is %s in space %s on pass %s: %s>' %
(ix, keyspaceIx, passIx, uuid.uuid4().hex))
v1[key] = value
testValues[key] = value
v1.flush()
time.sleep(0.01)
harness.teardown()
harness = self.getHarness(inMemory=True,
cachePathOverride=tempDir)
v1 = harness.newView()
for space in testKeyspaces:
rng = SharedState.KeyRange(space, 0, None, None, True,
False)
v1.subscribe(rng)
for key, value in testValues.iteritems():
self.assertSharedStateKeyResolvesToValue(
key, v1, testValues[key])
except:
logging.warn("Exception: %s", traceback.format_exc())
raise
finally:
time.sleep(0.01)
harness.teardown()
def test_two_keyspaces(self):
try:
toWrite = Queue.Queue()
space1 = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace1"), 1)
space2 = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace2"), 1)
def callback1(items):
for key in items.iterkeys():
self.assertEqual(key.keyspace, NativeJson.Json("TestSpace1"))
toWrite.task_done()
def callback2(items):
for key in items.iterkeys():
self.assertEqual(key.keyspace, NativeJson.Json("TestSpace2"))
toWrite.task_done()
def onConnected(_):
simpleView.subscribeToKeyspace(space1, 0, callback1)
simpleView.subscribeToKeyspace(space2, 0, callback2)
simpleView = AsyncView.AsyncView(self.harness.viewFactory,
onConnected)
simpleView.startService()
v1 = self.harness.newView()
v1.subscribe(SharedState.KeyRange(space1, 0, None, None, True, False),
True)
v1.subscribe(SharedState.KeyRange(space2, 0, None, None, True, False),
True)
for x in range(100):
space = space1
if random.random() < .5:
space = space2
toWrite.put((str(x), space))
with SharedState.Transaction(v1):
v1[SharedState.Key(space, (NativeJson.Json(str(x)),))] = space.name
toWrite.join()
finally:
if simpleView:
simpleView.stopService()
self.harness.teardown()
def test_async_view(self):
try:
simpleView = None
toWrite = {
NativeJson.Json(str(ix)): NativeJson.Json('value-%s' % ix)
for ix in range(100)
}
written = {}
v1 = self.harness.newView()
self.subscribeToTestKeyspace(v1)
for key, value in toWrite.iteritems():
with SharedState.Transaction(v1):
key = SharedState.Key(self.testKeyspace, (key,))
v1[key] = value
written[key[0]] = value
v1.flush()
received = {}
newKeysWritten = threading.Condition()
def onConnected(_):
def callback(keysDict):
for key, value in keysDict.iteritems():
received[key[0]] = value
with newKeysWritten:
newKeysWritten.notify()
simpleView.subscribeToKeyspace(self.testKeyspace, 0, callback)
simpleView = AsyncView.AsyncView(self.harness.viewFactory,
onConnectCallback=onConnected)
simpleView.startService()
while set(received.values()) != set(written.values()):
with newKeysWritten:
newKeysWritten.wait()
self.assertEqual(set(received.values()), set(written.values()))
finally:
if simpleView:
simpleView.stopService()
self.harness.teardown()
def test_basic_listener(self):
v1 = self.harness.newView()
v2 = self.harness.newView()
while not v1.waitConnectTimeout(.1):
pass
while not v2.waitConnectTimeout(.1):
pass
l = SharedState.Listener(v1)
v1.subscribe(
SharedState.KeyRange(_testKeyspace, 0, None, None, True, True),
True)
v2.subscribe(
SharedState.KeyRange(_testKeyspace, 0, None, None, True, True),
True)
self.assertIsInstance(l.get()[0][1],
SharedState.SharedStateNative.KeyRange)
self.assertTrue(len(l.get(.1)) == 0)
self.assertTrue(len(l.getNonblock()) == 0)
testKey = SharedState.Key(_testKeyspace, (NativeJson.Json('test'), ))
with SharedState.Transaction(v2):
v2[testKey] = NativeJson.Json('test2')
v2.flush()
v1.flush()
updates = l.get(1)
self.assertTrue(len(updates) == 1)
self.assertEqual(updates[0][0], "KeyUpdates")
self.assertIsInstance(updates[0][1], list)
self.assertTrue(updates[0][1][0] == testKey)
with SharedState.Transaction(v2):
v2[testKey] = NativeJson.Json('newer test')
v2.flush()
v1.flush()
updates = l.get(1)
self.assertTrue(len(updates) == 1)
self.assertEqual(updates[0][0], "KeyUpdates")
self.assertIsInstance(updates[0][1], list)
self.assertTrue(updates[0][1][0] == testKey)
def keysLoaded(self, keyValueDict, isInitialLoad):
for key in keyValueDict:
if key.isArray() and len(key) == 2 and key[0] == NativeJson.Json(
"CGSS"):
try:
keyPath = JsonPickle.fromJson(key[1])
except:
import traceback
logging.warn(
"Bad key encountered in KeyspaceUpdater for keyspace %s: %s\n%s",
self.keyspace, key, traceback.format_exc())
#skip this key
continue
node = Subspace(keyspace=self.keyspace, keyPath=keyPath)
val = keyValueDict[key]
if val is None:
node.value_ = None
else:
try:
node.value_ = (JsonPickle.fromJson(val), )
except:
import traceback
traceback.print_exc()
logging.warn(
"ComputedGraphSharedState loaded bad node value at %s, %s: %s",
self.keyspace, key, repr(val))
# else:
# print "ignoring ", key
if isInitialLoad:
self.keyspace.markLoaded()
def __init__(self, viewFactory):
Stoppable.Stoppable.__init__(self)
self.viewFactory = viewFactory
self.activeMachineIds = set()
self._outgoingNotifications = Queue.Queue()
self._lock = threading.RLock()
self._lastDisconnectTime = None
self._lastReconnectTime = None
self._registeredIpPortAndMachineId = None
logging.debug("Connecting to shared state: %s", viewFactory)
self.listeners_ = set()
self.clientID = None
self.isConnected = False
self.isConnecting = False
self.disconnectedWhileConnecting = False
self._triggerUpdateAfterDisconnectThreads = []
self.clientIdToIpPortAndMachineIdAsString = {}
self.machineIdToClientId = {}
self.workerStatusKeyspace = SharedState.Keyspace(
"ComparisonKeyType", NativeJson.Json(
(('P', 'CumulusNodeStatus'), )), 1)
self.asyncView = None
self.eventLoopThread = ManagedThread.ManagedThread(
target=self.eventDispatchLoop)
def disabled_read_write_kv_maps_using_storage(self):
for ix in range(1000):
self.harness.writeToKeyspace(
self.view,
self.keyspaceName,
NativeJson.Json('key%s' % ix),
NativeJson.Json('test%s' % ix)
)
logging.warn("tempDir: %s, tempDir2: %s", self.tempDir, self.tempDir2)
allItemsFromViewInitial = self.harness.getAllItemsFromView(self.view, self.keyspaceName)
keyspaceList = [x for x in self.harness.manager.getAllKeyspaces()
if x != SharedState.getClientInfoKeyspace()]
self.assertTrue(len(allItemsFromViewInitial) == 1000)
self.view.flush()
self.harness.sendPingAndCompact()
self.harness.teardown()
try:
storageA = SharedStateNative.Storage.FileStorage(self.tempDir, 100, 10)
storageB = SharedStateNative.Storage.FileStorage(self.tempDir2, 100, 10)
self.assertTrue(len(keyspaceList) == 1)
for keyspace in keyspaceList:
storageForKeyspaceA = storageA.storageForKeyspace(keyspace, 0)
storageForKeyspaceB = storageB.storageForKeyspace(keyspace, 0)
data = storageForKeyspaceA.readKeyValueMap()
self.assertTrue(len(data) > 0)
storageForKeyspaceB.writeKeyValueMap(data)
self.harness = self.newHarness(self.tempDir2)
self.view = self.harness.newView()
self.harness.subscribeToKeyspace(self.view, self.keyspaceName)
allItemsFromViewFinal = self.harness.getAllItemsFromView(self.view, self.keyspaceName)
self.assertEqual(allItemsFromViewInitial, allItemsFromViewFinal)
finally:
if storageA:
storageA.shutdown()
if storageB:
storageB.shutdown()
def test_listener(self):
harness = self.getHarness(inMemory=True)
try:
v1 = harness.newView()
v2 = harness.newView()
l = SharedState.Listener(v2)
space = SharedState.Keyspace("ComparisonKeyType",
NativeJson.Json("TestSpace"), 1)
rng = SharedState.KeyRange(space, 0, None, None, True, False)
v1.subscribe(rng)
v2.subscribe(rng)
keysToWrite = [JsonNative.Json(str(x)) for x in range(40)]
def writer():
with SharedState.Transaction(v1):
for keyName in keysToWrite:
key = SharedState.Key(space, (keyName, ))
v1[key] = JsonNative.Json("value")
changedKeys = []
def reader():
while len(changedKeys) < len(keysToWrite):
updates = l.get()
for updateType, update in updates:
if updateType == "KeyUpdates":
for key in update:
changedKeys.append(key[0])
writerThread = threading.Thread(target=writer)
readerThread = threading.Thread(target=reader)
writerThread.start()
readerThread.start()
writerThread.join()
readerThread.join()
self.assertEqual(set(keysToWrite), set(changedKeys))
finally:
time.sleep(0.01)
harness.teardown()
def ensureSubscribed(self):
assert SharedStateSynchronizer.isConnected()
if not self.isSubscribed_:
logging.info("ComputedGraphSharedState subscribing to: %s", self)
SharedStateSynchronizer.getSynchronizer().addKeyspaceListener(
self.keyspaceName, KeyspaceUpdater(self), NativeJson.Json(()))
self.isSubscribed_ = True
def test_corrupt_statefile(self):
self.writeDataUntilNStatefilesExist(2, lambda : NativeJson.Json('key0'), lambda : NativeJson.Json('test0'))
fileToCorrupt = sorted(getStateFiles(self.tempDir, self.keyspaceName))[0]
with open(fileToCorrupt, 'a') as f:
size = os.stat(fileToCorrupt).st_size
f.truncate(int(size * .9))
self.harness.teardown()
self.harness = self.newHarness()
view = self.harness.newView()
self.harness.subscribeToKeyspace(view, self.keyspaceName)
while getLogFileLen(self.tempDir, self.keyspaceName) < 10000:
self.harness.writeToKeyspace(view, self.keyspaceName, NativeJson.Json('key0'), NativeJson.Json('test0'))
## this segfaults in the failing case, so there's nothing to assert
self.harness.sendPingAndCompact()
def connectedClientInfo(view):
"""given a view subscribed to the client info keyspace, computes all the connected
clients"""
clientInfoKeyspace = SharedStateNative.getClientInfoKeyspace()
k = view.nextKey(Key(clientInfoKeyspace, (NativeJson.lowestValue(), NativeJson.lowestValue())))
maxId = 0
tr = set()
while k is not None:
if k.keyspace != clientInfoKeyspace.name:
return tr, maxId
if view[k].value() != NativeJson.Json('disconnected'):
tr.add(k[1])
maxId = max(maxId, view[k].id())
k = view.nextKey(k)
return tr, maxId
def setUp(self):
self.tempDir = tempfile.mkdtemp()
self.tempDir2 = tempfile.mkdtemp()
self.keyspaceName = NativeJson.Json('test-keyspace')
self.harness = self.newHarness(self.tempDir)
self.view = self.harness.newView()
self.harness.subscribeToKeyspace(self.view, self.keyspaceName)
def createLogFileDir(self):
self.tempdir = tempfile.mkdtemp()
self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType",
NativeJson.Json("test-space"), 1)
self.keyrange = SharedState.KeyRange(self.keyspace, 0, None, None,
True, False)
self.logFileDir = StorageNative.LogFileDirectory(
self.tempdir, self.keyspace, self.keyrange)
self.baseDir = os.path.split(self.logFileDir.getCurrentLogPath())[0]
def iterKeys(view, keyspace):
assert view.isFrozen
key = view.nextKey(
Key(keyspace,
tuple(NativeJson.lowestValue()
for x in range(keyspace.dimension))))
while key is not None:
yield key
key = view.nextKey(key)
def keyspaceIterItems(self, keyspace):
self.assertSingleThreaded()
startKey = tuple([NativeJson.lowestValue() for x in range(keyspace.dimension)])
with SharedState.Transaction(self._view):
k = self._view.nextKey(SharedState.Key(keyspace, startKey))
while k is not None and str(k.keyspace) == str(keyspace.name):
if self._view[k] is not None:
yield k, self._view[k].value()
k = self._view.nextKey(k)
def keyspaceIterItems(self, keyspace):
self.assertSingleThreaded()
startKey = tuple(
[NativeJson.lowestValue() for x in range(keyspace.dimension)])
with SharedState.Transaction(self._view):
k = self._view.nextKey(SharedState.Key(keyspace, startKey))
while k is not None and str(k.keyspace) == str(keyspace.name):
if self._view[k] is not None:
yield k, self._view[k].value()
k = self._view.nextKey(k)
def connectedClientInfo(view):
"""given a view subscribed to the client info keyspace, computes all the connected
clients"""
clientInfoKeyspace = SharedStateNative.getClientInfoKeyspace()
k = view.nextKey(
Key(clientInfoKeyspace,
(NativeJson.lowestValue(), NativeJson.lowestValue())))
maxId = 0
tr = set()
while k is not None:
if k.keyspace != clientInfoKeyspace.name:
return tr, maxId
if view[k].value() != NativeJson.Json('disconnected'):
tr.add(k[1])
maxId = max(maxId, view[k].id())
k = view.nextKey(k)
return tr, maxId
def addKeyspaceListener(self,
keyspaceName,
keyspaceListener,
keyPrefix,
blockUntilLoaded=False,
assertAlreadyLoaded=False):
"""Add a keyspace listener to the synchronizer."""
assert isinstance(keyPrefix, NativeJson.Json)
assert keyPrefix.isArray()
assert isinstance(keyspaceName, NativeJson.Json)
lookupKey = (keyspaceName, keyPrefix)
if keyspaceName not in self.keyspaceNameToPrefixes_:
self.keyspaceNameToPrefixes_[keyspaceName] = set()
for otherPrefix in self.keyspaceNameToPrefixes_[keyspaceName]:
assert not (
otherPrefix.arrayStartswith(keyPrefix)
and otherPrefix != keyPrefix
), ("Can't subscribe to %s:%s since we're already subscribed to %s:%s"
% (keyspaceName, keyPrefix, keyspaceName, otherPrefix))
self.keyspaceNameToPrefixes_[keyspaceName].add(keyPrefix)
if lookupKey not in self.keyspaceListenersByKeyspaceAndPrefix_:
if assertAlreadyLoaded:
assert False, "Keyspace %s was not already loaded." % keyspaceName
self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey] = set()
self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey].add(
keyspaceListener)
self.valuesByKeyspaceAndPrefix_[lookupKey] = {}
logging.info("shared state subscribing to %s:%s", keyspaceName,
keyPrefix)
keyspace = SharedState.Keyspace("TakeHighestIdKeyType",
keyspaceName, 1)
self.view_.subscribe(
SharedState.KeyRange(keyspace, 0, keyPrefix,
keyPrefix + NativeJson.Json([None]), True,
False), blockUntilLoaded)
if blockUntilLoaded:
self.update()
self.ensureMarkedLoaded(lookupKey)
else:
self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey].add(
keyspaceListener)
keyspaceListener.keysLoaded(
self.valuesByKeyspaceAndPrefix_[lookupKey], True)
def test_add_transaction(self):
try:
toWrite = {
NativeJson.Json(str(x)): NativeJson.Json('value-%s' % x)
for x in range(50)
}
simpleView = None
def onConnect(_):
def callback(value):
for key, value in toWrite.iteritems():
key = SharedState.Key(self.testKeyspace, (key,))
simpleView.pushTransaction(key, value)
deferred = simpleView.subscribeToKeyspace(self.testKeyspace, 0, callback)
deferred.addCallbacks(callback, lambda value: None)
simpleView = AsyncView.AsyncView(self.harness.viewFactory,
onConnectCallback=onConnect)
simpleView.startService()
v1 = self.harness.newView()
listener = SharedState.Listener(v1)
self.subscribeToTestKeyspace(v1)
toCheck = dict(toWrite)
while len(toCheck):
updates = listener.get()
for updateType, update in updates:
if updateType == "KeyUpdates":
for key in update:
if key[0] in toCheck:
with SharedState.Transaction(v1):
self.assertEqual(toCheck[key[0]], v1[key].value())
del toCheck[key[0]]
finally:
if simpleView:
simpleView.stopService()
self.harness.teardown()
def test_keyspace_iteration(self):
try:
toWrite = {
NativeJson.Json(str(x)): NativeJson.Json('value-%s' % x)
for x in range(50)
}
simpleView = None
v1 = self.harness.newView()
self.subscribeToTestKeyspace(v1)
for key, value in toWrite.iteritems():
key = SharedState.Key(self.testKeyspace, (key,))
with SharedState.Transaction(v1):
v1[key] = value
v1.flush()
def onConnect(_):
def checkIteration(value):
for key, value in simpleView.keyspaceItems(self.testKeyspace):
assert key[0] in toWrite
self.assertEqual(toWrite[key[0]], value)
deferred = simpleView.subscribeToKeyspace(self.testKeyspace,
0,
lambda keys: None)
deferred.addCallbacks(checkIteration, lambda value: None)
simpleView = AsyncView.AsyncView(self.harness.viewFactory,
onConnectCallback=onConnect)
simpleView.startService()
finally:
if simpleView:
simpleView.stopService()
self.harness.teardown()
def test(threadIx):
for subPassIx in range(subPasses):
logging.info("Thread %s starting pass %s", threadIx, subPassIx)
testKeyspace = SharedState.Keyspace(
"TakeHighestIdKeyType", NativeJson.Json("TestSpace"), 1)
view = viewFactory.createView()
rng = SharedState.KeyRange(testKeyspace, 0, None, None, True,
False)
view.subscribe(rng)
for ix in range(keysToWrite):
with SharedState.Transaction(view):
ix = random.randint(0, keyspaceSize)
key = SharedState.Key(
testKeyspace, (NativeJson.Json("key %s" % ix), ))
value = uuid.uuid4().hex
view[key] = NativeJson.Json(value)
worked[threadIx] = True
def test_file_management(self):
tempDir = tempfile.mkdtemp()
curOpenFiles = len(os.listdir('/proc/%s/fd' % os.getpid()))
OPEN_FILE_LIMIT = 200
if curOpenFiles >= OPEN_FILE_LIMIT:
os.system("ls -alh /proc/%s/fd" % os.getpid())
self.assertTrue(curOpenFiles < OPEN_FILE_LIMIT,
"Too many open files: %s" % curOpenFiles)
soft, hard = resource.getrlimit(resource.RLIMIT_OFILE)
harness = self.getHarness(inMemory=True,
cachePathOverride=tempDir,
maxOpenFiles=15)
try:
v1 = harness.newView()
resource.setrlimit(resource.RLIMIT_OFILE,
(curOpenFiles + 30, hard))
for ix in range(128):
space = SharedState.Keyspace(
"TakeHighestIdKeyType",
NativeJson.Json("TestSpace%s" % ix), 1)
rng = SharedState.KeyRange(space, 0, None, None, True, False)
v1.subscribe(rng)
key = SharedState.Key(space, (NativeJson.Json('key%s' % ix), ))
with SharedState.Transaction(v1):
v1[key] = NativeJson.Json('value %s' % ix)
finally:
time.sleep(0.01)
harness.teardown()
resource.setrlimit(resource.RLIMIT_OFILE, (soft, hard))
try:
shutil.rmtree(tempDir)
except:
pass
def test_require_subscription_manager(self):
tempDir = tempfile.mkdtemp()
try:
testKeyspace = SharedState.Keyspace("TakeHighestIdKeyType",
NativeJson.Json("TestSpace"),
1)
harness = self.getHarness(inMemory=True, cachePathOverride=tempDir)
v1 = harness.newView()
v1.subscribe(
SharedState.KeyRange(testKeyspace, 0, None, None, True, False))
for ix in range(1000):
with SharedState.Transaction(v1):
key = SharedState.Key(testKeyspace,
(NativeJson.Json("key"), ))
v1[key] = NativeJson.Json(str(ix))
time.sleep(2)
v2 = harness.newView()
v2.setMustSubscribe(False)
with SharedState.Transaction(v2):
key = SharedState.Key(testKeyspace, (NativeJson.Json("key"), ))
value = 'value is' + uuid.uuid4().hex
v2[key] = NativeJson.Json(value)
time.sleep(2)
def toTry():
with SharedState.Transaction(v2):
key = SharedState.Key(testKeyspace,
(NativeJson.Json("key"), ))
value = 'value is' + uuid.uuid4().hex
v2[key] = NativeJson.Json(value)
self.assertRaises(UserWarning, toTry)
finally:
time.sleep(0.01)
harness.teardown()
shutil.rmtree(tempDir)
def iterKeys(view, keyspace):
assert view.isFrozen
key = view.nextKey(Key(keyspace, tuple(NativeJson.lowestValue() for x in range(keyspace.dimension))))
while key is not None:
yield key
key = view.nextKey(key)