def test_events_in_subscription(self):
asynView = None
try:
v1 = self.harness.newView()
self.subscribeToTestKeyspace(v1)
for ix in range(100):
key = SharedState.Key(self.testKeyspace, (NativeJson.Json('test%s' % ix),))
with SharedState.Transaction(v1):
v1[key] = NativeJson.Json('value1')
done = threading.Event()
numKeyEvents = [0]
def onKeyEvents(eventDict):
numKeyEvents[0] += len(eventDict)
def onSubscriptionLoaded(_):
done.set()
def onConnected(_):
deferred = asynView.subscribeToKeyspace(self.testKeyspace, 0, onKeyEvents)
deferred.addCallbacks(onSubscriptionLoaded, lambda error: None)
asynView = AsyncView.AsyncView(self.harness.viewFactory,
onConnectCallback=onConnected)
asynView.startService()
while not done.wait(.1):
pass
self.assertEqual(numKeyEvents[0], 100)
finally:
asynView.stopService()
def runInTransaction(self, keyPathElement, toRun, waitForKeyToLoad = True):
"""
Runs a specified function in a shared state transaction.
toRun is a function that takes the following argumenst:
self - a self reference
key - the shared state key used by the test
view - the a shared state view subscribed to the key range
that contains key.
waitForKeyToLoad - wait for SharedState to actually load the
key
"""
view = self.sharedState.newView()
view.subscribe(self.keyRange)
passes = 0
while passes < 100:
with SharedState.Transaction(view):
key = SharedState.Key(self.keyspace, (Node.keyPathToKeyName((keyPathElement,)),))
if not waitForKeyToLoad or view[key] is not None:
toRun(self, key, view)
return
passes += 1
time.sleep(.1)
assert False, "Test timed out"
def test_flush(self):
harness = self.getHarness(inMemory=True)
try:
v1 = harness.newView()
space = SharedState.Keyspace("ComparisonKeyType",
NativeJson.Json("TestSpace"), 1)
rng = SharedState.KeyRange(space, 0, None, None, True, False)
v1.subscribe(rng)
keysToWrite = [str(x) for x in range(40)]
value = "value" * 100 * 1024
with SharedState.Transaction(v1):
self.assertRaises(UserWarning, v1.flush, True)
with SharedState.Transaction(v1):
for keyName in keysToWrite:
key = SharedState.Key(space, (NativeJson.Json(keyName), ))
v1[key] = NativeJson.Json(value)
v1.flush()
finally:
time.sleep(0.01)
harness.teardown()
def writeOwnRegistrationInformationIntoView_(self):
if self._registeredIpPortAndMachineId is None:
return
address, ports, machineIdAsString = self._registeredIpPortAndMachineId
logging.info(
"CumulusActiveMachine registering worker %s at %s:%s with id %s",
self.clientID,
address,
ports,
machineIdAsString
)
with self._lock:
self.activeMachineIds.add(machineIdAsString)
key = SharedState.Key(self.workerStatusKeyspace, (NativeJson.Json(str(self.clientID)),))
value = NativeJson.Json( (str(address), [str(x) for x in ports], machineIdAsString) )
doneWithCall = threading.Event()
self.asyncView.pushTransaction(key, value, lambda result: doneWithCall.set())
while not doneWithCall.wait(.1) and not self._stopFlag.is_set():
logging.debug('CumulusActiveMachines waiting on registerSelfAsActive')
def test_exception(self):
v = self.newView()
v.waitConnect()
v.subscribe(self.introRange)
key = SharedState.Key(self.introKeyspace, (NativeJson.Json("0"),NativeJson.Json(str(v.id))))
# this should raise because the view is not frozen
def shouldRaise():
v[key] = NativeJson.Json("some data")
self.assertRaises(UserWarning, shouldRaise)
def keyspaceIterItems(self, keyspace):
self.assertSingleThreaded()
startKey = tuple(
[NativeJson.lowestValue() for x in range(keyspace.dimension)])
with SharedState.Transaction(self._view):
k = self._view.nextKey(SharedState.Key(keyspace, startKey))
while k is not None and str(k.keyspace) == str(keyspace.name):
if self._view[k] is not None:
yield k, self._view[k].value()
k = self._view.nextKey(k)
def testWrite(passIx):
try:
testKeyspaces = [
SharedState.Keyspace("TakeHighestIdKeyType",
NativeJson.Json("TestSpace%s" % ix), 3) \
for ix in range(3)
]
harness = self.getHarness(inMemory=True,
cachePathOverride=tempDir)
v1 = harness.newView()
for space in testKeyspaces:
rng = SharedState.KeyRange(space, 0, None, None, True,
False)
v1.subscribe(rng)
for ix in range(9):
keyspaceIx = (ix + passIx) % len(testKeyspaces)
space = testKeyspaces[
keyspaceIx] #random.choice(testKeyspaces)
with SharedState.Transaction(v1):
key = SharedState.Key(
space,
(NativeJson.Json("key %s" % ix),
NativeJson.Json(''), NativeJson.Json('')))
value = JsonNative.Json(
'<value is %s in space %s on pass %s: %s>' %
(ix, keyspaceIx, passIx, uuid.uuid4().hex))
v1[key] = value
testValues[key] = value
v1.flush()
time.sleep(0.01)
harness.teardown()
harness = self.getHarness(inMemory=True,
cachePathOverride=tempDir)
v1 = harness.newView()
for space in testKeyspaces:
rng = SharedState.KeyRange(space, 0, None, None, True,
False)
v1.subscribe(rng)
for key, value in testValues.iteritems():
self.assertSharedStateKeyResolvesToValue(
key, v1, testValues[key])
except:
logging.warn("Exception: %s", traceback.format_exc())
raise
finally:
time.sleep(0.01)
harness.teardown()
def test_require_subscription_manager(self):
tempDir = tempfile.mkdtemp()
try:
testKeyspace = SharedState.Keyspace("TakeHighestIdKeyType",
NativeJson.Json("TestSpace"),
1)
harness = self.getHarness(inMemory=True, cachePathOverride=tempDir)
v1 = harness.newView()
v1.subscribe(
SharedState.KeyRange(testKeyspace, 0, None, None, True, False))
for ix in range(1000):
with SharedState.Transaction(v1):
key = SharedState.Key(testKeyspace,
(NativeJson.Json("key"), ))
v1[key] = NativeJson.Json(str(ix))
time.sleep(2)
v2 = harness.newView()
v2.setMustSubscribe(False)
with SharedState.Transaction(v2):
key = SharedState.Key(testKeyspace, (NativeJson.Json("key"), ))
value = 'value is' + uuid.uuid4().hex
v2[key] = NativeJson.Json(value)
time.sleep(2)
def toTry():
with SharedState.Transaction(v2):
key = SharedState.Key(testKeyspace,
(NativeJson.Json("key"), ))
value = 'value is' + uuid.uuid4().hex
v2[key] = NativeJson.Json(value)
self.assertRaises(UserWarning, toTry)
finally:
time.sleep(0.01)
harness.teardown()
shutil.rmtree(tempDir)
def test_two_keyspaces(self):
try:
toWrite = Queue.Queue()
space1 = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace1"), 1)
space2 = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace2"), 1)
def callback1(items):
for key in items.iterkeys():
self.assertEqual(key.keyspace, NativeJson.Json("TestSpace1"))
toWrite.task_done()
def callback2(items):
for key in items.iterkeys():
self.assertEqual(key.keyspace, NativeJson.Json("TestSpace2"))
toWrite.task_done()
def onConnected(_):
simpleView.subscribeToKeyspace(space1, 0, callback1)
simpleView.subscribeToKeyspace(space2, 0, callback2)
simpleView = AsyncView.AsyncView(self.harness.viewFactory,
onConnected)
simpleView.startService()
v1 = self.harness.newView()
v1.subscribe(SharedState.KeyRange(space1, 0, None, None, True, False),
True)
v1.subscribe(SharedState.KeyRange(space2, 0, None, None, True, False),
True)
for x in range(100):
space = space1
if random.random() < .5:
space = space2
toWrite.put((str(x), space))
with SharedState.Transaction(v1):
v1[SharedState.Key(space, (NativeJson.Json(str(x)),))] = space.name
toWrite.join()
finally:
if simpleView:
simpleView.stopService()
self.harness.teardown()
def test_async_view(self):
try:
simpleView = None
toWrite = {
NativeJson.Json(str(ix)): NativeJson.Json('value-%s' % ix)
for ix in range(100)
}
written = {}
v1 = self.harness.newView()
self.subscribeToTestKeyspace(v1)
for key, value in toWrite.iteritems():
with SharedState.Transaction(v1):
key = SharedState.Key(self.testKeyspace, (key,))
v1[key] = value
written[key[0]] = value
v1.flush()
received = {}
newKeysWritten = threading.Condition()
def onConnected(_):
def callback(keysDict):
for key, value in keysDict.iteritems():
received[key[0]] = value
with newKeysWritten:
newKeysWritten.notify()
simpleView.subscribeToKeyspace(self.testKeyspace, 0, callback)
simpleView = AsyncView.AsyncView(self.harness.viewFactory,
onConnectCallback=onConnected)
simpleView.startService()
while set(received.values()) != set(written.values()):
with newKeysWritten:
newKeysWritten.wait()
self.assertEqual(set(received.values()), set(written.values()))
finally:
if simpleView:
simpleView.stopService()
self.harness.teardown()
def test_basic_listener(self):
v1 = self.harness.newView()
v2 = self.harness.newView()
while not v1.waitConnectTimeout(.1):
pass
while not v2.waitConnectTimeout(.1):
pass
l = SharedState.Listener(v1)
v1.subscribe(
SharedState.KeyRange(_testKeyspace, 0, None, None, True, True),
True)
v2.subscribe(
SharedState.KeyRange(_testKeyspace, 0, None, None, True, True),
True)
self.assertIsInstance(l.get()[0][1],
SharedState.SharedStateNative.KeyRange)
self.assertTrue(len(l.get(.1)) == 0)
self.assertTrue(len(l.getNonblock()) == 0)
testKey = SharedState.Key(_testKeyspace, (NativeJson.Json('test'), ))
with SharedState.Transaction(v2):
v2[testKey] = NativeJson.Json('test2')
v2.flush()
v1.flush()
updates = l.get(1)
self.assertTrue(len(updates) == 1)
self.assertEqual(updates[0][0], "KeyUpdates")
self.assertIsInstance(updates[0][1], list)
self.assertTrue(updates[0][1][0] == testKey)
with SharedState.Transaction(v2):
v2[testKey] = NativeJson.Json('newer test')
v2.flush()
v1.flush()
updates = l.get(1)
self.assertTrue(len(updates) == 1)
self.assertEqual(updates[0][0], "KeyUpdates")
self.assertIsInstance(updates[0][1], list)
self.assertTrue(updates[0][1][0] == testKey)
def test_keyspace_iteration(self):
try:
toWrite = {
NativeJson.Json(str(x)): NativeJson.Json('value-%s' % x)
for x in range(50)
}
simpleView = None
v1 = self.harness.newView()
self.subscribeToTestKeyspace(v1)
for key, value in toWrite.iteritems():
key = SharedState.Key(self.testKeyspace, (key,))
with SharedState.Transaction(v1):
v1[key] = value
v1.flush()
def onConnect(_):
def checkIteration(value):
for key, value in simpleView.keyspaceItems(self.testKeyspace):
assert key[0] in toWrite
self.assertEqual(toWrite[key[0]], value)
deferred = simpleView.subscribeToKeyspace(self.testKeyspace,
0,
lambda keys: None)
deferred.addCallbacks(checkIteration, lambda value: None)
simpleView = AsyncView.AsyncView(self.harness.viewFactory,
onConnectCallback=onConnect)
simpleView.startService()
finally:
if simpleView:
simpleView.stopService()
self.harness.teardown()
def test(threadIx):
for subPassIx in range(subPasses):
logging.info("Thread %s starting pass %s", threadIx, subPassIx)
testKeyspace = SharedState.Keyspace(
"TakeHighestIdKeyType", NativeJson.Json("TestSpace"), 1)
view = viewFactory.createView()
rng = SharedState.KeyRange(testKeyspace, 0, None, None, True,
False)
view.subscribe(rng)
for ix in range(keysToWrite):
with SharedState.Transaction(view):
ix = random.randint(0, keyspaceSize)
key = SharedState.Key(
testKeyspace, (NativeJson.Json("key %s" % ix), ))
value = uuid.uuid4().hex
view[key] = NativeJson.Json(value)
worked[threadIx] = True
def test_file_management(self):
tempDir = tempfile.mkdtemp()
curOpenFiles = len(os.listdir('/proc/%s/fd' % os.getpid()))
OPEN_FILE_LIMIT = 200
if curOpenFiles >= OPEN_FILE_LIMIT:
os.system("ls -alh /proc/%s/fd" % os.getpid())
self.assertTrue(curOpenFiles < OPEN_FILE_LIMIT,
"Too many open files: %s" % curOpenFiles)
soft, hard = resource.getrlimit(resource.RLIMIT_OFILE)
harness = self.getHarness(inMemory=True,
cachePathOverride=tempDir,
maxOpenFiles=15)
try:
v1 = harness.newView()
resource.setrlimit(resource.RLIMIT_OFILE,
(curOpenFiles + 30, hard))
for ix in range(128):
space = SharedState.Keyspace(
"TakeHighestIdKeyType",
NativeJson.Json("TestSpace%s" % ix), 1)
rng = SharedState.KeyRange(space, 0, None, None, True, False)
v1.subscribe(rng)
key = SharedState.Key(space, (NativeJson.Json('key%s' % ix), ))
with SharedState.Transaction(v1):
v1[key] = NativeJson.Json('value %s' % ix)
finally:
time.sleep(0.01)
harness.teardown()
resource.setrlimit(resource.RLIMIT_OFILE, (soft, hard))
try:
shutil.rmtree(tempDir)
except:
pass
def toTry():
with SharedState.Transaction(v1):
key = SharedState.Key(testKeyspace, ("key", ))
v1[key] = NativeJson.Json('this is a test value')
def toTry():
with SharedState.Transaction(v2):
key = SharedState.Key(testKeyspace,
(NativeJson.Json("key"), ))
value = 'value is' + uuid.uuid4().hex
v2[key] = NativeJson.Json(value)
def producePartialEvents(view, keyspaces, keyBase, numWrites, numKeys, valueLen):
with open('/dev/urandom') as randBytes:
for ix in xrange(numWrites):
key = SharedState.Key(keyspaces[ix % len(keyspaces)],
(json('%s-%s' % (keyBase, ix % numKeys)),))
yield view.createKeyUpdate(key, json(randBytes.read(valueLen)))
def writeValue_(self, keyspaceName, keyName, value):
self.view_[SharedState.Key(
SharedState.Keyspace("TakeHighestIdKeyType", keyspaceName, 1),
(keyName, ))] = value
def writer():
with SharedState.Transaction(v1):
for keyName in keysToWrite:
key = SharedState.Key(space, (keyName, ))
v1[key] = JsonNative.Json("value")
def getKey(self, name):
return SharedState.Key(self.keyspace, (json(name), ))
def writeToKeyspace(self, view, spacename, key, val):
keyspace = SharedState.Keyspace("TakeHighestIdKeyType", spacename, 1)
with SharedState.Transaction(view):
view[SharedState.Key(keyspace, (key, ))] = val
def callback(value):
for key, value in toWrite.iteritems():
key = SharedState.Key(self.testKeyspace, (key,))
simpleView.pushTransaction(key, value)