def test_two_keyspaces(self):
try:
toWrite = Queue.Queue()
space1 = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace1"), 1)
space2 = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace2"), 1)
def callback1(items):
for key in items.iterkeys():
self.assertEqual(key.keyspace, NativeJson.Json("TestSpace1"))
toWrite.task_done()
def callback2(items):
for key in items.iterkeys():
self.assertEqual(key.keyspace, NativeJson.Json("TestSpace2"))
toWrite.task_done()
def onConnected(_):
simpleView.subscribeToKeyspace(space1, 0, callback1)
simpleView.subscribeToKeyspace(space2, 0, callback2)
simpleView = AsyncView.AsyncView(self.harness.viewFactory,
onConnected)
simpleView.startService()
v1 = self.harness.newView()
v1.subscribe(SharedState.KeyRange(space1, 0, None, None, True, False),
True)
v1.subscribe(SharedState.KeyRange(space2, 0, None, None, True, False),
True)
for x in range(100):
space = space1
if random.random() < .5:
space = space2
toWrite.put((str(x), space))
with SharedState.Transaction(v1):
v1[SharedState.Key(space, (NativeJson.Json(str(x)),))] = space.name
toWrite.join()
finally:
if simpleView:
simpleView.stopService()
self.harness.teardown()
def __init__(self, viewFactory):
Stoppable.Stoppable.__init__(self)
self.viewFactory = viewFactory
self.activeMachineIds = set()
self._outgoingNotifications = Queue.Queue()
self._lock = threading.RLock()
self._lastDisconnectTime = None
self._lastReconnectTime = None
self._registeredIpPortAndMachineId = None
logging.debug("Connecting to shared state: %s", viewFactory)
self.listeners_ = set()
self.clientID = None
self.isConnected = False
self.isConnecting = False
self.disconnectedWhileConnecting = False
self._triggerUpdateAfterDisconnectThreads = []
self.clientIdToIpPortAndMachineIdAsString = {}
self.machineIdToClientId = {}
self.workerStatusKeyspace = SharedState.Keyspace(
"ComparisonKeyType", NativeJson.Json(
(('P', 'CumulusNodeStatus'), )), 1)
self.asyncView = None
self.eventLoopThread = ManagedThread.ManagedThread(
target=self.eventDispatchLoop)
def test_flush(self):
harness = self.getHarness(inMemory=True)
try:
v1 = harness.newView()
space = SharedState.Keyspace("ComparisonKeyType",
NativeJson.Json("TestSpace"), 1)
rng = SharedState.KeyRange(space, 0, None, None, True, False)
v1.subscribe(rng)
keysToWrite = [str(x) for x in range(40)]
value = "value" * 100 * 1024
with SharedState.Transaction(v1):
self.assertRaises(UserWarning, v1.flush, True)
with SharedState.Transaction(v1):
for keyName in keysToWrite:
key = SharedState.Key(space, (NativeJson.Json(keyName), ))
v1[key] = NativeJson.Json(value)
v1.flush()
finally:
time.sleep(0.01)
harness.teardown()
def subscribeToKeyspace(view, keyspaceNames, keyspaceType):
keyspaces = [SharedState.Keyspace(keyspaceType, json(keyspaceName), 1)
for keyspaceName in keyspaceNames]
keyranges = [SharedState.KeyRange(keyspace, 0, None, None, True, True)
for keyspace in keyspaces]
view.subscribe(keyranges)
return keyspaces, keyranges
def createLogFileDir(self):
self.tempdir = tempfile.mkdtemp()
self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType",
NativeJson.Json("test-space"), 1)
self.keyrange = SharedState.KeyRange(self.keyspace, 0, None, None,
True, False)
self.logFileDir = StorageNative.LogFileDirectory(
self.tempdir, self.keyspace, self.keyrange)
self.baseDir = os.path.split(self.logFileDir.getCurrentLogPath())[0]
def setUp(self):
self.sharedState = SharedStateTestHarness(inMemory = True)
self.keyspacePath = (0, 1)
self.keyspaceName = Node.keyspacePathToKeyspaceName(self.keyspacePath)
self.keyName = 'key'
self.keyValue = 'value'
self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType", self.keyspaceName, 1)
self.keyRange = SharedState.KeyRange(self.keyspace, 0, None, None, True, False)
def addKeyspaceListener(self,
keyspaceName,
keyspaceListener,
keyPrefix,
blockUntilLoaded=False,
assertAlreadyLoaded=False):
"""Add a keyspace listener to the synchronizer."""
assert isinstance(keyPrefix, NativeJson.Json)
assert keyPrefix.isArray()
assert isinstance(keyspaceName, NativeJson.Json)
lookupKey = (keyspaceName, keyPrefix)
if keyspaceName not in self.keyspaceNameToPrefixes_:
self.keyspaceNameToPrefixes_[keyspaceName] = set()
for otherPrefix in self.keyspaceNameToPrefixes_[keyspaceName]:
assert not (
otherPrefix.arrayStartswith(keyPrefix)
and otherPrefix != keyPrefix
), ("Can't subscribe to %s:%s since we're already subscribed to %s:%s"
% (keyspaceName, keyPrefix, keyspaceName, otherPrefix))
self.keyspaceNameToPrefixes_[keyspaceName].add(keyPrefix)
if lookupKey not in self.keyspaceListenersByKeyspaceAndPrefix_:
if assertAlreadyLoaded:
assert False, "Keyspace %s was not already loaded." % keyspaceName
self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey] = set()
self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey].add(
keyspaceListener)
self.valuesByKeyspaceAndPrefix_[lookupKey] = {}
logging.info("shared state subscribing to %s:%s", keyspaceName,
keyPrefix)
keyspace = SharedState.Keyspace("TakeHighestIdKeyType",
keyspaceName, 1)
self.view_.subscribe(
SharedState.KeyRange(keyspace, 0, keyPrefix,
keyPrefix + NativeJson.Json([None]), True,
False), blockUntilLoaded)
if blockUntilLoaded:
self.update()
self.ensureMarkedLoaded(lookupKey)
else:
self.keyspaceListenersByKeyspaceAndPrefix_[lookupKey].add(
keyspaceListener)
keyspaceListener.keysLoaded(
self.valuesByKeyspaceAndPrefix_[lookupKey], True)
def testWrite(passIx):
try:
testKeyspaces = [
SharedState.Keyspace("TakeHighestIdKeyType",
NativeJson.Json("TestSpace%s" % ix), 3) \
for ix in range(3)
]
harness = self.getHarness(inMemory=True,
cachePathOverride=tempDir)
v1 = harness.newView()
for space in testKeyspaces:
rng = SharedState.KeyRange(space, 0, None, None, True,
False)
v1.subscribe(rng)
for ix in range(9):
keyspaceIx = (ix + passIx) % len(testKeyspaces)
space = testKeyspaces[
keyspaceIx] #random.choice(testKeyspaces)
with SharedState.Transaction(v1):
key = SharedState.Key(
space,
(NativeJson.Json("key %s" % ix),
NativeJson.Json(''), NativeJson.Json('')))
value = JsonNative.Json(
'<value is %s in space %s on pass %s: %s>' %
(ix, keyspaceIx, passIx, uuid.uuid4().hex))
v1[key] = value
testValues[key] = value
v1.flush()
time.sleep(0.01)
harness.teardown()
harness = self.getHarness(inMemory=True,
cachePathOverride=tempDir)
v1 = harness.newView()
for space in testKeyspaces:
rng = SharedState.KeyRange(space, 0, None, None, True,
False)
v1.subscribe(rng)
for key, value in testValues.iteritems():
self.assertSharedStateKeyResolvesToValue(
key, v1, testValues[key])
except:
logging.warn("Exception: %s", traceback.format_exc())
raise
finally:
time.sleep(0.01)
harness.teardown()
def test_simple_manager(self):
view = createViewWithNoChannel()
before = TCMalloc.getBytesUsed()
keyspace = SharedState.Keyspace("TakeHighestIdKeyType", json('test'), 1)
cache = SharedStateNative.KeyspaceManager(0, 1, 0x7fffffff, 0x7fffffff, None)
for event in producePartialEvents(view, [keyspace], 'test', 1024 * 32, 1, 8):
cache.addEvent(event)
view = None
gc.collect()
bytesUsed = TCMalloc.getBytesUsed() - before
self.assertTrue(bytesUsed < 1024 * 128)
def compressOrphandLogFiles(self):
for keyspaceDir in os.listdir(self.cachePath):
keyspaceType, dimensions, keyspaceName = keyspaceDir.split('::')
dimensions = int(dimensions)
keyspace = SharedState.Keyspace(
keyspaceType, NativeJson.Json.parse(keyspaceName), dimensions)
for i in range(dimensions):
logging.info("Compressing keyspace: %s", keyspaceName)
keyspaceStorage = self.keyspaceManager.storage.storageForKeyspace(
keyspace, i)
keyspaceStorage.compress()
def __init__(self, ChannelType, numKeys, keySize):
self.view = createViewWithNoChannel()
self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType", json('test'), 1)
self.viewChannel, self.managerChannel = ChannelType(callbackScheduler)
self.eventsRead = 0
self.numKeys = numKeys
self.keySize = keySize
self.iterator = enumerate(producePartialEvents(
self.view,
[self.keyspace],
'test',
self.numKeys,
1,
self.keySize))
def test_keyspace_cache(self):
numKeys = 1024 * 256
before = TCMalloc.getBytesUsed()
view = createViewWithNoChannel()
keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1)
keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True)
cache = SharedStateNative.KeyspaceCache(keyrange, None)
for event in producePartialEvents(view, [keyspace], 'test', numKeys, 1, 8):
cache.addEvent(event)
cache.newMinimumId(numKeys)
view = None
gc.collect()
bytesUsed = TCMalloc.getBytesUsed() - before
self.assertTrue(bytesUsed < 1024 * 16)
def test_listener(self):
harness = self.getHarness(inMemory=True)
try:
v1 = harness.newView()
v2 = harness.newView()
l = SharedState.Listener(v2)
space = SharedState.Keyspace("ComparisonKeyType",
NativeJson.Json("TestSpace"), 1)
rng = SharedState.KeyRange(space, 0, None, None, True, False)
v1.subscribe(rng)
v2.subscribe(rng)
keysToWrite = [JsonNative.Json(str(x)) for x in range(40)]
def writer():
with SharedState.Transaction(v1):
for keyName in keysToWrite:
key = SharedState.Key(space, (keyName, ))
v1[key] = JsonNative.Json("value")
changedKeys = []
def reader():
while len(changedKeys) < len(keysToWrite):
updates = l.get()
for updateType, update in updates:
if updateType == "KeyUpdates":
for key in update:
changedKeys.append(key[0])
writerThread = threading.Thread(target=writer)
readerThread = threading.Thread(target=reader)
writerThread.start()
readerThread.start()
writerThread.join()
readerThread.join()
self.assertEqual(set(keysToWrite), set(changedKeys))
finally:
time.sleep(0.01)
harness.teardown()
def test_require_subscription_view(self):
try:
testKeyspace = SharedState.Keyspace("TakeHighestIdKeyType",
NativeJson.Json("TestSpace"),
1)
harness = self.getHarness(inMemory=True)
v1 = harness.newView()
def toTry():
with SharedState.Transaction(v1):
key = SharedState.Key(testKeyspace, ("key", ))
v1[key] = NativeJson.Json('this is a test value')
self.assertRaises(UserWarning, toTry)
finally:
time.sleep(0.01)
harness.teardown()
def test_null_value_is_deleted(self):
numEvents = 1024 * 64
valueSize = 128
numKeys = numEvents
view = createViewWithNoChannel()
keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1)
keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True)
cache = SharedStateNative.KeyspaceCache(keyrange, None)
gc.collect()
m0 = TCMalloc.getBytesUsed()
for event in producePartialEvents(view, [keyspace], 'test', numKeys, numKeys, valueSize):
cache.addEvent(event)
cache.addEvent(produceNullifyingEvent(view, event))
cache.newMinimumId(numKeys * 2)
gc.collect()
self.assertLess(TCMalloc.getBytesUsed() - m0, 1024 * 4)
def test(threadIx):
for subPassIx in range(subPasses):
logging.info("Thread %s starting pass %s", threadIx, subPassIx)
testKeyspace = SharedState.Keyspace(
"TakeHighestIdKeyType", NativeJson.Json("TestSpace"), 1)
view = viewFactory.createView()
rng = SharedState.KeyRange(testKeyspace, 0, None, None, True,
False)
view.subscribe(rng)
for ix in range(keysToWrite):
with SharedState.Transaction(view):
ix = random.randint(0, keyspaceSize)
key = SharedState.Key(
testKeyspace, (NativeJson.Json("key %s" % ix), ))
value = uuid.uuid4().hex
view[key] = NativeJson.Json(value)
worked[threadIx] = True
def __init__(self, numIters, valueSize=128, useMemoChannel=True,
numKeyspaces=1000, keyType="ComparisonKeyType"):
self.cacheDir = tempfile.mkdtemp()
self.fileStore = SharedStateNative.Storage.FileStorage(
self.cacheDir, 100, .1, useMemoChannel)
self.view = createViewWithNoChannel()
self.keyspaces = [SharedState.Keyspace(keyType, json('test-%s' % ix), 1)
for ix in range(numKeyspaces)]
self._numIters = numIters
self.keyspaceStorage = {}
self.iterator = enumerate(
producePartialEvents(
self.view,
self.keyspaces,
'test',
self._numIters,
1,
valueSize))
def test_file_management(self):
tempDir = tempfile.mkdtemp()
curOpenFiles = len(os.listdir('/proc/%s/fd' % os.getpid()))
OPEN_FILE_LIMIT = 200
if curOpenFiles >= OPEN_FILE_LIMIT:
os.system("ls -alh /proc/%s/fd" % os.getpid())
self.assertTrue(curOpenFiles < OPEN_FILE_LIMIT,
"Too many open files: %s" % curOpenFiles)
soft, hard = resource.getrlimit(resource.RLIMIT_OFILE)
harness = self.getHarness(inMemory=True,
cachePathOverride=tempDir,
maxOpenFiles=15)
try:
v1 = harness.newView()
resource.setrlimit(resource.RLIMIT_OFILE,
(curOpenFiles + 30, hard))
for ix in range(128):
space = SharedState.Keyspace(
"TakeHighestIdKeyType",
NativeJson.Json("TestSpace%s" % ix), 1)
rng = SharedState.KeyRange(space, 0, None, None, True, False)
v1.subscribe(rng)
key = SharedState.Key(space, (NativeJson.Json('key%s' % ix), ))
with SharedState.Transaction(v1):
v1[key] = NativeJson.Json('value %s' % ix)
finally:
time.sleep(0.01)
harness.teardown()
resource.setrlimit(resource.RLIMIT_OFILE, (soft, hard))
try:
shutil.rmtree(tempDir)
except:
pass
def test_require_subscription_manager(self):
tempDir = tempfile.mkdtemp()
try:
testKeyspace = SharedState.Keyspace("TakeHighestIdKeyType",
NativeJson.Json("TestSpace"),
1)
harness = self.getHarness(inMemory=True, cachePathOverride=tempDir)
v1 = harness.newView()
v1.subscribe(
SharedState.KeyRange(testKeyspace, 0, None, None, True, False))
for ix in range(1000):
with SharedState.Transaction(v1):
key = SharedState.Key(testKeyspace,
(NativeJson.Json("key"), ))
v1[key] = NativeJson.Json(str(ix))
time.sleep(2)
v2 = harness.newView()
v2.setMustSubscribe(False)
with SharedState.Transaction(v2):
key = SharedState.Key(testKeyspace, (NativeJson.Json("key"), ))
value = 'value is' + uuid.uuid4().hex
v2[key] = NativeJson.Json(value)
time.sleep(2)
def toTry():
with SharedState.Transaction(v2):
key = SharedState.Key(testKeyspace,
(NativeJson.Json("key"), ))
value = 'value is' + uuid.uuid4().hex
v2[key] = NativeJson.Json(value)
self.assertRaises(UserWarning, toTry)
finally:
time.sleep(0.01)
harness.teardown()
shutil.rmtree(tempDir)
def getAllItemsFromView(self, view, spacename):
keyspace = SharedState.Keyspace("TakeHighestIdKeyType", spacename, 1)
with SharedState.Transaction(view):
return [(x[0], x[1].value())
for x in SharedState.iterItems(view, keyspace)]
def writeValue_(self, keyspaceName, keyName, value):
self.view_[SharedState.Key(
SharedState.Keyspace("TakeHighestIdKeyType", keyspaceName, 1),
(keyName, ))] = value
def setUp(self):
print
self.keyspace = SharedState.Keyspace("TakeHighestIdType",
json('test_keyspace'), 1)
self.keyrange = SharedState.KeyRange(self.keyspace, 0, None, None,
True, True)
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import ufora.distributed.SharedState.SharedState as SharedState
import ufora.distributed.SharedState.tests.SharedStateTestHarness as SharedStateTestHarness
import ufora.native.Json as NativeJson
_testKeyspace = SharedState.Keyspace("TakeHighestIdKeyType",
NativeJson.Json("TestSpace"), 1)
class ListenerTest(unittest.TestCase):
def setUp(self):
self.harness = SharedStateTestHarness.SharedStateTestHarness(True)
def test_basic_listener(self):
v1 = self.harness.newView()
v2 = self.harness.newView()
while not v1.waitConnectTimeout(.1):
pass
while not v2.waitConnectTimeout(.1):
pass
def subscribeToKeyspace(self, view, spacename):
keyspace = SharedState.Keyspace("TakeHighestIdKeyType", spacename, 1)
view.subscribe(
SharedState.KeyRange(keyspace, 0, None, None, True, False), True)
def setUp(self):
self.testKeyspace = SharedState.Keyspace("TakeHighestIdKeyType",
NativeJson.Json("TestSpace"),
1)
self.harness = SharedStateTestHarness.SharedStateTestHarness(inMemory=True)
def getAllKeysFromView(self, view, spacename):
keyspace = SharedState.Keyspace("TakeHighestIdKeyType", spacename, 1)
with SharedState.Transaction(view):
return [x for x in SharedState.iterKeys(view, keyspace)]
def writeToKeyspace(self, view, spacename, key, val):
keyspace = SharedState.Keyspace("TakeHighestIdKeyType", spacename, 1)
with SharedState.Transaction(view):
view[SharedState.Key(keyspace, (key, ))] = val
