示例#1
0
    def test_events_in_subscription(self):
        asynView = None
        try:
            v1 = self.harness.newView()
            self.subscribeToTestKeyspace(v1)

            for ix in range(100):
                key = SharedState.Key(self.testKeyspace, (NativeJson.Json('test%s' % ix),))
                with SharedState.Transaction(v1):
                    v1[key] = NativeJson.Json('value1')

            done = threading.Event()
            numKeyEvents = [0]
            def onKeyEvents(eventDict):
                numKeyEvents[0] += len(eventDict)


            def onSubscriptionLoaded(_):
                done.set()

            def onConnected(_):
                deferred = asynView.subscribeToKeyspace(self.testKeyspace, 0, onKeyEvents)
                deferred.addCallbacks(onSubscriptionLoaded, lambda error: None)


            asynView = AsyncView.AsyncView(self.harness.viewFactory,
                                           onConnectCallback=onConnected)
            asynView.startService()

            while not done.wait(.1):
                pass

            self.assertEqual(numKeyEvents[0], 100)
        finally:
            asynView.stopService()
示例#2
0
    def runInTransaction(self, keyPathElement, toRun, waitForKeyToLoad = True):
        """
        Runs a specified function in a shared state transaction.

        toRun is a function that takes the following argumenst:
            self - a self reference
            key  - the shared state key used by the test
            view - the a shared state view subscribed to the key range
                   that contains key.
            waitForKeyToLoad - wait for SharedState to actually load the
                    key
        """
        view = self.sharedState.newView()
        view.subscribe(self.keyRange)

        passes = 0
        while passes < 100:
            with SharedState.Transaction(view):
                key = SharedState.Key(self.keyspace, (Node.keyPathToKeyName((keyPathElement,)),))

                if not waitForKeyToLoad or view[key] is not None:
                    toRun(self, key, view)
                    return

            passes += 1
            time.sleep(.1)

        assert False, "Test timed out"
示例#3
0
    def test_flush(self):
        harness = self.getHarness(inMemory=True)

        try:
            v1 = harness.newView()

            space = SharedState.Keyspace("ComparisonKeyType",
                                         NativeJson.Json("TestSpace"), 1)
            rng = SharedState.KeyRange(space, 0, None, None, True, False)

            v1.subscribe(rng)

            keysToWrite = [str(x) for x in range(40)]

            value = "value" * 100 * 1024

            with SharedState.Transaction(v1):
                self.assertRaises(UserWarning, v1.flush, True)

            with SharedState.Transaction(v1):
                for keyName in keysToWrite:
                    key = SharedState.Key(space, (NativeJson.Json(keyName), ))
                    v1[key] = NativeJson.Json(value)

            v1.flush()
        finally:
            time.sleep(0.01)
            harness.teardown()
示例#4
0
    def writeOwnRegistrationInformationIntoView_(self):
        if self._registeredIpPortAndMachineId is None:
            return

        address, ports, machineIdAsString = self._registeredIpPortAndMachineId

        logging.info(
            "CumulusActiveMachine registering worker %s at %s:%s with id %s",
            self.clientID,
            address,
            ports,
            machineIdAsString
            )

        with self._lock:
            self.activeMachineIds.add(machineIdAsString)

        key = SharedState.Key(self.workerStatusKeyspace, (NativeJson.Json(str(self.clientID)),))
        value = NativeJson.Json( (str(address), [str(x) for x in ports], machineIdAsString) )

        doneWithCall = threading.Event()
        self.asyncView.pushTransaction(key, value, lambda result: doneWithCall.set())

        while not doneWithCall.wait(.1) and not self._stopFlag.is_set():
            logging.debug('CumulusActiveMachines waiting on registerSelfAsActive')
    def test_exception(self):
        v = self.newView()
        v.waitConnect()
        v.subscribe(self.introRange)

        key = SharedState.Key(self.introKeyspace, (NativeJson.Json("0"),NativeJson.Json(str(v.id))))
        # this should raise because the view is not frozen
        def shouldRaise():
            v[key] = NativeJson.Json("some data")
        self.assertRaises(UserWarning, shouldRaise)
示例#6
0
 def keyspaceIterItems(self, keyspace):
     self.assertSingleThreaded()
     startKey = tuple(
         [NativeJson.lowestValue() for x in range(keyspace.dimension)])
     with SharedState.Transaction(self._view):
         k = self._view.nextKey(SharedState.Key(keyspace, startKey))
         while k is not None and str(k.keyspace) == str(keyspace.name):
             if self._view[k] is not None:
                 yield k, self._view[k].value()
             k = self._view.nextKey(k)
示例#7
0
            def testWrite(passIx):
                try:
                    testKeyspaces = [
                        SharedState.Keyspace("TakeHighestIdKeyType",
                                             NativeJson.Json("TestSpace%s" % ix), 3) \
                        for ix in range(3)
                        ]

                    harness = self.getHarness(inMemory=True,
                                              cachePathOverride=tempDir)
                    v1 = harness.newView()
                    for space in testKeyspaces:
                        rng = SharedState.KeyRange(space, 0, None, None, True,
                                                   False)
                        v1.subscribe(rng)

                    for ix in range(9):
                        keyspaceIx = (ix + passIx) % len(testKeyspaces)

                        space = testKeyspaces[
                            keyspaceIx]  #random.choice(testKeyspaces)
                        with SharedState.Transaction(v1):
                            key = SharedState.Key(
                                space,
                                (NativeJson.Json("key %s" % ix),
                                 NativeJson.Json(''), NativeJson.Json('')))
                            value = JsonNative.Json(
                                '<value is %s in space %s on pass %s: %s>' %
                                (ix, keyspaceIx, passIx, uuid.uuid4().hex))
                            v1[key] = value
                            testValues[key] = value

                    v1.flush()

                    time.sleep(0.01)
                    harness.teardown()
                    harness = self.getHarness(inMemory=True,
                                              cachePathOverride=tempDir)

                    v1 = harness.newView()
                    for space in testKeyspaces:
                        rng = SharedState.KeyRange(space, 0, None, None, True,
                                                   False)
                        v1.subscribe(rng)

                    for key, value in testValues.iteritems():
                        self.assertSharedStateKeyResolvesToValue(
                            key, v1, testValues[key])
                except:
                    logging.warn("Exception: %s", traceback.format_exc())
                    raise
                finally:
                    time.sleep(0.01)
                    harness.teardown()
示例#8
0
    def test_require_subscription_manager(self):
        tempDir = tempfile.mkdtemp()
        try:
            testKeyspace = SharedState.Keyspace("TakeHighestIdKeyType",
                                                NativeJson.Json("TestSpace"),
                                                1)
            harness = self.getHarness(inMemory=True, cachePathOverride=tempDir)

            v1 = harness.newView()
            v1.subscribe(
                SharedState.KeyRange(testKeyspace, 0, None, None, True, False))

            for ix in range(1000):
                with SharedState.Transaction(v1):
                    key = SharedState.Key(testKeyspace,
                                          (NativeJson.Json("key"), ))
                    v1[key] = NativeJson.Json(str(ix))
            time.sleep(2)

            v2 = harness.newView()
            v2.setMustSubscribe(False)
            with SharedState.Transaction(v2):
                key = SharedState.Key(testKeyspace, (NativeJson.Json("key"), ))
                value = 'value is' + uuid.uuid4().hex
                v2[key] = NativeJson.Json(value)
            time.sleep(2)

            def toTry():
                with SharedState.Transaction(v2):
                    key = SharedState.Key(testKeyspace,
                                          (NativeJson.Json("key"), ))
                    value = 'value is' + uuid.uuid4().hex
                    v2[key] = NativeJson.Json(value)

            self.assertRaises(UserWarning, toTry)
        finally:
            time.sleep(0.01)
            harness.teardown()
            shutil.rmtree(tempDir)
示例#9
0
    def test_two_keyspaces(self):
        try:
            toWrite = Queue.Queue()
            space1 = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace1"), 1)
            space2 = SharedState.Keyspace("TakeHighestIdKeyType", NativeJson.Json("TestSpace2"), 1)


            def callback1(items):
                for key in items.iterkeys():
                    self.assertEqual(key.keyspace, NativeJson.Json("TestSpace1"))
                    toWrite.task_done()

            def callback2(items):
                for key in items.iterkeys():
                    self.assertEqual(key.keyspace, NativeJson.Json("TestSpace2"))
                    toWrite.task_done()


            def onConnected(_):
                simpleView.subscribeToKeyspace(space1, 0, callback1)
                simpleView.subscribeToKeyspace(space2, 0, callback2)

            simpleView = AsyncView.AsyncView(self.harness.viewFactory,
                                             onConnected)

            simpleView.startService()

            v1 = self.harness.newView()

            v1.subscribe(SharedState.KeyRange(space1, 0, None, None, True, False),
                         True)

            v1.subscribe(SharedState.KeyRange(space2, 0, None, None, True, False),
                         True)


            for x in range(100):
                space = space1
                if random.random() < .5:
                    space = space2
                toWrite.put((str(x), space))
                with SharedState.Transaction(v1):
                    v1[SharedState.Key(space, (NativeJson.Json(str(x)),))] = space.name

            toWrite.join()


        finally:
            if simpleView:
                simpleView.stopService()
            self.harness.teardown()
示例#10
0
    def test_async_view(self):
        try:
            simpleView = None
            toWrite = {
                NativeJson.Json(str(ix)): NativeJson.Json('value-%s' % ix)
                for ix in range(100)
                }
            written = {}

            v1 = self.harness.newView()

            self.subscribeToTestKeyspace(v1)

            for key, value in toWrite.iteritems():
                with SharedState.Transaction(v1):
                    key = SharedState.Key(self.testKeyspace, (key,))
                    v1[key] = value
                    written[key[0]] = value

            v1.flush()


            received = {}

            newKeysWritten = threading.Condition()
            def onConnected(_):
                def callback(keysDict):
                    for key, value in keysDict.iteritems():
                        received[key[0]] = value

                    with newKeysWritten:
                        newKeysWritten.notify()

                simpleView.subscribeToKeyspace(self.testKeyspace, 0, callback)

            simpleView = AsyncView.AsyncView(self.harness.viewFactory,
                                             onConnectCallback=onConnected)

            simpleView.startService()

            while set(received.values()) != set(written.values()):
                with newKeysWritten:
                    newKeysWritten.wait()

            self.assertEqual(set(received.values()), set(written.values()))

        finally:
            if simpleView:
                simpleView.stopService()
            self.harness.teardown()
示例#11
0
    def test_basic_listener(self):
        v1 = self.harness.newView()
        v2 = self.harness.newView()

        while not v1.waitConnectTimeout(.1):
            pass

        while not v2.waitConnectTimeout(.1):
            pass

        l = SharedState.Listener(v1)
        v1.subscribe(
            SharedState.KeyRange(_testKeyspace, 0, None, None, True, True),
            True)
        v2.subscribe(
            SharedState.KeyRange(_testKeyspace, 0, None, None, True, True),
            True)

        self.assertIsInstance(l.get()[0][1],
                              SharedState.SharedStateNative.KeyRange)

        self.assertTrue(len(l.get(.1)) == 0)

        self.assertTrue(len(l.getNonblock()) == 0)

        testKey = SharedState.Key(_testKeyspace, (NativeJson.Json('test'), ))
        with SharedState.Transaction(v2):
            v2[testKey] = NativeJson.Json('test2')

        v2.flush()
        v1.flush()

        updates = l.get(1)
        self.assertTrue(len(updates) == 1)
        self.assertEqual(updates[0][0], "KeyUpdates")
        self.assertIsInstance(updates[0][1], list)
        self.assertTrue(updates[0][1][0] == testKey)

        with SharedState.Transaction(v2):
            v2[testKey] = NativeJson.Json('newer test')

        v2.flush()
        v1.flush()

        updates = l.get(1)
        self.assertTrue(len(updates) == 1)
        self.assertEqual(updates[0][0], "KeyUpdates")
        self.assertIsInstance(updates[0][1], list)
        self.assertTrue(updates[0][1][0] == testKey)
示例#12
0
    def test_keyspace_iteration(self):
        try:
            toWrite = {
                NativeJson.Json(str(x)): NativeJson.Json('value-%s' % x)
                for x in range(50)
                }
            simpleView = None

            v1 = self.harness.newView()
            self.subscribeToTestKeyspace(v1)

            for key, value in toWrite.iteritems():
                key = SharedState.Key(self.testKeyspace, (key,))
                with SharedState.Transaction(v1):
                    v1[key] = value

            v1.flush()

            def onConnect(_):
                def checkIteration(value):
                    for key, value in simpleView.keyspaceItems(self.testKeyspace):
                        assert key[0] in toWrite
                        self.assertEqual(toWrite[key[0]], value)

                deferred = simpleView.subscribeToKeyspace(self.testKeyspace,
                                                          0,
                                                          lambda keys: None)
                deferred.addCallbacks(checkIteration, lambda value: None)


            simpleView = AsyncView.AsyncView(self.harness.viewFactory,
                                             onConnectCallback=onConnect)

            simpleView.startService()



        finally:
            if simpleView:
                simpleView.stopService()
            self.harness.teardown()
示例#13
0
        def test(threadIx):
            for subPassIx in range(subPasses):
                logging.info("Thread %s starting pass %s", threadIx, subPassIx)
                testKeyspace = SharedState.Keyspace(
                    "TakeHighestIdKeyType", NativeJson.Json("TestSpace"), 1)

                view = viewFactory.createView()

                rng = SharedState.KeyRange(testKeyspace, 0, None, None, True,
                                           False)
                view.subscribe(rng)

                for ix in range(keysToWrite):
                    with SharedState.Transaction(view):
                        ix = random.randint(0, keyspaceSize)
                        key = SharedState.Key(
                            testKeyspace, (NativeJson.Json("key %s" % ix), ))
                        value = uuid.uuid4().hex
                        view[key] = NativeJson.Json(value)

            worked[threadIx] = True
示例#14
0
    def test_file_management(self):
        tempDir = tempfile.mkdtemp()

        curOpenFiles = len(os.listdir('/proc/%s/fd' % os.getpid()))

        OPEN_FILE_LIMIT = 200

        if curOpenFiles >= OPEN_FILE_LIMIT:
            os.system("ls -alh /proc/%s/fd" % os.getpid())

        self.assertTrue(curOpenFiles < OPEN_FILE_LIMIT,
                        "Too many open files: %s" % curOpenFiles)

        soft, hard = resource.getrlimit(resource.RLIMIT_OFILE)

        harness = self.getHarness(inMemory=True,
                                  cachePathOverride=tempDir,
                                  maxOpenFiles=15)
        try:
            v1 = harness.newView()
            resource.setrlimit(resource.RLIMIT_OFILE,
                               (curOpenFiles + 30, hard))

            for ix in range(128):
                space = SharedState.Keyspace(
                    "TakeHighestIdKeyType",
                    NativeJson.Json("TestSpace%s" % ix), 1)
                rng = SharedState.KeyRange(space, 0, None, None, True, False)
                v1.subscribe(rng)
                key = SharedState.Key(space, (NativeJson.Json('key%s' % ix), ))
                with SharedState.Transaction(v1):
                    v1[key] = NativeJson.Json('value %s' % ix)
        finally:
            time.sleep(0.01)
            harness.teardown()
            resource.setrlimit(resource.RLIMIT_OFILE, (soft, hard))
            try:
                shutil.rmtree(tempDir)
            except:
                pass
示例#15
0
 def toTry():
     with SharedState.Transaction(v1):
         key = SharedState.Key(testKeyspace, ("key", ))
         v1[key] = NativeJson.Json('this is a test value')
示例#16
0
 def toTry():
     with SharedState.Transaction(v2):
         key = SharedState.Key(testKeyspace,
                               (NativeJson.Json("key"), ))
         value = 'value is' + uuid.uuid4().hex
         v2[key] = NativeJson.Json(value)
示例#17
0
def producePartialEvents(view, keyspaces, keyBase, numWrites, numKeys, valueLen):
    with open('/dev/urandom') as randBytes:
        for ix in xrange(numWrites):
            key = SharedState.Key(keyspaces[ix  % len(keyspaces)],
                    (json('%s-%s' % (keyBase, ix % numKeys)),))
            yield view.createKeyUpdate(key, json(randBytes.read(valueLen)))
示例#18
0
 def writeValue_(self, keyspaceName, keyName, value):
     self.view_[SharedState.Key(
         SharedState.Keyspace("TakeHighestIdKeyType", keyspaceName, 1),
         (keyName, ))] = value
示例#19
0
 def writer():
     with SharedState.Transaction(v1):
         for keyName in keysToWrite:
             key = SharedState.Key(space, (keyName, ))
             v1[key] = JsonNative.Json("value")
示例#20
0
 def getKey(self, name):
     return SharedState.Key(self.keyspace, (json(name), ))
示例#21
0
    def writeToKeyspace(self, view, spacename, key, val):
        keyspace = SharedState.Keyspace("TakeHighestIdKeyType", spacename, 1)

        with SharedState.Transaction(view):
            view[SharedState.Key(keyspace, (key, ))] = val
示例#22
0
 def callback(value):
     for key, value in toWrite.iteritems():
         key = SharedState.Key(self.testKeyspace, (key,))
         simpleView.pushTransaction(key, value)