コード例 #1
0
    def test_persistentCacheUnderLoad(self):
        cppView1 = CumulusNative.PersistentCacheIndex(
            self.sharedState.newView(), callbackScheduler)

        t0 = time.time()
        #add 100k pages, which is enough for about 5 TB of data
        for index in range(100000):
            if index % 1000 == 0 and index > 0:
                print index, (time.time() - t0) / (index /
                                                   1000.0), " seconds per 1000"
            cppView1.addPage(sha1("page" + str(index)), HashSet(), 1, sha1(""))

        print "took ", time.time() - t0, " to add 100k."

        t1 = time.time()

        bytes0 = TCMallocNative.getBytesUsed()

        cppView2 = CumulusNative.PersistentCacheIndex(
            self.sharedState.newView(), callbackScheduler)

        while cppView2.totalBytesInCache() < 100000:
            time.sleep(.1)
            print cppView2.totalBytesInCache()

        print "took ", time.time() - t1, " to load 100k. Total RAM is ", (TCMallocNative.getBytesUsed() - bytes0) / 1024 / 1024.0, " MB per view"\
コード例 #2
0
    def test_persistentCacheUnderLoad(self):
        cppView1 = CumulusNative.PersistentCacheIndex(
            self.sharedState.newView(),
            callbackScheduler
            )

        t0 = time.time()
        #add 100k pages, which is enough for about 5 TB of data
        for index in range(100000):
            if index % 1000 == 0 and index > 0:
                print index, (time.time() - t0) / (index / 1000.0), " seconds per 1000"
            cppView1.addPage(sha1("page" + str(index)), HashSet(), 1, sha1(""))

        print "took ", time.time() - t0, " to add 100k."

        t1 = time.time()

        bytes0 = TCMallocNative.getBytesUsed()

        cppView2 = CumulusNative.PersistentCacheIndex(
            self.sharedState.newView(),
            callbackScheduler
            )

        while cppView2.totalBytesInCache() < 100000:
            time.sleep(.1)
            print cppView2.totalBytesInCache()

        print "took ", time.time() - t1, " to load 100k. Total RAM is ", (TCMallocNative.getBytesUsed() - bytes0) / 1024 / 1024.0, " MB per view"\
コード例 #3
0
ファイル: TCMalloc_test.py プロジェクト: Sandy4321/ufora
 def test_strings(self):
     bytes = TCMallocNative.getBytesUsed()
     for ix in range(100):
         s = str(ix) * 1000000
         s2 = TCMallocNative.returnStringArg(s)
         self.assertEqual(s, s2)
     finalBytes = TCMallocNative.getBytesUsed()
     self.assertTrue(finalBytes < bytes + 10000000)
コード例 #4
0
 def test_strings(self):
     bytes = TCMallocNative.getBytesUsed()
     for ix in range(100):
         s = str(ix) * 1000000
         s2 = TCMallocNative.returnStringArg(s)
         self.assertEqual(s, s2)
     finalBytes = TCMallocNative.getBytesUsed()
     self.assertTrue(finalBytes < bytes + 10000000)
コード例 #5
0
    def test_simple_manager(self):
        view = createViewWithNoChannel()
        before = TCMalloc.getBytesUsed()
        keyspace = SharedState.Keyspace("TakeHighestIdKeyType", json('test'), 1)
        cache = SharedStateNative.KeyspaceManager(0, 1, 0x7fffffff, 0x7fffffff, None)
        for event in producePartialEvents(view, [keyspace], 'test', 1024 * 32, 1, 8):
            cache.addEvent(event)
        view = None

        gc.collect()
        bytesUsed = TCMalloc.getBytesUsed() - before
        self.assertTrue(bytesUsed < 1024 * 128)
コード例 #6
0
    def test_simple_manager(self):
        view = createViewWithNoChannel()
        before = TCMalloc.getBytesUsed()
        keyspace = SharedState.Keyspace("TakeHighestIdKeyType", json('test'), 1)
        cache = SharedStateNative.KeyspaceManager(0, 1, 0x7fffffff, 0x7fffffff, None)
        for event in producePartialEvents(view, [keyspace], 'test', 1024 * 32, 1, 8):
            cache.addEvent(event)
        view = None

        gc.collect()
        bytesUsed = TCMalloc.getBytesUsed() - before
        self.assertTrue(bytesUsed < 1024 * 128)
コード例 #7
0
def memoryFreeingThreadLoop(queue):
    while True:
        element = queue.get()

        if element == "exit":
            return

        address, event = element

        TCMallocNative.freeAtAddress(address)

        event.set()
コード例 #8
0
ファイル: TCMalloc_test.py プロジェクト: Sandy4321/ufora
def memoryFreeingThreadLoop(queue):
    while True:
        element = queue.get()

        if element == "exit":
            return

        address, event = element

        TCMallocNative.freeAtAddress(address)

        event.set()
コード例 #9
0
    def test_keyspace_cache(self):
        numKeys = 1024 * 256
        before = TCMalloc.getBytesUsed()
        view = createViewWithNoChannel()
        keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1)
        keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True)
        cache = SharedStateNative.KeyspaceCache(keyrange, None)
        for event in producePartialEvents(view, [keyspace], 'test', numKeys, 1, 8):
            cache.addEvent(event)
        cache.newMinimumId(numKeys)
        view = None

        gc.collect()
        bytesUsed = TCMalloc.getBytesUsed() - before
        self.assertTrue(bytesUsed < 1024 * 16)
コード例 #10
0
    def test_keyspace_cache(self):
        numKeys = 1024 * 256
        before = TCMalloc.getBytesUsed()
        view = createViewWithNoChannel()
        keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1)
        keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True)
        cache = SharedStateNative.KeyspaceCache(keyrange, None)
        for event in producePartialEvents(view, [keyspace], 'test', numKeys, 1, 8):
            cache.addEvent(event)
        cache.newMinimumId(numKeys)
        view = None

        gc.collect()
        bytesUsed = TCMalloc.getBytesUsed() - before
        self.assertTrue(bytesUsed < 1024 * 16)
コード例 #11
0
 def getMemoryUseArray(self, tester):
     try:
         done = False
         memory = numpy.zeros(tester.numIters() + 2)
         memory[0] = TCMalloc.getBytesUsed()
         while not done:
             try:
                 ix = tester.next()
                 memory[ix + 1] = TCMalloc.getBytesUsed()
             except StopIteration:
                 done = True
         memory[-1] = TCMalloc.getBytesUsed()
         return memory
     finally:
         tester.tearDown()
コード例 #12
0
 def getMemoryUseArray(self, tester):
     try:
         done = False
         memory = numpy.zeros(tester.numIters() + 2)
         memory[0] = TCMalloc.getBytesUsed()
         while not done:
             try:
                 ix = tester.next()
                 memory[ix + 1] = TCMalloc.getBytesUsed()
             except StopIteration:
                 done = True
         memory[-1] = TCMalloc.getBytesUsed()
         return memory
     finally:
         tester.tearDown()
コード例 #13
0
    def test_null_value_is_deleted(self):
        numEvents = 1024 * 64
        valueSize = 128
        numKeys = numEvents

        view = createViewWithNoChannel()
        keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1)
        keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True)
        cache = SharedStateNative.KeyspaceCache(keyrange, None)
        gc.collect()
        m0 = TCMalloc.getBytesUsed()
        for event in producePartialEvents(view, [keyspace], 'test', numKeys, numKeys, valueSize):
            cache.addEvent(event)
            cache.addEvent(produceNullifyingEvent(view, event))

        cache.newMinimumId(numKeys * 2)
        gc.collect()
        self.assertLess(TCMalloc.getBytesUsed() - m0, 1024 * 4)
コード例 #14
0
    def test_null_value_is_deleted(self):
        numEvents = 1024 * 64
        valueSize = 128
        numKeys = numEvents

        view = createViewWithNoChannel()
        keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1)
        keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True)
        cache = SharedStateNative.KeyspaceCache(keyrange, None)
        gc.collect()
        m0 = TCMalloc.getBytesUsed()
        for event in producePartialEvents(view, [keyspace], 'test', numKeys, numKeys, valueSize):
            cache.addEvent(event)
            cache.addEvent(produceNullifyingEvent(view, event))

        cache.newMinimumId(numKeys * 2)
        gc.collect()
        self.assertLess(TCMalloc.getBytesUsed() - m0, 1024 * 4)
コード例 #15
0
    def DISABLEDtest_memory_load(self):
        b0 = TCMallocNative.getBytesUsed()
        b1 = b0
        for ix in range(10000):
            self.reasonAboutExpression("""
                let g = fun(t) { size(t) }
                let f = fun(t) { let c = g; c((1,) + t) + c((2,) + t) + c((3,) + t) };
                let f2 = fun(t) { let c = f; c((1,) + t) + c((2,) + t) + c((3,) + t) };
                let f3 = fun(t) { let c = f2; c((1,) + t) + c((2,) + t) + c((3,) + t) };
                let f4 = fun(t) { let c = f3; c((1,) + t) + c((2,) + t) + c((3,) + t) };
                
                f4((x,))
                """,
                                       x=str(ix))

            print (b1 - b0) / (ix+1) / 1024 / 1024.0, " MB per for a total of ",\
                (TCMallocNative.getBytesUsed() - b0) / 1024 / 1024.0, " MB allocated."

            b1 = TCMallocNative.getBytesUsed()
コード例 #16
0
    def DISABLEDtest_memory_load(self):
        b0 = TCMallocNative.getBytesUsed()
        b1 = b0
        for ix in range(10000):
            self.reasonAboutExpression(
                """
                let g = fun(t) { size(t) }
                let f = fun(t) { let c = g; c((1,) + t) + c((2,) + t) + c((3,) + t) };
                let f2 = fun(t) { let c = f; c((1,) + t) + c((2,) + t) + c((3,) + t) };
                let f3 = fun(t) { let c = f2; c((1,) + t) + c((2,) + t) + c((3,) + t) };
                let f4 = fun(t) { let c = f3; c((1,) + t) + c((2,) + t) + c((3,) + t) };
                
                f4((x,))
                """, x = str(ix)
                )

            print (b1 - b0) / (ix+1) / 1024 / 1024.0, " MB per for a total of ",\
                (TCMallocNative.getBytesUsed() - b0) / 1024 / 1024.0, " MB allocated."

            b1 = TCMallocNative.getBytesUsed()
コード例 #17
0
ファイル: TCMalloc_test.py プロジェクト: Sandy4321/ufora
    def test_realloc(self):
        #verify that TCMalloc accounts for resizing correctly
        measurements = [TCMallocNative.getBytesUsed()]

        for ix in range(10):
            addr = TCMallocNative.mallocAndReturnAddress(10 * 1024 * 1024)
            addr = TCMallocNative.reallocAtAddress(addr, 20 * 1024 * 1024)
            measurements.append(TCMallocNative.getBytesUsed())
            addr = TCMallocNative.reallocAtAddress(addr, 10 * 1024 * 1024)
            addr = TCMallocNative.reallocAtAddress(addr, 5 * 1024 * 1024)
            TCMallocNative.freeAtAddress(addr)

            measurements.append(TCMallocNative.getBytesUsed())

        self.assertTrue(
            measurements[-1] < measurements[0] + 10 * 1024 * 1024,
            "Expected %s to be less than 10 MB larger than %s" % (measurements[-1], measurements[0])
            )
コード例 #18
0
    def test_realloc(self):
        #verify that TCMalloc accounts for resizing correctly
        measurements = [TCMallocNative.getBytesUsed()]

        for ix in range(10):
            addr = TCMallocNative.mallocAndReturnAddress(10 * 1024 * 1024)
            addr = TCMallocNative.reallocAtAddress(addr, 20 * 1024 * 1024)
            measurements.append(TCMallocNative.getBytesUsed())
            addr = TCMallocNative.reallocAtAddress(addr, 10 * 1024 * 1024)
            addr = TCMallocNative.reallocAtAddress(addr, 5 * 1024 * 1024)
            TCMallocNative.freeAtAddress(addr)

            measurements.append(TCMallocNative.getBytesUsed())

        self.assertTrue(
            measurements[-1] < measurements[0] + 10 * 1024 * 1024,
            "Expected %s to be less than 10 MB larger than %s" %
            (measurements[-1], measurements[0]))
コード例 #19
0
ファイル: TCMalloc_test.py プロジェクト: Sandy4321/ufora
    def test_allocInOneThreadAndDeallocateInAnother(self):
        queue = Queue.Queue()

        thread = threading.Thread(target=memoryFreeingThreadLoop, args=(queue,))
        thread.start()

        #allocate 100 GB and free in the other thread
        for ix in range(1000):
            event = threading.Event()
            address = TCMallocNative.mallocAndReturnAddress(100 * 1024 * 1024)

            queue.put((address, event))

            event.wait()

        queue.put("exit")

        thread.join()
コード例 #20
0
    def test_allocInOneThreadAndDeallocateInAnother(self):
        queue = Queue.Queue()

        thread = threading.Thread(target=memoryFreeingThreadLoop,
                                  args=(queue, ))
        thread.start()

        #allocate 100 GB and free in the other thread
        for ix in range(1000):
            event = threading.Event()
            address = TCMallocNative.mallocAndReturnAddress(100 * 1024 * 1024)

            queue.put((address, event))

            event.wait()

        queue.put("exit")

        thread.join()
    def test_PythonIoTaskServiceInLoop(self):
        bytesUsed = []
        for ix in range(20):
            bytesUsed.append(TCMallocNative.getMemoryStat("generic.current_allocated_bytes") / 1024 / 1024.0)

            s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()

            s3.setThroughputPerMachine(1024 * 1024 * 20)

            for ix in range(35):
                s3().setKeyValue(
                    "bucketname",
                    "key_%s" % ix,
                    " " * 10 * 1024 * 1024
                    )

            text = """datasets.s3('bucketname', 'key_0').sum()"""

            self.computeUsingSeveralWorkers(text, s3, 4, timeout = 120, blockUntilConnected=True)

        self.assertTrue(bytesUsed[0] < bytesUsed[-1] - 100, bytesUsed)
コード例 #22
0
    def test_PythonIoTaskServiceInLoop(self):
        bytesUsed = []
        for ix in range(20):
            bytesUsed.append(TCMallocNative.getMemoryStat("generic.current_allocated_bytes") / 1024 / 1024.0)

            s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()

            s3.setThroughputPerMachine(1024 * 1024 * 20)

            for ix in range(35):
                s3().setKeyValue(
                    "bucketname",
                    "key_%s" % ix,
                    " " * 10 * 1024 * 1024
                    )

            text = """datasets.s3('bucketname', 'key_0').sum()"""

            self.computeUsingSeveralWorkers(text, s3, 4, timeout = 120, blockUntilConnected=True)

        self.assertTrue(bytesUsed[0] < bytesUsed[-1] - 100, bytesUsed)
コード例 #23
0
ファイル: test.py プロジェクト: Sandy4321/ufora
def printTestResults(moduleName, testName, testResultWithTimes, verbose = False):
    failct = 0
    passct = 0

    result, timeElapsed = testResultWithTimes

    if result is not True:
        failct += 1

        print moduleName, ": ", testName,

        print time.strftime("%Y-%m-%d %H:%M:%S"), "--", "TEST FAILED:", moduleName, testName, ":",

        if isinstance(result, FORAException):
            print "throw", sanitizeErrString(str(result.foraVal))
        else:
            print result,
        print "is not True"

    else:
        passct += 1
        if verbose:
            print time.strftime("%Y-%m-%d %H:%M:%S"), "--", moduleName, testName, ": TEST SUCCEEDED in %2.4f" % timeElapsed, ". memory usage is ", TCMallocNative.getBytesUsed() / 1024 / 1024.0, " MB"

    return (passct, failct)
コード例 #24
0
ファイル: test.py プロジェクト: data-processing/ufora
def printTestResults(moduleName, testName, testResultWithTimes, verbose=False):
    failct = 0
    passct = 0

    result, timeElapsed = testResultWithTimes

    if result is not True:
        failct += 1

        print moduleName, ": ", testName,

        print time.strftime("%Y-%m-%d %H:%M:%S"
                            ), "--", "TEST FAILED:", moduleName, testName, ":",

        if isinstance(result, FORAException):
            print "throw", sanitizeErrString(str(result.foraVal))
        else:
            print result,
        print "is not True"

    else:
        passct += 1
        if verbose:
            print time.strftime(
                "%Y-%m-%d %H:%M:%S"
            ), "--", moduleName, testName, ": TEST SUCCEEDED in %2.4f" % timeElapsed, ". memory usage is ", TCMallocNative.getBytesUsed(
            ) / 1024 / 1024.0, " MB"

    return (passct, failct)
コード例 #25
0
    def start(self):
        with setupLock:
            assert self.subprocessOutThread is None or not self.subprocessOutThread.is_alive(
            )

            stdInRead, stdInWrite = os.pipe()
            stdOutRead, stdOutWrite = os.pipe()
            stdErrRead, stdErrWrite = os.pipe()

            self.subprocessStdIn = os.fdopen(stdInWrite, 'w', 1)
            self.subprocessStdOut = os.fdopen(stdOutRead, 'r', 1)
            self.subprocessStdErr = os.fdopen(stdErrRead, 'r', 1)

            if self.enablePartialLineOutput:
                # enable non-blocking reads
                fcntl.fcntl(self.subprocessStdOut, fcntl.F_SETFL,
                            os.O_NONBLOCK)
                fcntl.fcntl(self.subprocessStdErr, fcntl.F_SETFL,
                            os.O_NONBLOCK)

            self.subprocessStdInFileDescriptor = stdInWrite
            self.subprocessStdOutFileDescriptor = stdOutRead
            self.subprocessStdErrFileDescriptor = stdErrRead

            self.subprocessStdOutFromOtherSide = os.fdopen(stdOutWrite, 'w', 1)
            self.subprocessStdErrFromOtherSide = os.fdopen(stdErrWrite, 'w', 1)

            #start our reading threads BEFORE we open the process
            self.subprocessOutThread = ManagedThread.ManagedThread(
                target=self.processOutputLoop,
                args=('stdOut', self.subprocessStdOut, self.onStdOut))

            self.subprocessOutThread.start()

            self.subprocessErrThread = ManagedThread.ManagedThread(
                target=self.processOutputLoop,
                args=('stdErr', self.subprocessStdErr, self.onStdErr))
            self.subprocessErrThread.start()

            logging.debug(
                "SubprocessRunner subprocess.Popen call starting with arguments %s",
                self.subprocessArguments)

            subprocessEvent = threading.Event()

            def startSubprocess():
                self.process = subprocess.Popen(self.subprocessArguments,
                                                stdin=stdInRead,
                                                stdout=stdOutWrite,
                                                stderr=stdErrWrite,
                                                env=self.env)
                subprocessEvent.set()

            startSubprocessThread = ManagedThread.ManagedThread(
                target=startSubprocess)
            startSubprocessThread.start()

            subprocessEvent.wait(10.0)

            if not subprocessEvent.isSet():
                import ufora.native.TCMalloc as TCMallocNative
                logging.error("Failed to start subprocess: Total MB used: %s",
                              TCMallocNative.getBytesUsed() / 1024 / 1024.0)
            assert subprocessEvent.isSet(
            ), "Failed to start the subprocess process."

            os.close(stdInRead)

            self.isStarted = True
            # return self to allow chaining like: runner.start().wait(...)
            return self
コード例 #26
0
ファイル: SubprocessRunner.py プロジェクト: Sandy4321/ufora
    def start(self):
        with setupLock:
            assert self.subprocessOutThread is None or not self.subprocessOutThread.is_alive()

            stdInRead, stdInWrite = os.pipe()
            stdOutRead, stdOutWrite = os.pipe()
            stdErrRead, stdErrWrite = os.pipe()

            self.subprocessStdIn = os.fdopen(stdInWrite, 'w', 1)
            self.subprocessStdOut = os.fdopen(stdOutRead, 'r', 1)
            self.subprocessStdErr = os.fdopen(stdErrRead, 'r', 1)


            if self.enablePartialLineOutput:
                # enable non-blocking reads
                fcntl.fcntl(self.subprocessStdOut, fcntl.F_SETFL, os.O_NONBLOCK)
                fcntl.fcntl(self.subprocessStdErr, fcntl.F_SETFL, os.O_NONBLOCK)


            self.subprocessStdInFileDescriptor = stdInWrite
            self.subprocessStdOutFileDescriptor = stdOutRead
            self.subprocessStdErrFileDescriptor = stdErrRead

            self.subprocessStdOutFromOtherSide = os.fdopen(stdOutWrite, 'w', 1)
            self.subprocessStdErrFromOtherSide = os.fdopen(stdErrWrite, 'w', 1)

            #start our reading threads BEFORE we open the process
            self.subprocessOutThread = ManagedThread.ManagedThread(
                target=self.processOutputLoop,
                args=('stdOut', self.subprocessStdOut, self.onStdOut)
                )

            self.subprocessOutThread.start()

            self.subprocessErrThread = ManagedThread.ManagedThread(
                target=self.processOutputLoop,
                args=('stdErr', self.subprocessStdErr, self.onStdErr)
                )
            self.subprocessErrThread.start()

            logging.debug("SubprocessRunner subprocess.Popen call starting with arguments %s",
                          self.subprocessArguments)

            subprocessEvent = threading.Event()
            def startSubprocess():
                self.process = subprocess.Popen(
                        self.subprocessArguments,
                        stdin=stdInRead,
                        stdout=stdOutWrite,
                        stderr=stdErrWrite,
                        env=self.env
                        )
                subprocessEvent.set()

            startSubprocessThread = ManagedThread.ManagedThread(target=startSubprocess)
            startSubprocessThread.start()

            subprocessEvent.wait(10.0)

            if not subprocessEvent.isSet():
                import ufora.native.TCMalloc as TCMallocNative
                logging.error("Failed to start subprocess: Total MB used: %s", TCMallocNative.getBytesUsed() / 1024 / 1024.0)
            assert subprocessEvent.isSet(), "Failed to start the subprocess process."

            os.close(stdInRead)

            self.isStarted = True
            # return self to allow chaining like: runner.start().wait(...)
            return self