def test_persistentCacheUnderLoad(self):
cppView1 = CumulusNative.PersistentCacheIndex(
self.sharedState.newView(), callbackScheduler)
t0 = time.time()
#add 100k pages, which is enough for about 5 TB of data
for index in range(100000):
if index % 1000 == 0 and index > 0:
print index, (time.time() - t0) / (index /
1000.0), " seconds per 1000"
cppView1.addPage(sha1("page" + str(index)), HashSet(), 1, sha1(""))
print "took ", time.time() - t0, " to add 100k."
t1 = time.time()
bytes0 = TCMallocNative.getBytesUsed()
cppView2 = CumulusNative.PersistentCacheIndex(
self.sharedState.newView(), callbackScheduler)
while cppView2.totalBytesInCache() < 100000:
time.sleep(.1)
print cppView2.totalBytesInCache()
print "took ", time.time() - t1, " to load 100k. Total RAM is ", (TCMallocNative.getBytesUsed() - bytes0) / 1024 / 1024.0, " MB per view"\
def test_persistentCacheUnderLoad(self):
cppView1 = CumulusNative.PersistentCacheIndex(
self.sharedState.newView(),
callbackScheduler
)
t0 = time.time()
#add 100k pages, which is enough for about 5 TB of data
for index in range(100000):
if index % 1000 == 0 and index > 0:
print index, (time.time() - t0) / (index / 1000.0), " seconds per 1000"
cppView1.addPage(sha1("page" + str(index)), HashSet(), 1, sha1(""))
print "took ", time.time() - t0, " to add 100k."
t1 = time.time()
bytes0 = TCMallocNative.getBytesUsed()
cppView2 = CumulusNative.PersistentCacheIndex(
self.sharedState.newView(),
callbackScheduler
)
while cppView2.totalBytesInCache() < 100000:
time.sleep(.1)
print cppView2.totalBytesInCache()
print "took ", time.time() - t1, " to load 100k. Total RAM is ", (TCMallocNative.getBytesUsed() - bytes0) / 1024 / 1024.0, " MB per view"\
def test_strings(self):
bytes = TCMallocNative.getBytesUsed()
for ix in range(100):
s = str(ix) * 1000000
s2 = TCMallocNative.returnStringArg(s)
self.assertEqual(s, s2)
finalBytes = TCMallocNative.getBytesUsed()
self.assertTrue(finalBytes < bytes + 10000000)
def test_strings(self):
bytes = TCMallocNative.getBytesUsed()
for ix in range(100):
s = str(ix) * 1000000
s2 = TCMallocNative.returnStringArg(s)
self.assertEqual(s, s2)
finalBytes = TCMallocNative.getBytesUsed()
self.assertTrue(finalBytes < bytes + 10000000)
def test_simple_manager(self):
view = createViewWithNoChannel()
before = TCMalloc.getBytesUsed()
keyspace = SharedState.Keyspace("TakeHighestIdKeyType", json('test'), 1)
cache = SharedStateNative.KeyspaceManager(0, 1, 0x7fffffff, 0x7fffffff, None)
for event in producePartialEvents(view, [keyspace], 'test', 1024 * 32, 1, 8):
cache.addEvent(event)
view = None
gc.collect()
bytesUsed = TCMalloc.getBytesUsed() - before
self.assertTrue(bytesUsed < 1024 * 128)
def test_simple_manager(self):
view = createViewWithNoChannel()
before = TCMalloc.getBytesUsed()
keyspace = SharedState.Keyspace("TakeHighestIdKeyType", json('test'), 1)
cache = SharedStateNative.KeyspaceManager(0, 1, 0x7fffffff, 0x7fffffff, None)
for event in producePartialEvents(view, [keyspace], 'test', 1024 * 32, 1, 8):
cache.addEvent(event)
view = None
gc.collect()
bytesUsed = TCMalloc.getBytesUsed() - before
self.assertTrue(bytesUsed < 1024 * 128)
def memoryFreeingThreadLoop(queue):
while True:
element = queue.get()
if element == "exit":
return
address, event = element
TCMallocNative.freeAtAddress(address)
event.set()
def memoryFreeingThreadLoop(queue):
while True:
element = queue.get()
if element == "exit":
return
address, event = element
TCMallocNative.freeAtAddress(address)
event.set()
def test_keyspace_cache(self):
numKeys = 1024 * 256
before = TCMalloc.getBytesUsed()
view = createViewWithNoChannel()
keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1)
keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True)
cache = SharedStateNative.KeyspaceCache(keyrange, None)
for event in producePartialEvents(view, [keyspace], 'test', numKeys, 1, 8):
cache.addEvent(event)
cache.newMinimumId(numKeys)
view = None
gc.collect()
bytesUsed = TCMalloc.getBytesUsed() - before
self.assertTrue(bytesUsed < 1024 * 16)
def test_keyspace_cache(self):
numKeys = 1024 * 256
before = TCMalloc.getBytesUsed()
view = createViewWithNoChannel()
keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1)
keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True)
cache = SharedStateNative.KeyspaceCache(keyrange, None)
for event in producePartialEvents(view, [keyspace], 'test', numKeys, 1, 8):
cache.addEvent(event)
cache.newMinimumId(numKeys)
view = None
gc.collect()
bytesUsed = TCMalloc.getBytesUsed() - before
self.assertTrue(bytesUsed < 1024 * 16)
def getMemoryUseArray(self, tester):
try:
done = False
memory = numpy.zeros(tester.numIters() + 2)
memory[0] = TCMalloc.getBytesUsed()
while not done:
try:
ix = tester.next()
memory[ix + 1] = TCMalloc.getBytesUsed()
except StopIteration:
done = True
memory[-1] = TCMalloc.getBytesUsed()
return memory
finally:
tester.tearDown()
def getMemoryUseArray(self, tester):
try:
done = False
memory = numpy.zeros(tester.numIters() + 2)
memory[0] = TCMalloc.getBytesUsed()
while not done:
try:
ix = tester.next()
memory[ix + 1] = TCMalloc.getBytesUsed()
except StopIteration:
done = True
memory[-1] = TCMalloc.getBytesUsed()
return memory
finally:
tester.tearDown()
def test_null_value_is_deleted(self):
numEvents = 1024 * 64
valueSize = 128
numKeys = numEvents
view = createViewWithNoChannel()
keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1)
keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True)
cache = SharedStateNative.KeyspaceCache(keyrange, None)
gc.collect()
m0 = TCMalloc.getBytesUsed()
for event in producePartialEvents(view, [keyspace], 'test', numKeys, numKeys, valueSize):
cache.addEvent(event)
cache.addEvent(produceNullifyingEvent(view, event))
cache.newMinimumId(numKeys * 2)
gc.collect()
self.assertLess(TCMalloc.getBytesUsed() - m0, 1024 * 4)
def test_null_value_is_deleted(self):
numEvents = 1024 * 64
valueSize = 128
numKeys = numEvents
view = createViewWithNoChannel()
keyspace = SharedState.Keyspace("ComparisonKeyType", json('test'), 1)
keyrange = SharedState.KeyRange(keyspace, 0, None, None, True, True)
cache = SharedStateNative.KeyspaceCache(keyrange, None)
gc.collect()
m0 = TCMalloc.getBytesUsed()
for event in producePartialEvents(view, [keyspace], 'test', numKeys, numKeys, valueSize):
cache.addEvent(event)
cache.addEvent(produceNullifyingEvent(view, event))
cache.newMinimumId(numKeys * 2)
gc.collect()
self.assertLess(TCMalloc.getBytesUsed() - m0, 1024 * 4)
def DISABLEDtest_memory_load(self):
b0 = TCMallocNative.getBytesUsed()
b1 = b0
for ix in range(10000):
self.reasonAboutExpression("""
let g = fun(t) { size(t) }
let f = fun(t) { let c = g; c((1,) + t) + c((2,) + t) + c((3,) + t) };
let f2 = fun(t) { let c = f; c((1,) + t) + c((2,) + t) + c((3,) + t) };
let f3 = fun(t) { let c = f2; c((1,) + t) + c((2,) + t) + c((3,) + t) };
let f4 = fun(t) { let c = f3; c((1,) + t) + c((2,) + t) + c((3,) + t) };
f4((x,))
""",
x=str(ix))
print (b1 - b0) / (ix+1) / 1024 / 1024.0, " MB per for a total of ",\
(TCMallocNative.getBytesUsed() - b0) / 1024 / 1024.0, " MB allocated."
b1 = TCMallocNative.getBytesUsed()
def DISABLEDtest_memory_load(self):
b0 = TCMallocNative.getBytesUsed()
b1 = b0
for ix in range(10000):
self.reasonAboutExpression(
"""
let g = fun(t) { size(t) }
let f = fun(t) { let c = g; c((1,) + t) + c((2,) + t) + c((3,) + t) };
let f2 = fun(t) { let c = f; c((1,) + t) + c((2,) + t) + c((3,) + t) };
let f3 = fun(t) { let c = f2; c((1,) + t) + c((2,) + t) + c((3,) + t) };
let f4 = fun(t) { let c = f3; c((1,) + t) + c((2,) + t) + c((3,) + t) };
f4((x,))
""", x = str(ix)
)
print (b1 - b0) / (ix+1) / 1024 / 1024.0, " MB per for a total of ",\
(TCMallocNative.getBytesUsed() - b0) / 1024 / 1024.0, " MB allocated."
b1 = TCMallocNative.getBytesUsed()
def test_realloc(self):
#verify that TCMalloc accounts for resizing correctly
measurements = [TCMallocNative.getBytesUsed()]
for ix in range(10):
addr = TCMallocNative.mallocAndReturnAddress(10 * 1024 * 1024)
addr = TCMallocNative.reallocAtAddress(addr, 20 * 1024 * 1024)
measurements.append(TCMallocNative.getBytesUsed())
addr = TCMallocNative.reallocAtAddress(addr, 10 * 1024 * 1024)
addr = TCMallocNative.reallocAtAddress(addr, 5 * 1024 * 1024)
TCMallocNative.freeAtAddress(addr)
measurements.append(TCMallocNative.getBytesUsed())
self.assertTrue(
measurements[-1] < measurements[0] + 10 * 1024 * 1024,
"Expected %s to be less than 10 MB larger than %s" % (measurements[-1], measurements[0])
)
def test_realloc(self):
#verify that TCMalloc accounts for resizing correctly
measurements = [TCMallocNative.getBytesUsed()]
for ix in range(10):
addr = TCMallocNative.mallocAndReturnAddress(10 * 1024 * 1024)
addr = TCMallocNative.reallocAtAddress(addr, 20 * 1024 * 1024)
measurements.append(TCMallocNative.getBytesUsed())
addr = TCMallocNative.reallocAtAddress(addr, 10 * 1024 * 1024)
addr = TCMallocNative.reallocAtAddress(addr, 5 * 1024 * 1024)
TCMallocNative.freeAtAddress(addr)
measurements.append(TCMallocNative.getBytesUsed())
self.assertTrue(
measurements[-1] < measurements[0] + 10 * 1024 * 1024,
"Expected %s to be less than 10 MB larger than %s" %
(measurements[-1], measurements[0]))
def test_allocInOneThreadAndDeallocateInAnother(self):
queue = Queue.Queue()
thread = threading.Thread(target=memoryFreeingThreadLoop, args=(queue,))
thread.start()
#allocate 100 GB and free in the other thread
for ix in range(1000):
event = threading.Event()
address = TCMallocNative.mallocAndReturnAddress(100 * 1024 * 1024)
queue.put((address, event))
event.wait()
queue.put("exit")
thread.join()
def test_allocInOneThreadAndDeallocateInAnother(self):
queue = Queue.Queue()
thread = threading.Thread(target=memoryFreeingThreadLoop,
args=(queue, ))
thread.start()
#allocate 100 GB and free in the other thread
for ix in range(1000):
event = threading.Event()
address = TCMallocNative.mallocAndReturnAddress(100 * 1024 * 1024)
queue.put((address, event))
event.wait()
queue.put("exit")
thread.join()
def test_PythonIoTaskServiceInLoop(self):
bytesUsed = []
for ix in range(20):
bytesUsed.append(TCMallocNative.getMemoryStat("generic.current_allocated_bytes") / 1024 / 1024.0)
s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()
s3.setThroughputPerMachine(1024 * 1024 * 20)
for ix in range(35):
s3().setKeyValue(
"bucketname",
"key_%s" % ix,
" " * 10 * 1024 * 1024
)
text = """datasets.s3('bucketname', 'key_0').sum()"""
self.computeUsingSeveralWorkers(text, s3, 4, timeout = 120, blockUntilConnected=True)
self.assertTrue(bytesUsed[0] < bytesUsed[-1] - 100, bytesUsed)
def test_PythonIoTaskServiceInLoop(self):
bytesUsed = []
for ix in range(20):
bytesUsed.append(TCMallocNative.getMemoryStat("generic.current_allocated_bytes") / 1024 / 1024.0)
s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()
s3.setThroughputPerMachine(1024 * 1024 * 20)
for ix in range(35):
s3().setKeyValue(
"bucketname",
"key_%s" % ix,
" " * 10 * 1024 * 1024
)
text = """datasets.s3('bucketname', 'key_0').sum()"""
self.computeUsingSeveralWorkers(text, s3, 4, timeout = 120, blockUntilConnected=True)
self.assertTrue(bytesUsed[0] < bytesUsed[-1] - 100, bytesUsed)
def printTestResults(moduleName, testName, testResultWithTimes, verbose = False):
failct = 0
passct = 0
result, timeElapsed = testResultWithTimes
if result is not True:
failct += 1
print moduleName, ": ", testName,
print time.strftime("%Y-%m-%d %H:%M:%S"), "--", "TEST FAILED:", moduleName, testName, ":",
if isinstance(result, FORAException):
print "throw", sanitizeErrString(str(result.foraVal))
else:
print result,
print "is not True"
else:
passct += 1
if verbose:
print time.strftime("%Y-%m-%d %H:%M:%S"), "--", moduleName, testName, ": TEST SUCCEEDED in %2.4f" % timeElapsed, ". memory usage is ", TCMallocNative.getBytesUsed() / 1024 / 1024.0, " MB"
return (passct, failct)
def printTestResults(moduleName, testName, testResultWithTimes, verbose=False):
failct = 0
passct = 0
result, timeElapsed = testResultWithTimes
if result is not True:
failct += 1
print moduleName, ": ", testName,
print time.strftime("%Y-%m-%d %H:%M:%S"
), "--", "TEST FAILED:", moduleName, testName, ":",
if isinstance(result, FORAException):
print "throw", sanitizeErrString(str(result.foraVal))
else:
print result,
print "is not True"
else:
passct += 1
if verbose:
print time.strftime(
"%Y-%m-%d %H:%M:%S"
), "--", moduleName, testName, ": TEST SUCCEEDED in %2.4f" % timeElapsed, ". memory usage is ", TCMallocNative.getBytesUsed(
) / 1024 / 1024.0, " MB"
return (passct, failct)
def start(self):
with setupLock:
assert self.subprocessOutThread is None or not self.subprocessOutThread.is_alive(
)
stdInRead, stdInWrite = os.pipe()
stdOutRead, stdOutWrite = os.pipe()
stdErrRead, stdErrWrite = os.pipe()
self.subprocessStdIn = os.fdopen(stdInWrite, 'w', 1)
self.subprocessStdOut = os.fdopen(stdOutRead, 'r', 1)
self.subprocessStdErr = os.fdopen(stdErrRead, 'r', 1)
if self.enablePartialLineOutput:
# enable non-blocking reads
fcntl.fcntl(self.subprocessStdOut, fcntl.F_SETFL,
os.O_NONBLOCK)
fcntl.fcntl(self.subprocessStdErr, fcntl.F_SETFL,
os.O_NONBLOCK)
self.subprocessStdInFileDescriptor = stdInWrite
self.subprocessStdOutFileDescriptor = stdOutRead
self.subprocessStdErrFileDescriptor = stdErrRead
self.subprocessStdOutFromOtherSide = os.fdopen(stdOutWrite, 'w', 1)
self.subprocessStdErrFromOtherSide = os.fdopen(stdErrWrite, 'w', 1)
#start our reading threads BEFORE we open the process
self.subprocessOutThread = ManagedThread.ManagedThread(
target=self.processOutputLoop,
args=('stdOut', self.subprocessStdOut, self.onStdOut))
self.subprocessOutThread.start()
self.subprocessErrThread = ManagedThread.ManagedThread(
target=self.processOutputLoop,
args=('stdErr', self.subprocessStdErr, self.onStdErr))
self.subprocessErrThread.start()
logging.debug(
"SubprocessRunner subprocess.Popen call starting with arguments %s",
self.subprocessArguments)
subprocessEvent = threading.Event()
def startSubprocess():
self.process = subprocess.Popen(self.subprocessArguments,
stdin=stdInRead,
stdout=stdOutWrite,
stderr=stdErrWrite,
env=self.env)
subprocessEvent.set()
startSubprocessThread = ManagedThread.ManagedThread(
target=startSubprocess)
startSubprocessThread.start()
subprocessEvent.wait(10.0)
if not subprocessEvent.isSet():
import ufora.native.TCMalloc as TCMallocNative
logging.error("Failed to start subprocess: Total MB used: %s",
TCMallocNative.getBytesUsed() / 1024 / 1024.0)
assert subprocessEvent.isSet(
), "Failed to start the subprocess process."
os.close(stdInRead)
self.isStarted = True
# return self to allow chaining like: runner.start().wait(...)
return self
def start(self):
with setupLock:
assert self.subprocessOutThread is None or not self.subprocessOutThread.is_alive()
stdInRead, stdInWrite = os.pipe()
stdOutRead, stdOutWrite = os.pipe()
stdErrRead, stdErrWrite = os.pipe()
self.subprocessStdIn = os.fdopen(stdInWrite, 'w', 1)
self.subprocessStdOut = os.fdopen(stdOutRead, 'r', 1)
self.subprocessStdErr = os.fdopen(stdErrRead, 'r', 1)
if self.enablePartialLineOutput:
# enable non-blocking reads
fcntl.fcntl(self.subprocessStdOut, fcntl.F_SETFL, os.O_NONBLOCK)
fcntl.fcntl(self.subprocessStdErr, fcntl.F_SETFL, os.O_NONBLOCK)
self.subprocessStdInFileDescriptor = stdInWrite
self.subprocessStdOutFileDescriptor = stdOutRead
self.subprocessStdErrFileDescriptor = stdErrRead
self.subprocessStdOutFromOtherSide = os.fdopen(stdOutWrite, 'w', 1)
self.subprocessStdErrFromOtherSide = os.fdopen(stdErrWrite, 'w', 1)
#start our reading threads BEFORE we open the process
self.subprocessOutThread = ManagedThread.ManagedThread(
target=self.processOutputLoop,
args=('stdOut', self.subprocessStdOut, self.onStdOut)
)
self.subprocessOutThread.start()
self.subprocessErrThread = ManagedThread.ManagedThread(
target=self.processOutputLoop,
args=('stdErr', self.subprocessStdErr, self.onStdErr)
)
self.subprocessErrThread.start()
logging.debug("SubprocessRunner subprocess.Popen call starting with arguments %s",
self.subprocessArguments)
subprocessEvent = threading.Event()
def startSubprocess():
self.process = subprocess.Popen(
self.subprocessArguments,
stdin=stdInRead,
stdout=stdOutWrite,
stderr=stdErrWrite,
env=self.env
)
subprocessEvent.set()
startSubprocessThread = ManagedThread.ManagedThread(target=startSubprocess)
startSubprocessThread.start()
subprocessEvent.wait(10.0)
if not subprocessEvent.isSet():
import ufora.native.TCMalloc as TCMallocNative
logging.error("Failed to start subprocess: Total MB used: %s", TCMallocNative.getBytesUsed() / 1024 / 1024.0)
assert subprocessEvent.isSet(), "Failed to start the subprocess process."
os.close(stdInRead)
self.isStarted = True
# return self to allow chaining like: runner.start().wait(...)
return self
