def test_PythonIoTaskServiceInLoop(self):
bytesUsed = []
for ix in range(20):
bytesUsed.append(TCMallocNative.getMemoryStat("generic.current_allocated_bytes") / 1024 / 1024.0)
s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()
s3.setThroughputPerMachine(1024 * 1024 * 20)
for ix in range(35):
s3().setKeyValue(
"bucketname",
"key_%s" % ix,
" " * 10 * 1024 * 1024
)
text = """datasets.s3('bucketname', 'key_0').sum()"""
self.computeUsingSeveralWorkers(text, s3, 4, timeout = 120, blockUntilConnected=True)
self.assertTrue(bytesUsed[0] < bytesUsed[-1] - 100, bytesUsed)
def test_PythonIoTaskServiceInLoop(self):
bytesUsed = []
for ix in range(20):
bytesUsed.append(TCMallocNative.getMemoryStat("generic.current_allocated_bytes") / 1024 / 1024.0)
s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()
s3.setThroughputPerMachine(1024 * 1024 * 20)
for ix in range(35):
s3().setKeyValue(
"bucketname",
"key_%s" % ix,
" " * 10 * 1024 * 1024
)
text = """datasets.s3('bucketname', 'key_0').sum()"""
self.computeUsingSeveralWorkers(text, s3, 4, timeout = 120, blockUntilConnected=True)
self.assertTrue(bytesUsed[0] < bytesUsed[-1] - 100, bytesUsed)
