def test():
logging.basicConfig(level=logging.DEBUG)
from dpark.context import DparkContext
dc = DparkContext("local")
dc.start()
nums = dc.parallelize(range(100), 10)
tracker = CacheTracker(True)
tracker.registerRDD(nums.id, len(nums))
split = nums.splits[0]
print list(tracker.getOrCompute(nums, split))
print list(tracker.getOrCompute(nums, split))
print tracker.getLocationsSnapshot()
tracker.stop()
def test():
logging.basicConfig(level=logging.DEBUG)
from dpark.context import DparkContext
dc = DparkContext("local")
dc.start()
nums = dc.parallelize(range(100), 10)
tracker = CacheTracker(True)
tracker.registerRDD(nums.id, len(nums))
split = nums.splits[0]
print list(tracker.getOrCompute(nums, split))
print list(tracker.getOrCompute(nums, split))
print tracker.getLocationsSnapshot()
tracker.stop()
def test():
logging.basicConfig(level=logging.DEBUG)
cache = mmapCache
pool = multiprocessing.Pool(2)
assert pool.apply(set_cache) == True
assert pool.apply(get_cache) == 'b'
pool.close()
pool.join()
assert cache.get('a') == 'b'
from dpark.context import DparkContext
dc = DparkContext("local")
nums = dc.parallelize(range(100), 10)
cache = mmapCache
tracker = CacheTracker(True)
tracker.registerRDD(nums.id, len(nums))
split = nums.splits[0]
print tracker.getOrCompute(nums, split)
print tracker.getOrCompute(nums, split)
print tracker.getLocationsSnapshot()
tracker.stop()