from dpark.env import env
env.start()
path = LocalFileShuffle.getOutputFile(1, 0, 0)
d = compress(pickle.dumps({'key': 'value'}, -1))
f = open(path, 'w')
f.write(pack_header(len(d), False, False) + d)
f.close()
uri = LocalFileShuffle.getServerUri()
env.mapOutputTracker.registerMapOutputs(1, [uri])
fetcher = SimpleShuffleFetcher()
def func(it):
k, v = next(it)
assert k == 'key'
assert v == 'value'
fetcher.fetch(1, 0, func)
tracker = MapOutputTracker()
tracker.registerMapOutputs(2, [None, uri, None, None, None])
assert tracker.getServerUris(2) == [None, uri, None, None, None]
tracker.stop()
if __name__ == '__main__':
from dpark.shuffle import test
test()
random.shuffle(d)
l.append(SortedItems(d))
hl = heap_merged(l, lambda x, y: x + y, MAX_SHUFFLE_MEMORY)
for i in range(10):
print i, hl.next()
path = LocalFileShuffle.getOutputFile(1, 0, 0)
d = compress(cPickle.dumps({'key': 'value'}, -1))
f = open(path, 'w')
f.write('p' + struct.pack('I', 5 + len(d)) + d)
f.close()
uri = LocalFileShuffle.getServerUri()
env.mapOutputTracker.registerMapOutputs(1, [uri])
fetcher = SimpleShuffleFetcher()
def func(it):
k, v = next(it)
assert k == 'key'
assert v == 'value'
fetcher.fetch(1, 0, func)
tracker = MapOutputTracker()
tracker.registerMapOutputs(2, [None, uri, None, None, None])
assert tracker.getServerUris(2) == [None, uri, None, None, None]
tracker.stop()
if __name__ == '__main__':
from dpark.shuffle import test
test()