def test_single():
# filesize, piece size pairs
pairs = [
(3, 20),
(20, 17),
(10000, 32),
(100, 20),
(1, 131072),
]
for pair in pairs:
pm = PieceManager(FileSet([('hello_world', pair[0])], pair[1]))
pm.initialize()
chunks = list(pm.iter_pieces())
hashes = list(pm.iter_hashes())
try:
assert len(chunks) == int(ceil(1.*pair[0]/pair[1]))
assert len(hashes) == int(ceil(1.*pair[0]/pair[1]))
for chunk in chunks[:-1]:
assert len(chunk) == pair[1]
_, leftover = divmod(pair[0], pair[1])
if leftover:
assert len(chunks[-1]) == leftover
for hash in hashes:
assert len(hash) == 20
finally:
pm.destroy()
def test_many():
d1 = tempfile.mkdtemp()
d2 = tempfile.mkdtemp()
d3 = tempfile.mkdtemp()
deadbeef = struct.pack('>I', 0xDEADBEEF)
with open(os.path.join(d1, 'one.txt'), 'w') as fp:
for k in range(1):
fp.write(deadbeef)
with open(os.path.join(d1, 'two.txt'), 'w') as fp:
for k in range(2):
fp.write(deadbeef)
with open(os.path.join(d1, 'three.txt'), 'w') as fp:
for k in range(3):
fp.write(deadbeef)
with open(os.path.join(d2, 'two.txt'), 'w') as fp:
for k in range(2):
fp.write(deadbeef)
with open(os.path.join(d2, 'four.txt'), 'w') as fp:
for k in range(4):
fp.write(deadbeef)
with open(os.path.join(d3, 'six.txt'), 'w') as fp:
for k in range(6):
fp.write(deadbeef)
pm1 = PieceManager(
FileSet([('one.txt', 4), ('two.txt', 8), ('three.txt', 12)], piece_size=13),
chroot=d1)
pm2 = PieceManager(FileSet([('two.txt', 8), ('four.txt', 16)], piece_size=13), chroot=d2)
pm3 = PieceManager(FileSet([('six.txt', 24)], piece_size=13), chroot=d3)
pm1.initialize()
pm2.initialize()
pm3.initialize()
try:
assert list(pm1.iter_pieces()) == list(pm2.iter_pieces())
assert list(pm1.iter_pieces()) == list(pm3.iter_pieces())
assert list(pm1.iter_hashes()) == list(pm2.iter_hashes())
assert list(pm1.iter_hashes()) == list(pm3.iter_hashes())
finally:
pm1.destroy()
pm2.destroy()
pm3.destroy()
def test_empty_input():
pm = PieceManager(FileSet((), 1))
pm.initialize()
chunks = list(pm.iter_pieces())
hashes = list(pm.iter_hashes())
try:
assert chunks == []
assert hashes == []
finally:
pm.destroy()
def test_main(self):
piece_size = 4
filelist = (('a.txt', b'hello'), ('b.txt', b'world'), ('c.txt', b'hello world'))
td, fs, metainfo = make_metainfo(filelist, piece_size)
pm = PieceManager(fs, chroot=td)
pm.initialize()
log.info('pb1')
pb1 = PieceBroker(
fs,
chroot=td,
piece_hashes=list(pm.iter_hashes()),
io_loop=self.io_loop)
pb1.initialize()
log.info('pb2')
pb2 = PieceBroker(
fs,
piece_hashes=list(pm.iter_hashes()),
io_loop=self.io_loop)
pb2.initialize()
# make sure pb1 is complete and pb2 is not
for k in range(fs.num_pieces):
assert pb1.whole_piece(k) in pb1
assert pb1.whole_piece(k) not in pb2
a_txt = b''.join((yield [
gen.Task(pb1.read, Request(0, 0, 4)),
gen.Task(pb1.read, Request(1, 0, 1))]))
b_txt = b''.join((yield [
gen.Task(pb1.read, Request(1, 1, 3)),
gen.Task(pb1.read, Request(2, 0, 2))]))
c_txt = yield gen.Task(pb1.read, Request(2, 2, 11))
assert a_txt == b'hello'
assert b_txt == b'world'
assert c_txt == b'hello world'
# Test request iter
assert list(pm.iter_blocks(0, 1)) == [
Request(0, 0, 1), Request(0, 1, 1), Request(0, 2, 1), Request(0, 3, 1)]
assert list(pm.iter_blocks(0, 2)) == [Request(0, 0, 2), Request(0, 2, 2)]
assert list(pm.iter_blocks(0, 3)) == [Request(0, 0, 3), Request(0, 3, 1)]
assert list(pm.iter_blocks(0, 4)) == [Request(0, 0, 4)]
assert list(pm.iter_blocks(0, 5)) == [Request(0, 0, 4)]
# Test iter of last piece
assert list(pm.iter_blocks(5, 1)) == [Request(5, 0, 1)]
assert list(pm.iter_blocks(5, 4)) == [Request(5, 0, 1)]
# Write hello
assert pb2.assembled_size == 0
assert Request(0, 0, 4) not in pb2
yield gen.Task(pb2.write, Piece(0, 0, 5, b'hello'))
assert Request(0, 0, 4) in pb2
assert pb2.assembled_size == 4
# swallow dupe
yield gen.Task(pb2.write, Piece(0, 0, 3, b'hel'))
assert Request(0, 0, 4) in pb2
# only assert pieces are in if we've validated sha
assert Request(1, 0, 1) not in pb2
# but the slice is still in there
assert pb2.to_slice(Request(1, 0, 1)) in pb2.slices
# write garbage and Request(1, 0, 1) slice goes away b/c the whole extent is bad
assert pb2.assembled_size == 4
yield gen.Task(pb2.write, Piece(1, 1, 3, b'poo'))
assert pb2.to_slice(Request(1, 0, 1)) not in pb2.slices
assert pb2.assembled_size == 4
# write owo, rld, to make sure partial chunk is not computed
yield [gen.Task(pb2.write, Piece(1, 0, 3, b'owo')),
gen.Task(pb2.write, Piece(1, 3, 3, b'rld'))]
assert Request(1, 0, 4) in pb2
assert pb2.assembled_size == 8
"""
TODO(wickman) Fix the logic so that this works.
yield gen.Task(pb2.write, Piece(0, 0, 21, b'helloworldhello world'))
"""
yield gen.Task(pb2.write, Piece(5, 0, 1, b'd'))
assert pb2.assembled_size == 9
yield [
gen.Task(pb2.write, Piece(0, 0, 4, b'hell')),
gen.Task(pb2.write, Piece(1, 0, 4, b'owor')),
gen.Task(pb2.write, Piece(2, 0, 4, b'ldhe')),
gen.Task(pb2.write, Piece(3, 0, 4, b'llo ')),
gen.Task(pb2.write, Piece(4, 0, 4, b'worl'))]
assert pb2.assembled_size == 21
for k in range(fs.num_pieces):
assert pb2.whole_piece(k) in pb2
# the safe_mkdtemp will auto destroy but do this to test the
# code path.
pb2.destroy()