def test_read_alligned(self, delay=0.001, piece_size=1024, read_block=2000):
size=TEST_FILE_SIZE
client=DummyClient(self.fname, delay)
bt = HTFile(self.fname, './', size, piece_size, client.request)
self.assertEqual(bt.last_piece,14)
client.serve(bt)
buf=StringIO()
c=bt.create_cursor()
with open(self.fname, 'rb') as inf:
ofs=0
while True:
sz=read_block
res=c.read(sz)
inf.seek(ofs)
if res:
ch=inf.read(len(res))
self.assertEqual(len(res), len(ch))
self.assertEqual(res,ch, msg="Unequal ot ofs %d"%ofs)
ofs+=len(ch)
buf.write(res)
else:
break
with open(self.fname, 'rb') as f:
#f.seek(1 * piece_size + 700)
ref=f.read(size)
self.assertTrue(bt.is_complete)
self.assertEqual(bt.downloaded, size)
self.assertEqual(len(ref), len(buf.getvalue()))
self.assertEqual(ref, buf.getvalue())
c.close()
def test_reopen(self):
bt = self.test_seek()
bt.close()
size = TEST_FILE_SIZE
def no(a, b):
raise Exception('Should not be called!')
bt = HTFile(self.tname, self.base, size, 512, no)
buf = StringIO()
c = bt.create_cursor()
c.seek(555)
while True:
sz = 1024
res = c.read(sz)
if res:
buf.write(res)
else:
break
with open(self.fname, 'rb') as f:
f.seek(555)
ref = f.read(size)
self.assertEqual(len(ref), len(buf.getvalue()))
self.assertEqual(ref, buf.getvalue())
c.close()
def test_seek_end(self):
size=TEST_FILE_SIZE
client=DummyClient(self.fname)
bt = HTFile(self.fname, './', size, 1024, client.request)
self.assertEqual(bt.last_piece, 14)
client.serve(bt)
buf=StringIO()
c=bt.create_cursor()
c.seek(size-64)
while True:
sz=8
res=c.read(sz)
if res:
buf.write(res)
else:
break
with open(self.fname, 'rb') as f:
f.seek(size-64)
ref=f.read(64)
self.assertTrue(not bt.is_complete)
self.assertEqual(bt.downloaded, 1024)
self.assertEqual(len(ref), len(buf.getvalue()))
self.assertEqual(ref, buf.getvalue())
c.close()
def test(self):
f = DummyFile(fname)
piece_size = 1024
c = HTTPLoader(URL, 0)
c.init()
first = c.load_piece(0, piece_size)
self.assertEqual(len(first.data), piece_size)
self.assertEqual(first.piece, 0)
self.assertEqual(first.type, 'video/x-msvideo')
self.assertEqual(first.total_size, f.size)
piece_size = 2048 * 1024
last_piece = f.size // piece_size
buf = StringIO()
tmp_file = tempfile.mktemp()
base, tmp_file = os.path.split(tmp_file)
htfile = HTFile(tmp_file, base, f.size, piece_size,
lambda a, b: self.fail('Should not need prioritize'))
for piece in xrange(last_piece + 1):
print "Piece %d" % piece
p = c.load_piece(piece, piece_size)
self.assertTrue(p.data)
if piece != last_piece:
self.assertEqual(len(p.data), piece_size)
if piece == last_piece:
self.assertEqual(len(p.data), f.size % piece_size)
buf.write(p.data)
htfile.update_piece(piece, p.data)
data = buf.getvalue()
with f.create_cursor() as reader:
ref = reader.read()
self.assertEqual(data, ref, "Diffrent")
buf = StringIO()
with htfile.create_cursor() as reader:
while True:
d = reader.read()
if not d:
break
buf.write(d)
data2 = buf.getvalue()
self.assertEqual(len(data2), len(ref))
self.assertEqual(data2, ref, "Different")
def test_seek2(self):
size = TEST_FILE_SIZE
client = DummyClient(self.fname)
bt = HTFile(self.tname, self.base, size, 768, client.request)
client.serve(bt)
c = bt.create_cursor()
c.seek(555)
buf = StringIO()
while True:
sz = 1024
res = c.read(sz)
if res:
buf.write(res)
else:
break
with open(self.fname, 'rb') as f:
f.seek(555)
ref = f.read(size)
self.assertEqual(len(ref), len(buf.getvalue()))
self.assertEqual(ref, buf.getvalue())
buf = StringIO()
c.seek(10000)
while True:
sz = 512
res = c.read(sz)
if res:
buf.write(res)
else:
break
with open(self.fname, 'rb') as f:
f.seek(10000)
ref = f.read(size)
self.assertEqual(len(ref), len(buf.getvalue()))
self.assertEqual(ref, buf.getvalue())
with open(self.fname, 'rb') as f:
for _i in xrange(10):
seek = random.randint(0, size)
c.seek(seek)
res = c.read(100)
f.seek(seek)
ref = f.read(len(res))
self.assertTrue(res, ref)
c.close()
def test_seek2(self):
size=TEST_FILE_SIZE
client=DummyClient(self.fname)
bt = HTFile(self.tname, self.base, size, 768, client.request)
client.serve(bt)
c=bt.create_cursor()
c.seek(555)
buf=StringIO()
while True:
sz=1024
res=c.read(sz)
if res:
buf.write(res)
else:
break
with open(self.fname, 'rb') as f:
f.seek(555)
ref=f.read(size)
self.assertEqual(len(ref), len(buf.getvalue()))
self.assertEqual(ref, buf.getvalue())
buf=StringIO()
c.seek(10000)
while True:
sz=512
res=c.read(sz)
if res:
buf.write(res)
else:
break
with open(self.fname, 'rb') as f:
f.seek(10000)
ref=f.read(size)
self.assertEqual(len(ref), len(buf.getvalue()))
self.assertEqual(ref, buf.getvalue())
with open(self.fname, 'rb') as f:
for _i in xrange(10):
seek=random.randint(0,size)
c.seek(seek)
res=c.read(100)
f.seek(seek)
ref=f.read(len(res))
self.assertTrue(res,ref)
c.close()
def test(self):
f=DummyFile(fname)
piece_size=1024
c=HTTPLoader(URL, 0)
c.init()
first=c.load_piece(0, piece_size)
self.assertEqual(len(first.data), piece_size)
self.assertEqual(first.piece,0)
self.assertEqual(first.type,'video/x-msvideo')
self.assertEqual(first.total_size, f.size)
piece_size=2048*1024
last_piece=f.size // piece_size
buf=StringIO()
tmp_file=tempfile.mktemp()
base,tmp_file=os.path.split(tmp_file)
htfile=HTFile(tmp_file, base, f.size, piece_size, lambda a,b: self.fail('Should not need prioritize'))
for piece in xrange(last_piece+1):
print "Piece %d"%piece
p=c.load_piece(piece, piece_size)
self.assertTrue(p.data)
if piece!=last_piece:
self.assertEqual(len(p.data), piece_size)
if piece==last_piece:
self.assertEqual(len(p.data), f.size % piece_size)
buf.write(p.data)
htfile.update_piece(piece, p.data)
data=buf.getvalue()
with f.create_cursor() as reader:
ref=reader.read()
self.assertEqual(data, ref, "Diffrent")
buf=StringIO()
with htfile.create_cursor() as reader:
while True:
d=reader.read()
if not d:
break
buf.write(d)
data2=buf.getvalue()
self.assertEqual(len(data2),len(ref))
self.assertEqual(data2, ref, "Different")
def test(self):
f = DummyFile(fname)
piece_size = 1024
c = HTTPLoader(URL, 0)
c.init()
first = c.load_piece(0, piece_size)
self.assertEqual(len(first.data), piece_size)
self.assertEqual(first.piece, 0)
self.assertEqual(first.type, 'video/x-msvideo')
self.assertEqual(first.total_size, f.size)
piece_size = 2048 * 1024 / 2
last_piece = f.size // piece_size
tmp_file = tempfile.mktemp()
base, tmp_file = os.path.split(tmp_file)
htfile = HTFile(tmp_file, base, f.size, piece_size,
lambda a, b: self.fail('Should not need prioritize'))
pool = Pool(piece_size, [c],
htfile.update_piece,
speed_limit=MyResolver.SPEED_LIMIT)
def gen_loader(url, i):
return HTTPLoader(url, i)
for i in xrange(1, 3):
pool.add_worker_async(i, gen_loader, (URL, i))
for i in xrange(last_piece + 1):
pool.add_piece(i, 10 - i)
while not htfile.is_complete:
time.sleep(1)
print htfile.pieces
with f.create_cursor() as reader:
ref = reader.read()
buf = StringIO()
with htfile.create_cursor() as reader:
while True:
d = reader.read()
if not d:
break
buf.write(d)
data2 = buf.getvalue()
self.assertEqual(len(data2), len(ref))
self.assertEqual(data2, ref, "Different")
def test_clone(self, close_first=False):
size=TEST_FILE_SIZE
client=DummyClient(self.fname)
bt = HTFile(self.tname, self.base, size, 512, client.request)
client.serve(bt)
c=bt.create_cursor()
c.seek(5000)
c.read(100)
sleep(0.1)
self.assertTrue(all(c._cache._cache))
if close_first:
c.close()
c2=bt.create_cursor(offset=5000)
self.assertTrue(all(c2._cache._cache))
def no(*args):
raise Exception('Should not be called')
client.request=no
c2.seek(5000)
c2.read(100)
def test(self):
f=DummyFile(fname)
piece_size=1024
c=HTTPLoader(URL, 0)
first=c.load_piece(0, piece_size)
self.assertEqual(len(first.data), piece_size)
self.assertEqual(first.piece,0)
self.assertEqual(first.type,'video/x-msvideo')
self.assertEqual(first.total_size, f.size)
piece_size=2048*1024/2
last_piece=f.size // piece_size
tmp_file=tempfile.mktemp()
base,tmp_file=os.path.split(tmp_file)
htfile=HTFile(tmp_file, base, f.size, piece_size, lambda a,b: self.fail('Should not need prioritize'))
pool=Pool(piece_size, [c], htfile.update_piece, speed_limit=MyResolver.SPEED_LIMIT)
def gen_loader(url,i):
return HTTPLoader(url, i)
for i in xrange(1,3):
pool.add_worker_async(i, gen_loader, (URL,i))
for i in xrange(last_piece+1):
pool.add_piece(i, 10-i)
while not htfile.is_complete:
time.sleep(1)
print htfile.pieces
with f.create_cursor() as reader:
ref=reader.read()
buf=StringIO()
with htfile.create_cursor() as reader:
while True:
d=reader.read()
if not d:
break
buf.write(d)
data2=buf.getvalue()
self.assertEqual(len(data2),len(ref))
self.assertEqual(data2, ref, "Different")
def test_clone(self, close_first=False):
size = TEST_FILE_SIZE
client = DummyClient(self.fname)
bt = HTFile(self.tname, self.base, size, 512, client.request)
client.serve(bt)
c = bt.create_cursor()
c.seek(5000)
c.read(100)
sleep(0.1)
self.assertTrue(all(c._cache._cache))
if close_first:
c.close()
c2 = bt.create_cursor(offset=5000)
self.assertTrue(all(c2._cache._cache))
def no(*args):
raise Exception('Should not be called')
client.request = no
c2.seek(5000)
c2.read(100)
def test_seek(self):
size = TEST_FILE_SIZE
client = DummyClient(self.fname)
bt = HTFile(self.tname, self.base, size, 512, client.request)
client.serve(bt)
buf = StringIO()
c = bt.create_cursor()
c.seek(555)
while True:
sz = 1024
res = c.read(sz)
if res:
buf.write(res)
else:
break
with open(self.fname, 'rb') as f:
f.seek(555)
ref = f.read(size)
self.assertEqual(len(ref), len(buf.getvalue()))
self.assertEqual(ref, buf.getvalue())
c.close()
return bt
def test_reopen(self):
bt=self.test_seek()
bt.close()
size=TEST_FILE_SIZE
def no(a,b):
raise Exception('Should not be called!')
bt = HTFile(self.tname, self.base, size, 512, no)
buf=StringIO()
c=bt.create_cursor()
c.seek(555)
while True:
sz=1024
res=c.read(sz)
if res:
buf.write(res)
else:
break
with open(self.fname, 'rb') as f:
f.seek(555)
ref=f.read(size)
self.assertEqual(len(ref), len(buf.getvalue()))
self.assertEqual(ref, buf.getvalue())
c.close()
def test_seek(self):
size=TEST_FILE_SIZE
client=DummyClient(self.fname)
bt = HTFile(self.tname, self.base, size, 512, client.request)
client.serve(bt)
buf=StringIO()
c=bt.create_cursor()
c.seek(555)
while True:
sz=1024
res=c.read(sz)
if res:
buf.write(res)
else:
break
with open(self.fname, 'rb') as f:
f.seek(555)
ref=f.read(size)
self.assertEqual(len(ref), len(buf.getvalue()))
self.assertEqual(ref, buf.getvalue())
c.close()
return bt
