def test_P4P(self):
minion_uploader = Minions('res_file.tgz',
upload_rate=150 * 1024**2,
tracker=TRACKER)
minion_uploader.upload_res('./res_file.tgz')
time.sleep(1)
requests.get('http://localhost:5000/peer/?res=res_file.tgz')
minions = []
try:
for i in range(1):
minion = Minions('res_file.tgz',
'/tmp/res_file%s.tgz' % i,
tracker=TRACKER)
minions.append(minion)
minion.download_res(rate=1500 * 1024**2)
time.sleep(0.1)
time.sleep(2)
for m in minions:
m.wait_for_res()
except Exception:
import traceback
traceback.print_exc()
raise
finally:
for m in minions:
m.close()
minion_uploader.close()
def get_minion(args):
if args.data_stdout:
disorder = False
else:
disorder = True
if args.ignore_qp:
upload_res_url = strip_url_qp(args.url, args.ignore_qp)
minion = Minions(args.url,
download_dest=args.dest_path,
download_rate=args.download_rate,
upload_rate=args.upload_rate,
upload_res_url=upload_res_url,
fallback=args.fallback,
tracker=args.tracker,
disorder=disorder)
else:
minion = Minions(args.url,
download_dest=args.dest_path,
download_rate=args.download_rate,
upload_rate=args.upload_rate,
fallback=args.fallback,
tracker=args.tracker,
disorder=disorder)
return minion
def test_no_peers(self):
with self.assertRaises(NoPeersFound):
minion_1 = Minions('res_file.tgz',
'/tmp/minions_1.tgz',
tracker=TRACKER)
minion_1.download_res()
def test_P4P_ignore_url_param(self):
minion_uploader = Minions('res_file.tgz', tracker=TRACKER)
minion_uploader.upload_res('./res_file.tgz')
time.sleep(1)
requests.get('http://localhost:5000/peer/?res=res_file.tgz')
minions = []
try:
for i in range(3):
minion = Minions('res_file.tgz?a=%s&b=%s' % (i, i),
'/tmp/res_file%s.tgz' % i,
tracker=TRACKER,
upload_res_url="res_file.tgz")
minion.download_res(rate=20 * 1024**2, thread=True)
time.sleep(0.1)
minions.append(minion)
time.sleep(2)
for m in minions:
m.wait_for_res()
except Exception:
import traceback
traceback.print_exc()
raise
finally:
for m in minions:
m.close()
minion_uploader.close()
def test_keepalive(self):
minion_uploader = Minions('res_file.tgz', tracker=TRACKER)
minion_uploader.upload_res('./res_file.tgz')
time.sleep(1)
port = minion_uploader.port
try:
# test if whether get block api interface works
r = requests.get(
"http://localhost:%s/?res_url=res_file.tgz&pieces=all" % port,
stream=True,
headers={"Range": "bytes=0-1"},
timeout=1)
print r.content
s = socket.socket()
s.settimeout(0.2)
s.connect(('localhost', port))
s.send("GET /?res_url=res_file.tgz HTTP/1.1\r\n"
"Host: localhost:62630\r\n"
"Accept-Encoding: identity\r\n"
"Content-Length: 0\r\nRange: bytes=0-1,3-4\r\n"
"\r\n")
print s.recv(200)
s.recv(1024**2 + 400)
with self.assertRaises(socket.timeout):
s.recv(1024)
s.send("GET /?res_url=res_file.tgz HTTP/1.1\r\n"
"Host: localhost:62630\r\n"
"Accept-Encoding: identity\r\n"
"Content-Length: 0\r\n"
"Range: bytes=0-1,3-4\r\n"
"\r\n")
print s.recv(200)
# conn = httplib.HTTPConnection("localhost", port)
# conn.debuglevel = 1
# conn.request(
# "GET", "/?res_url=res_file.tgz",
# "", headers={"Range": "bytes=0-1,3-4"})
# r = conn.getresponse()
finally:
minion_uploader.close()
minion_uploader.stop_upload()
def test_multipart(self):
minion_uploader = Minions('res_file.tgz', tracker=TRACKER)
minion_uploader.upload_res('./res_file.tgz')
time.sleep(1)
port = minion_uploader.port
try:
r = requests.get("http://localhost:%s/?res_url=res_file.tgz" %
port,
headers={"Range": "bytes=0-1,3-4"})
self.assertEqual(len(r.content), 4)
finally:
minion_uploader.close()
minion_uploader.stop_upload()
def _func():
minion_uploader = Minions('res_file.tgz', tracker=TRACKER)
minion_uploader.upload_res('./res_file.tgz')
while True:
time.sleep(1000)
def test_runout_peer(self):
minion_uploader = Minions('res_file.tgz', tracker=TRACKER)
minion_uploader.upload_res('./res_file.tgz')
time.sleep(1)
requests.get('http://localhost:5000/peer/?res=res_file.tgz')
with self.assertRaises(NoPeersFound):
try:
minion_1 = Minions('res_file.tgz',
'/tmp/res_file.tgz',
tracker=TRACKER)
minion_1.download_res(rate=10 * 1024**2, thread=True)
time.sleep(2)
minion_uploader.close()
minion_1.wait_for_res()
except Exception as e:
import traceback
traceback.print_exc()
raise e
finally:
minion_1.close()
minion_uploader.close()
def test_get_peer_strict(self):
minion_uploader = Minions('res_file.tgz', tracker=TRACKER)
minion_uploader.set_adt_info({'site': 'mysite1'})
minion_uploader.upload_res('./res_file.tgz')
minion_2 = None
try:
with self.assertRaises(NoPeersFound):
minion_1 = Minions('res_file.tgz',
'/tmp/res_file1.tgz',
strict='site',
tracker=TRACKER)
minion_1.set_adt_info({'site': 'mysite2'})
minion_1.download_res()
minion_2 = Minions('res_file.tgz',
'/tmp/res_file2.tgz',
strict='site',
tracker=TRACKER)
minion_2.set_adt_info({'site': 'mysite1'})
minion_2.download_res()
finally:
minion_uploader.close()
minion_1.close()
if minion_2:
minion_2.close()
def test_fallback(self):
import BaseHTTPServer
from SimpleHTTPServer import SimpleHTTPRequestHandler
server_address = ('', 8000)
httpd = BaseHTTPServer.HTTPServer(server_address,
SimpleHTTPRequestHandler)
t = Thread(target=httpd.serve_forever)
t.start()
try:
minion_1 = Minions('http://localhost:8000/res_file.tgz',
'/tmp/res_file.tgz',
fallback=True,
tracker=TRACKER)
minion_1.download_res()
time.sleep(1)
minion_2 = Minions('http://localhost:8000/res_file.tgz',
'/tmp/res_file1.tgz',
tracker=TRACKER)
minion_2.download_res(rate=20 * 1024**2)
finally:
minion_1.close()
minion_2.close()
httpd.shutdown()
def test_keepalive_timeout(self):
minion_uploader = Minions('res_file.tgz', tracker=TRACKER)
minion_uploader.upload_res('./res_file.tgz')
time.sleep(1)
port = minion_uploader.port
import logging
logging.basicConfig(level=logging.ERROR)
def foo(session):
size = 1024**2
r = session.get("http://localhost:%s/?res_url=res_file.tgz" % port,
stream=True,
headers={"Range": "bytes=0-%s" % (size)},
timeout=2)
# r.raw.close()
r.content
return r
try:
s = requests.Session()
g = []
for i in range(500):
g.append(gevent.spawn(foo, s))
gevent.sleep(1)
for i in g:
i.get()
# print "------------"
# r = s.get("http://localhost:%s/?res_url=res_file.tgz" % port,
# stream=True,
# headers={"Range": "bytes=0-%s" %(size)},
# timeout=1)
# print len(r.raw.read(1023))
# len(r.raw.read(1024**2 + 1))
# time.sleep(1)
# r.content
# print r
# r.raw.close()
# r.close()
# r.raw.close()
# del r
# r = s.get(
# "http://localhost:%s/?res_url=res_file.tgz&pieces=all" %
# port,
# stream=True,
# headers={"Range": "bytes=0-%s" % (size)},
# timeout=1)
# #print len(r.raw.read(1024**2 * 2 + 1))
# r.close()
# r.content
# r = s.get(
# "http://localhost:%s/?res_url=res_file.tgz&pieces=all"
# % port,
# stream=True,
# headers={"Range": "bytes=0-%s" % (size)},
# timeout=1)
# r.content
# r.close()
# print "------------"
finally:
print 'finally'
minion_uploader.close()
minion_uploader.stop_upload()