def _testUploadDownload(self, vhd_filename, vdi_mb, chunksize):
record = put_vhd(self, vhd_filename, vdi_mb, self.VDI_RAW)
protocol = "http"
dest = "download-frag-%s.vhd" % (vdi_mb)
logging.debug("About to download " + record['url_path'])
logging.debug("Fagment size is %d" % chunksize)
bits.download(protocol,record['ip'] + ":80" , record['url_path'] + '.vhd' ,dest, record['username'], record['password'], chunksize)
vhd.diff(vhd_filename, dest)
return record
def _testUploadDownload(self, vhd_filename, vdi_mb):
#Setup record for the VDI we're wanting to download
record = put_vhd(self, vhd_filename, vdi_mb, self.VDI_RAW)
protocol = "http"
dest = "download-%s.vhd" % (vdi_mb)
logging.debug("About to download " + record['url_path'])
bits.download(protocol,record['ip'] + ":80" , record['url_path'] ,dest, record['username'], record['password'])
vhd.diff(vhd_filename, dest)
return record
def _testUploadDownload(self, vhd_filename, vdi_mb):
#Setup record for the VDI we're wanting to download
record = put_vhd(self, vhd_filename, vdi_mb, self.VDI_RAW)
protocol = "http"
dest = "download-%s.vhd" % (vdi_mb)
logging.debug("About to download " + record['url_path'])
bits.download(protocol, record['ip'] + ":80", record['url_path'], dest,
record['username'], record['password'])
vhd.diff(vhd_filename, dest)
return record
def _testUploadDownload(self, vhd_filename, vdi_mb, chunksize):
record = put_vhd(self, vhd_filename, vdi_mb, self.VDI_RAW)
protocol = "http"
dest = "download-frag-%s.vhd" % (vdi_mb)
logging.debug("About to download " + record['url_path'])
logging.debug("Fagment size is %d" % chunksize)
bits.download(protocol, record['ip'] + ":80",
record['url_path'] + '.vhd', dest, record['username'],
record['password'], chunksize)
vhd.diff(vhd_filename, dest)
return record
def do_vhd_put_get_test(self, vdi_mb, vdi_raw, pattern, blocks = None):
MAX_FRAG_SIZE = 100*K
TMP_RAW_RESPONSE_FILE = "response.tmp"
TMP_VHD_RESPONSE_FILE = "response.vhd"
REFERENCE_FILE = "reference.tmp"
VHD_FILE = "tmp.vhd"
bitmap = None
vhd.create(VHD_FILE, vdi_mb)
vhd.fill(VHD_FILE, REFERENCE_FILE, vdi_mb, pattern)
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=vdi_mb, vdi_raw=vdi_raw)
args = {'vdi_uuid': vdi, 'network_uuid': network, 'transfer_mode': 'http'}
if blocks:
bitmap = vhd.to_bitmap(blocks, vdi_mb * M)
args['vhd_blocks'] = base64.b64encode(zlib.compress(bitmap))
args['vhd_uuid'] = vdi #vhd uuid needed for get - using vdi value for test purposes
transferclient.expose(hostname, **args)
record = transferclient.get_record(hostname, vdi_uuid=vdi)
url_path = record['url_path']
# test "put"
f = open(VHD_FILE, 'r')
put_data = f.read()
f.close()
record['url_path'] = url_path + ".vhd"
put_status, put_headers = http_put(record, put_data, None, vdi_mb * M)
self.assertEqual(put_status, 200)
record['url_path'] = url_path
get_status, get_headers, get_respdata = http_get(record)
if len(get_respdata) > vdi_mb * M:
# raw VDIs can be larger than requested size
get_respdata = get_respdata[:vdi_mb * M]
f = open(TMP_RAW_RESPONSE_FILE, 'w')
f.write(get_respdata)
f.close()
cmd = "diff %s %s" % (TMP_RAW_RESPONSE_FILE, REFERENCE_FILE)
util.doexec(cmd, 0)
# test "get"
record['url_path'] = url_path + ".vhd"
frag_size = random.randint(1, MAX_FRAG_SIZE)
range_start = 0
range_end = frag_size
head_status, head_headers = http_head(record)
vhd_size = int(head_headers['content-length'])
total = vhd_size
total_received = 0
total_chunks = 0
f = open(TMP_VHD_RESPONSE_FILE, 'w')
while range_start < total:
if range_end > total:
range_end = total
#print "Getting range %d-%d" % (range_start, range_end)
get_status, get_headers, get_respdata = http_get(record, content_range=(range_start, range_end, total))
self.assertEquals(get_status, 200)
#print "Got response of length %d" % len(get_respdata)
total_received += len(get_respdata)
total_chunks += 1
f.write(get_respdata)
range_start = range_end
frag_size = random.randint(1, MAX_FRAG_SIZE)
range_end += frag_size
f.close()
#print "Got total length %d in %d chunks" % (total_received, total_chunks)
if blocks:
vhd.mask(REFERENCE_FILE, vdi_mb * M, blocks)
vhd.extract(TMP_VHD_RESPONSE_FILE, TMP_RAW_RESPONSE_FILE)
cmd = "diff %s %s" % (TMP_RAW_RESPONSE_FILE, REFERENCE_FILE)
util.doexec(cmd, 0)
else:
vhd.diff(TMP_VHD_RESPONSE_FILE, VHD_FILE)
clean_up()
def _doTest(self, src, frag_size, sleep_time, delta, randomise_delta=False):
"""Tests how we handle the client droping/losing packets
when they are POSTing a BITS request"""
physize = os.stat(src).st_size
vdi_mb = vhd.get_virtual_size(src)
#Expose a new VDI
record, hostname = create_and_expose(vdi_mb)
url_path = record['url_path'] + ".vhd"
#Create BITS Session
conn, session = create_BITS_session(record)
logging.debug("Frag_Size = %d" % frag_size)
#Adjust frag size for the case we have small disks
if physize < frag_size:
frag_size = physize
#Initialise Upload variables
range_start = 0
range_end = frag_size
file_offset = 0
fh = open(src, 'r')
count = 0
while file_offset < physize:
if randomise_delta:
#Generate a random number within the request range
delta = random.randint(0, range_end - range_start)
logging.debug("Random Delta = %d" % delta)
if range_end == physize:
delta = 0
logging.debug("Uploading %d-%d/%d" % (range_start, range_end, physize))
data_size = (range_end - range_start - delta)
if count == 1:
data_size = 10
logging.debug("Actually uploading %d bytes" % data_size)
data = fh.read(data_size)
logging.debug("Data Read")
try:
bits.fragment(conn, session, url_path, data, range_start, range_end, physize)
except:
logging.debug("Fragment Sent partially %d-%d/%d" % (range_start, range_end, physize))
file_offset += (range_end - range_start - delta)
range_start = file_offset + 1
count += 1
if count > 1:
sys.exit(1)
#Condition for last frag upload
range_end = file_offset + frag_size
if physize < range_end:
range_end = physize
#Sleep before creating new connection
logging.debug("Sleeping for %d seconds" % sleep_time)
time.sleep(sleep_time)
#Garbage Collect to make sure we don't run out of memory
data = ''
gc.collect()
#Create new connection
logging.debug("Get a new connection...")
conn = bits.open_connection(get_proto(record), record['ip'], record['port'])
#Close file handle
fh.close()
#Unexpose the transfervm
logging.debug("Unexposing the Transfer VM for vdi %s" % record['vdi_uuid'])
transferclient.unexpose(hostname, vdi_uuid=record['vdi_uuid'])
#Retrieve the blockmap to expose the VDI as a vhd
bitmap = transferclient.get_vdi_bitmap(hostname, record['vdi_uuid'])
logging.debug("Got bitmap %s" % bitmap)
#Expose disk with vhd_block_map
args = {'transfer_mode': 'bits',
'vdi_uuid': record['vdi_uuid'],
'network_uuid': 'management',
'vhd_blocks': bitmap,
'vhd_uuid': record['vdi_uuid']}
transferclient.expose(hostname, **args)
record = transferclient.get_record(hostname, vdi_uuid=record['vdi_uuid'])
logging.debug("Got new record: %s" % record)
dst = 'test-download.vhd'
logging.debug("Destination file will be %s" % dst)
request_size = 200*M
vhd_tests.bits_download_vhd(record, dst, request_size)
#Compare the two VHDs to check they are identical
rc = vhd.diff(src, dst)
logging.debug("Return Code %s" % rc)
#Unexpose the transfervm
logging.debug("Unexposing the Transfer VM for vdi %s" % record['vdi_uuid'])
transferclient.unexpose(hostname, vdi_uuid=record['vdi_uuid'])
#Cleanup Disks
transferclient.remove_vdi(hostname, record['vdi_uuid'])
#Remove Downloaded File
os.unlink(dst)
def do_vhd_put_get_test(self, vdi_mb, vdi_raw, pattern, blocks=None):
MAX_FRAG_SIZE = 100 * K
TMP_RAW_RESPONSE_FILE = "response.tmp"
TMP_VHD_RESPONSE_FILE = "response.vhd"
REFERENCE_FILE = "reference.tmp"
VHD_FILE = "tmp.vhd"
bitmap = None
vhd.create(VHD_FILE, vdi_mb)
vhd.fill(VHD_FILE, REFERENCE_FILE, vdi_mb, pattern)
hostname, network, vdi = testsetup.setup_host_and_network(templates=1,
vdi_mb=vdi_mb,
vdi_raw=vdi_raw)
args = {'vdi_uuid': vdi, 'network_uuid': network, 'transfer_mode': 'http'}
if blocks:
bitmap = vhd.to_bitmap(blocks, vdi_mb * M)
args['vhd_blocks'] = base64.b64encode(zlib.compress(bitmap))
args[
'vhd_uuid'] = vdi #vhd uuid needed for get - using vdi value for test purposes
transferclient.expose(hostname, **args)
record = transferclient.get_record(hostname, vdi_uuid=vdi)
url_path = record['url_path']
# test "put"
f = open(VHD_FILE, 'r')
put_data = f.read()
f.close()
record['url_path'] = url_path + ".vhd"
put_status, put_headers = http_put(record, put_data, None, vdi_mb * M)
self.assertEqual(put_status, 200)
record['url_path'] = url_path
get_status, get_headers, get_respdata = http_get(record)
if len(get_respdata) > vdi_mb * M:
# raw VDIs can be larger than requested size
get_respdata = get_respdata[:vdi_mb * M]
f = open(TMP_RAW_RESPONSE_FILE, 'w')
f.write(get_respdata)
f.close()
cmd = "diff %s %s" % (TMP_RAW_RESPONSE_FILE, REFERENCE_FILE)
util.doexec(cmd, 0)
# test "get"
record['url_path'] = url_path + ".vhd"
frag_size = random.randint(1, MAX_FRAG_SIZE)
range_start = 0
range_end = frag_size
head_status, head_headers = http_head(record)
vhd_size = int(head_headers['content-length'])
total = vhd_size
total_received = 0
total_chunks = 0
f = open(TMP_VHD_RESPONSE_FILE, 'w')
while range_start < total:
if range_end > total:
range_end = total
#print "Getting range %d-%d" % (range_start, range_end)
get_status, get_headers, get_respdata = http_get(
record, content_range=(range_start, range_end, total))
self.assertEquals(get_status, 200)
#print "Got response of length %d" % len(get_respdata)
total_received += len(get_respdata)
total_chunks += 1
f.write(get_respdata)
range_start = range_end
frag_size = random.randint(1, MAX_FRAG_SIZE)
range_end += frag_size
f.close()
#print "Got total length %d in %d chunks" % (total_received, total_chunks)
if blocks:
vhd.mask(REFERENCE_FILE, vdi_mb * M, blocks)
vhd.extract(TMP_VHD_RESPONSE_FILE, TMP_RAW_RESPONSE_FILE)
cmd = "diff %s %s" % (TMP_RAW_RESPONSE_FILE, REFERENCE_FILE)
util.doexec(cmd, 0)
else:
vhd.diff(TMP_VHD_RESPONSE_FILE, VHD_FILE)
clean_up()