def do_put_test(self, vdi_mb, put_data_size, put_offset, check_border_size, unexpose_reexpose=False):
self.assert_(put_data_size + put_offset <= vdi_mb * M, 'Test has invalid data offsets.')
lower_border_size = min(put_offset, check_border_size)
upper_border_size = min(vdi_mb * M - put_offset - put_data_size, check_border_size)
put_data = 'abcdefgh' * (put_data_size / 8)
expect_data = ('\0' * lower_border_size) + put_data + ('\0' * upper_border_size)
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=vdi_mb)
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='http')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
put_status, put_headers = http_put(record, put_data, put_offset, vdi_mb * M)
self.assertEqual(put_status, 200)
if unexpose_reexpose:
transferclient.unexpose(hostname, vdi_uuid=vdi)
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='http')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
get_status, get_headers, get_respdata = http_get(record, (put_offset - lower_border_size, put_offset + put_data_size + upper_border_size))
self.assert_(get_status == 200 or get_status == 206, 'GET status code %d is not success.' % get_status)
self.assertEquals(len(get_respdata), len(expect_data))
self.assertEquals(get_respdata, expect_data)
clean_up()
def testGetRecordWorksWhenMultipleVDIsAreExposed(self):
hostname, network, vdi1 = testsetup.setup_host_and_network(templates=1, vdi_mb=10)
vdi2 = transferclient.create_vdi(hostname, 'Second Test VDI', 12 * 1024 * 1024)
vdi3 = transferclient.create_vdi(hostname, 'Third Test VDI', 14 * 1024 * 1024)
vdi4 = transferclient.create_vdi(hostname, 'Fourth Test VDI', 16 * 1024 * 1024)
transferclient.expose(hostname, vdi_uuid=vdi2, network_uuid=network, transfer_mode='http')
transferclient.expose(hostname, vdi_uuid=vdi3, network_uuid=network, transfer_mode='http')
record1 = transferclient.get_record(hostname, vdi_uuid=vdi1)
record2 = transferclient.get_record(hostname, vdi_uuid=vdi2)
record3 = transferclient.get_record(hostname, vdi_uuid=vdi3)
record4 = transferclient.get_record(hostname, vdi_uuid=vdi4)
self.assertVdiStatus(record1, vdi1, 'unused')
self.assertVdiStatus(record2, vdi2, 'exposed')
self.assertVdiStatus(record3, vdi3, 'exposed')
self.assertVdiStatus(record4, vdi4, 'unused')
clean_up()
def testGetRecordWithUnusedVDI(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=10)
# No expose called.
record = transferclient.get_record(hostname, vdi_uuid=vdi)
self.assertRecordFields(record, ['status', 'vdi_uuid'])
self.assertVdiStatus(record, vdi, 'unused')
clean_up()
def create_and_expose(vdi_mb):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=vdi_mb)
args = {}
args['vdi_uuid'] = vdi
args['network_uuid'] = 'management'
args['transfer_mode'] = 'bits'
transferclient.expose(hostname, **args)
return transferclient.get_record(hostname, vdi_uuid=vdi), hostname
def testHttpServerSSLCertificateMatchesTheOneReturnedByGetRecord(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=16)
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='http', use_ssl='true')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
self.assert_('ssl_cert' in record)
servercert = get_server_cert(record['ip'], record['port'])
#Translating '\n' for '|' because newlines won't pass through XenStore - '|' used instead
translated_cert = servercert.replace("\n","|") + "|"
self.assertEqual(translated_cert, record['ssl_cert'])
def setup_and_get_record(timeout_minutes):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=16)
if timeout_minutes is not None:
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='http', timeout_minutes=str(timeout_minutes))
logging.debug('Timeout Minutes')
logging.debug(timeout_minutes)
else:
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='http')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
return hostname, record
def testGetRecordWithBITSExposedVDI(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=10)
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='bits')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
self.assertStandardFields(record)
self.assertVdiStatus(record, vdi, 'exposed')
self.assertRecordFields(record, ['url_path', 'url_full'])
self.assertEqual('bits', record['transfer_mode'])
self.assertEqual('80', record['port']) # Standard HTTP port
clean_up()
def setup_and_get_record(vdi_mb, trans_mode, hostname, vdi_uuid):
logging.debug("starting function")
if hostname and vdi_uuid:
logging.debug("%s %s" % (hostname, vdi_uuid))
transferclient.expose(hostname, vdi_uuid=vdi_uuid, network_uuid="management", transfer_mode=trans_mode)
else:
hostname, network, vdi_uuid = testsetup.setup_host_and_network(templates=1, vdi_mb=vdi_mb)
transferclient.expose(hostname, vdi_uuid=vdi_uuid, network_uuid=network, transfer_mode=trans_mode)
record = transferclient.get_record(hostname, vdi_uuid=vdi_uuid)
return record, hostname
def testGetRecordWithISCSIExposedVDI(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=10)
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='iscsi')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
self.assertStandardFields(record)
self.assertVdiStatus(record, vdi, 'exposed')
self.assertRecordFields(record, ['iscsi_iqn', 'iscsi_lun', 'iscsi_sn'])
self.assertEqual('iscsi', record['transfer_mode'])
self.assertEqual('3260', record['port']) # Standard iSCSI port
clean_up()
def testGetRecordWorksWhenReexposingVDIMultipleTimes(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=10)
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='http')
retval = transferclient.unexpose(hostname, vdi_uuid=vdi)
self.assertEquals(retval, 'OK', 'Unexpose failed, never got to get_record testing.')
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='http')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
self.assertVdiStatus(record, vdi, 'exposed')
clean_up()
def testPut(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=16)
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='http', use_ssl='true')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
data = 'a' * (1*M)
putstatus, putheaders = https_put(record, data, 2*M, 16*M)
self.assertEqual(putstatus, 200)
expecteddata = ('\0' * (2*M)) + data + ('\0' * (13*M))
assertVdiEqualsUsingHttps(self, record, expecteddata)
def testExposeWithParallelGetRecord(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=10)
asyncexpose = ExposeThread(hostname, network, vdi)
asyncexpose.start()
record = {'status': 'unused'}
while record['status'] == 'unused':
logging.debug('VDI status still unused, getting record')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
moreasserts.assertVdiIsZeroUsingHttpGet(self, record, 10)
def expose_vdi_as_vhd(hostname, vdi_uuid, bitmap):
args = {}
args['vhd_blocks'] = bitmap
args['vhd_uuid'] = vdi_uuid #take anything, for test purposes
args['vdi_uuid'] = vdi_uuid
args['network_uuid'] = 'management'
args['transfer_mode'] = 'bits'
args['get_log'] = 'true'
transferclient.expose(hostname, **args)
record = transferclient.get_record(hostname, vdi_uuid=vdi_uuid)
return record
def testExposeWhileHammeringCleanup(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=10)
asyncexpose = ExposeThread(hostname, network, vdi)
asyncexpose.start()
# Assumes that transferclient.expose is slow.
while asyncexpose.isAlive():
logging.debug('Cleaning up')
transferclient.cleanup(hostname)
time.sleep(0.0001)
record = transferclient.get_record(hostname, vdi_uuid=vdi)
moreasserts.assertVdiIsZeroUsingHttpGet(self, record, 10)
def testSendMultiplePutRequestsInOneHttpsSession(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=16)
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='http', use_ssl='true')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
conn = httplib.HTTPSConnection(record['ip'], int(record['port']))
try:
for i in xrange(1, 5):
data = 'a' * (i * 100*K)
status, headers = http_test.http_put_request(conn, record, data, i * M * 2 + 234*K, 16*M)
self.assertEqual(status, 200)
finally:
conn.close()
def testParallelExposes(self):
parallelism = 3
hostname, network, firstvdi = testsetup.setup_host_and_network(templates=1, vdi_mb=10)
vdis = [firstvdi] + [transferclient.create_vdi(hostname, 'Test VDI', 10*M) for i in xrange(parallelism - 1)]
threads = [ExposeThread(hostname, network, vdi) for vdi in vdis]
for t in threads:
t.start()
for t in threads:
t.join()
records = [transferclient.get_record(hostname, vdi_uuid=vdi) for vdi in vdis]
for record in records:
moreasserts.assertVdiIsZeroUsingHttpGet(self, record, 10)
def testGetRecordWithISCSIExposedVDI(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1,
vdi_mb=10)
transferclient.expose(hostname,
vdi_uuid=vdi,
network_uuid=network,
transfer_mode='iscsi')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
self.assertStandardFields(record)
self.assertVdiStatus(record, vdi, 'exposed')
self.assertRecordFields(record, ['iscsi_iqn', 'iscsi_lun', 'iscsi_sn'])
self.assertEqual('iscsi', record['transfer_mode'])
self.assertEqual('3260', record['port']) # Standard iSCSI port
clean_up()
def testHttpServerSSLCertificateMatchesTheOneReturnedByGetRecord(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1,
vdi_mb=16)
transferclient.expose(hostname,
vdi_uuid=vdi,
network_uuid=network,
transfer_mode='http',
use_ssl='true')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
self.assert_('ssl_cert' in record)
servercert = get_server_cert(record['ip'], record['port'])
#Translating '\n' for '|' because newlines won't pass through XenStore - '|' used instead
translated_cert = servercert.replace("\n", "|") + "|"
self.assertEqual(translated_cert, record['ssl_cert'])
def dotest(self, vdi_mb, responselength, rangebounds):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=vdi_mb)
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='http')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
status, headers, data = http_get(record, rangebounds)
if rangebounds is None:
self.assertEqual(status, httplib.OK)
else:
self.assertEqual(status, httplib.PARTIAL_CONTENT)
self.assert_('content-range' in map(str.lower, headers.iterkeys()))
self.assertEquals(responselength, len(data))
self.assertEquals('\0' * responselength, data)
clean_up()
def testTargetHostUUIDConfig(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=10)
if self.REMOVE_TEMPLATE:
#Remove the TVM template to catch errors that would occur on first run (when the template doesn't exist)
testsetup.remove_tvm_template(hostname)
host_uuid = get_host_uuids(hostname)
transfer_mode=self.TRANSFER_MODE
asyncexpose = ExposeThread(hostname, network, vdi, transfer_mode=transfer_mode, target_host_uuid=host_uuid)
asyncexpose.start()
while asyncexpose.isAlive():
time.sleep(1)
record = transferclient.get_record(hostname, vdi_uuid=vdi)
logging.debug(record)
transferclient.unexpose(hostname, vdi_uuid=vdi)
testsetup.clean_host(hostname)
def testGetRecordWithHTTPSExposedVDI(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1,
vdi_mb=10)
transferclient.expose(hostname,
vdi_uuid=vdi,
network_uuid=network,
transfer_mode='http',
use_ssl='true')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
self.assertStandardFields(record)
self.assertVdiStatus(record, vdi, 'exposed')
self.assertRecordFields(record, ['url_path', 'url_full', 'ssl_cert'])
self.assertEqual('http', record['transfer_mode'])
self.assertEqual('443', record['port']) # Standard HTTPS port
clean_up()
def put_vhd(self, file_name, vdi_mb, vdi_raw):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=vdi_mb, vdi_raw=vdi_raw)
args = {'vdi_uuid': vdi, 'network_uuid':network, 'transfer_mode':'bits'}
print "Putting %s" % (file_name)
#Calculate allocated blocks for the sake of sparseness
args['vhd_blocks'] = get_encoded_bitmap_from_file(file_name)
logging.debug(args['vhd_blocks'])
args['vhd_uuid'] = vdi #just take anything for test purposes
transferclient.expose(hostname, **args)
record = transferclient.get_record(hostname, vdi_uuid=vdi)
bits_upload_vhd(record, file_name)
return record
def testPut(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1,
vdi_mb=16)
transferclient.expose(hostname,
vdi_uuid=vdi,
network_uuid=network,
transfer_mode='http',
use_ssl='true')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
data = 'a' * (1 * M)
putstatus, putheaders = https_put(record, data, 2 * M, 16 * M)
self.assertEqual(putstatus, 200)
expecteddata = ('\0' * (2 * M)) + data + ('\0' * (13 * M))
assertVdiEqualsUsingHttps(self, record, expecteddata)
def assertPutContentRangeResponse(self, vdi_mb, data_size, range_str, response_status):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=vdi_mb)
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='http')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
headers = {'Authorization': auth_header(record['username'], record['password']),
'Content-Range': range_str}
data = 'a' * data_size
conn = httplib.HTTPConnection(record['ip'], record['port'])
try:
conn.request('PUT', record['url_path'], data, headers)
resp = conn.getresponse()
resp.read(0)
self.assertEquals(resp.status, response_status)
finally:
conn.close()
clean_up()
def put_vhd(self, file_name, vdi_mb, vdi_raw):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1,
vdi_mb=vdi_mb,
vdi_raw=vdi_raw)
args = {'vdi_uuid': vdi, 'network_uuid': network, 'transfer_mode': 'bits'}
print "Putting %s" % (file_name)
#Calculate allocated blocks for the sake of sparseness
args['vhd_blocks'] = get_encoded_bitmap_from_file(file_name)
logging.debug(args['vhd_blocks'])
args['vhd_uuid'] = vdi #just take anything for test purposes
transferclient.expose(hostname, **args)
record = transferclient.get_record(hostname, vdi_uuid=vdi)
bits_upload_vhd(record, file_name)
return record
def setup_and_get_record(timeout_minutes):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1,
vdi_mb=16)
if timeout_minutes is not None:
transferclient.expose(hostname,
vdi_uuid=vdi,
network_uuid=network,
transfer_mode='http',
timeout_minutes=str(timeout_minutes))
logging.debug('Timeout Minutes')
logging.debug(timeout_minutes)
else:
transferclient.expose(hostname,
vdi_uuid=vdi,
network_uuid=network,
transfer_mode='http')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
return hostname, record
def dotest(self, vdi_mb, responselength, rangebounds):
hostname, network, vdi = testsetup.setup_host_and_network(
templates=1, vdi_mb=vdi_mb)
transferclient.expose(hostname,
vdi_uuid=vdi,
network_uuid=network,
transfer_mode='http')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
status, headers, data = http_get(record, rangebounds)
if rangebounds is None:
self.assertEqual(status, httplib.OK)
else:
self.assertEqual(status, httplib.PARTIAL_CONTENT)
self.assert_('content-range' in map(str.lower, headers.iterkeys()))
self.assertEquals(responselength, len(data))
self.assertEquals('\0' * responselength, data)
clean_up()
def testSendMultiplePutRequestsInOneHttpSession(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1,
vdi_mb=16)
transferclient.expose(hostname,
vdi_uuid=vdi,
network_uuid=network,
transfer_mode='http')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
conn = httplib.HTTPConnection(record['ip'], record['port'])
try:
for i in xrange(1, 5):
data = 'a' * (i * 100 * K)
status, headers = http_put_request(conn, record, data,
i * M * 2 + 234 * K, 16 * M)
self.assertEqual(status, 200)
finally:
conn.close()
clean_up()
def testGetRecordWorksWhenReexposingVDIMultipleTimes(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1,
vdi_mb=10)
transferclient.expose(hostname,
vdi_uuid=vdi,
network_uuid=network,
transfer_mode='http')
retval = transferclient.unexpose(hostname, vdi_uuid=vdi)
self.assertEquals(retval, 'OK',
'Unexpose failed, never got to get_record testing.')
transferclient.expose(hostname,
vdi_uuid=vdi,
network_uuid=network,
transfer_mode='http')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
self.assertVdiStatus(record, vdi, 'exposed')
clean_up()
def testVdiRecordStatusIsUnusedAfterUnexpose(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=10)
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='http')
transferclient.unexpose(hostname, vdi_uuid=vdi)
record = transferclient.get_record(hostname, vdi_uuid=vdi)
self.assertEqual(record['status'], 'unused')
def testGet(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=2)
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='http', use_ssl='true')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
assertVdiEqualsUsingHttps(self, record, '\0'*(2*M))
def testConnection(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=10, dangerous_test=True)
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='http', network_mode='manual', network_ip=self.ip, network_mask=self.mask, network_gateway=self.gw)
record = transferclient.get_record(hostname, vdi_uuid=vdi)
# Test GET to the ip
assertVdiZero(self, self.ip, record['port'], record, 10)
def testConfiguration(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=10, dangerous_test=True)
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='http', network_mode='manual', network_ip=self.ip, network_mask=self.mask, network_gateway=self.gw)
record = transferclient.get_record(hostname, vdi_uuid=vdi)
self.assertEquals(self.ip, record['ip'])
def do_vhd_put_get_test(self, vdi_mb, vdi_raw, pattern, blocks = None):
MAX_FRAG_SIZE = 100*K
TMP_RAW_RESPONSE_FILE = "response.tmp"
TMP_VHD_RESPONSE_FILE = "response.vhd"
REFERENCE_FILE = "reference.tmp"
VHD_FILE = "tmp.vhd"
bitmap = None
vhd.create(VHD_FILE, vdi_mb)
vhd.fill(VHD_FILE, REFERENCE_FILE, vdi_mb, pattern)
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=vdi_mb, vdi_raw=vdi_raw)
args = {'vdi_uuid': vdi, 'network_uuid': network, 'transfer_mode': 'http'}
if blocks:
bitmap = vhd.to_bitmap(blocks, vdi_mb * M)
args['vhd_blocks'] = base64.b64encode(zlib.compress(bitmap))
args['vhd_uuid'] = vdi #vhd uuid needed for get - using vdi value for test purposes
transferclient.expose(hostname, **args)
record = transferclient.get_record(hostname, vdi_uuid=vdi)
url_path = record['url_path']
# test "put"
f = open(VHD_FILE, 'r')
put_data = f.read()
f.close()
record['url_path'] = url_path + ".vhd"
put_status, put_headers = http_put(record, put_data, None, vdi_mb * M)
self.assertEqual(put_status, 200)
record['url_path'] = url_path
get_status, get_headers, get_respdata = http_get(record)
if len(get_respdata) > vdi_mb * M:
# raw VDIs can be larger than requested size
get_respdata = get_respdata[:vdi_mb * M]
f = open(TMP_RAW_RESPONSE_FILE, 'w')
f.write(get_respdata)
f.close()
cmd = "diff %s %s" % (TMP_RAW_RESPONSE_FILE, REFERENCE_FILE)
util.doexec(cmd, 0)
# test "get"
record['url_path'] = url_path + ".vhd"
frag_size = random.randint(1, MAX_FRAG_SIZE)
range_start = 0
range_end = frag_size
head_status, head_headers = http_head(record)
vhd_size = int(head_headers['content-length'])
total = vhd_size
total_received = 0
total_chunks = 0
f = open(TMP_VHD_RESPONSE_FILE, 'w')
while range_start < total:
if range_end > total:
range_end = total
#print "Getting range %d-%d" % (range_start, range_end)
get_status, get_headers, get_respdata = http_get(record, content_range=(range_start, range_end, total))
self.assertEquals(get_status, 200)
#print "Got response of length %d" % len(get_respdata)
total_received += len(get_respdata)
total_chunks += 1
f.write(get_respdata)
range_start = range_end
frag_size = random.randint(1, MAX_FRAG_SIZE)
range_end += frag_size
f.close()
#print "Got total length %d in %d chunks" % (total_received, total_chunks)
if blocks:
vhd.mask(REFERENCE_FILE, vdi_mb * M, blocks)
vhd.extract(TMP_VHD_RESPONSE_FILE, TMP_RAW_RESPONSE_FILE)
cmd = "diff %s %s" % (TMP_RAW_RESPONSE_FILE, REFERENCE_FILE)
util.doexec(cmd, 0)
else:
vhd.diff(TMP_VHD_RESPONSE_FILE, VHD_FILE)
clean_up()
def do_vhd_put_get_test(self, vdi_mb, vdi_raw, pattern, blocks=None):
MAX_FRAG_SIZE = 100 * K
TMP_RAW_RESPONSE_FILE = "response.tmp"
TMP_VHD_RESPONSE_FILE = "response.vhd"
REFERENCE_FILE = "reference.tmp"
VHD_FILE = "tmp.vhd"
bitmap = None
vhd.create(VHD_FILE, vdi_mb)
vhd.fill(VHD_FILE, REFERENCE_FILE, vdi_mb, pattern)
hostname, network, vdi = testsetup.setup_host_and_network(templates=1,
vdi_mb=vdi_mb,
vdi_raw=vdi_raw)
args = {'vdi_uuid': vdi, 'network_uuid': network, 'transfer_mode': 'http'}
if blocks:
bitmap = vhd.to_bitmap(blocks, vdi_mb * M)
args['vhd_blocks'] = base64.b64encode(zlib.compress(bitmap))
args[
'vhd_uuid'] = vdi #vhd uuid needed for get - using vdi value for test purposes
transferclient.expose(hostname, **args)
record = transferclient.get_record(hostname, vdi_uuid=vdi)
url_path = record['url_path']
# test "put"
f = open(VHD_FILE, 'r')
put_data = f.read()
f.close()
record['url_path'] = url_path + ".vhd"
put_status, put_headers = http_put(record, put_data, None, vdi_mb * M)
self.assertEqual(put_status, 200)
record['url_path'] = url_path
get_status, get_headers, get_respdata = http_get(record)
if len(get_respdata) > vdi_mb * M:
# raw VDIs can be larger than requested size
get_respdata = get_respdata[:vdi_mb * M]
f = open(TMP_RAW_RESPONSE_FILE, 'w')
f.write(get_respdata)
f.close()
cmd = "diff %s %s" % (TMP_RAW_RESPONSE_FILE, REFERENCE_FILE)
util.doexec(cmd, 0)
# test "get"
record['url_path'] = url_path + ".vhd"
frag_size = random.randint(1, MAX_FRAG_SIZE)
range_start = 0
range_end = frag_size
head_status, head_headers = http_head(record)
vhd_size = int(head_headers['content-length'])
total = vhd_size
total_received = 0
total_chunks = 0
f = open(TMP_VHD_RESPONSE_FILE, 'w')
while range_start < total:
if range_end > total:
range_end = total
#print "Getting range %d-%d" % (range_start, range_end)
get_status, get_headers, get_respdata = http_get(
record, content_range=(range_start, range_end, total))
self.assertEquals(get_status, 200)
#print "Got response of length %d" % len(get_respdata)
total_received += len(get_respdata)
total_chunks += 1
f.write(get_respdata)
range_start = range_end
frag_size = random.randint(1, MAX_FRAG_SIZE)
range_end += frag_size
f.close()
#print "Got total length %d in %d chunks" % (total_received, total_chunks)
if blocks:
vhd.mask(REFERENCE_FILE, vdi_mb * M, blocks)
vhd.extract(TMP_VHD_RESPONSE_FILE, TMP_RAW_RESPONSE_FILE)
cmd = "diff %s %s" % (TMP_RAW_RESPONSE_FILE, REFERENCE_FILE)
util.doexec(cmd, 0)
else:
vhd.diff(TMP_VHD_RESPONSE_FILE, VHD_FILE)
clean_up()
def setup_and_get_record(vdi_mb=VDI_MB, vdi_raw=False):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=vdi_mb, vdi_raw=vdi_raw)
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='bits')
return transferclient.get_record(hostname, vdi_uuid=vdi)
def testGetOnPort123(self):
hostname, network, vdi = testsetup.setup_host_and_network(templates=1, vdi_mb=10)
transferclient.expose(hostname, vdi_uuid=vdi, network_uuid=network, transfer_mode='http', network_port='123')
record = transferclient.get_record(hostname, vdi_uuid=vdi)
assertVdiZero(self, record['ip'], '123', record, 10)
def _doTest(self, src, frag_size, sleep_time, delta, randomise_delta=False):
"""Tests how we handle the client droping/losing packets
when they are POSTing a BITS request"""
physize = os.stat(src).st_size
vdi_mb = vhd.get_virtual_size(src)
#Expose a new VDI
record, hostname = create_and_expose(vdi_mb)
url_path = record['url_path'] + ".vhd"
#Create BITS Session
conn, session = create_BITS_session(record)
logging.debug("Frag_Size = %d" % frag_size)
#Adjust frag size for the case we have small disks
if physize < frag_size:
frag_size = physize
#Initialise Upload variables
range_start = 0
range_end = frag_size
file_offset = 0
fh = open(src, 'r')
count = 0
while file_offset < physize:
if randomise_delta:
#Generate a random number within the request range
delta = random.randint(0, range_end - range_start)
logging.debug("Random Delta = %d" % delta)
if range_end == physize:
delta = 0
logging.debug("Uploading %d-%d/%d" % (range_start, range_end, physize))
data_size = (range_end - range_start - delta)
if count == 1:
data_size = 10
logging.debug("Actually uploading %d bytes" % data_size)
data = fh.read(data_size)
logging.debug("Data Read")
try:
bits.fragment(conn, session, url_path, data, range_start, range_end, physize)
except:
logging.debug("Fragment Sent partially %d-%d/%d" % (range_start, range_end, physize))
file_offset += (range_end - range_start - delta)
range_start = file_offset + 1
count += 1
if count > 1:
sys.exit(1)
#Condition for last frag upload
range_end = file_offset + frag_size
if physize < range_end:
range_end = physize
#Sleep before creating new connection
logging.debug("Sleeping for %d seconds" % sleep_time)
time.sleep(sleep_time)
#Garbage Collect to make sure we don't run out of memory
data = ''
gc.collect()
#Create new connection
logging.debug("Get a new connection...")
conn = bits.open_connection(get_proto(record), record['ip'], record['port'])
#Close file handle
fh.close()
#Unexpose the transfervm
logging.debug("Unexposing the Transfer VM for vdi %s" % record['vdi_uuid'])
transferclient.unexpose(hostname, vdi_uuid=record['vdi_uuid'])
#Retrieve the blockmap to expose the VDI as a vhd
bitmap = transferclient.get_vdi_bitmap(hostname, record['vdi_uuid'])
logging.debug("Got bitmap %s" % bitmap)
#Expose disk with vhd_block_map
args = {'transfer_mode': 'bits',
'vdi_uuid': record['vdi_uuid'],
'network_uuid': 'management',
'vhd_blocks': bitmap,
'vhd_uuid': record['vdi_uuid']}
transferclient.expose(hostname, **args)
record = transferclient.get_record(hostname, vdi_uuid=record['vdi_uuid'])
logging.debug("Got new record: %s" % record)
dst = 'test-download.vhd'
logging.debug("Destination file will be %s" % dst)
request_size = 200*M
vhd_tests.bits_download_vhd(record, dst, request_size)
#Compare the two VHDs to check they are identical
rc = vhd.diff(src, dst)
logging.debug("Return Code %s" % rc)
#Unexpose the transfervm
logging.debug("Unexposing the Transfer VM for vdi %s" % record['vdi_uuid'])
transferclient.unexpose(hostname, vdi_uuid=record['vdi_uuid'])
#Cleanup Disks
transferclient.remove_vdi(hostname, record['vdi_uuid'])
#Remove Downloaded File
os.unlink(dst)