def put_SLO(url, token):
# Create temp files
assembly = []
for i in range(1,10):
oname = '/tmp/slo_chunk_%d' % i
f = open(oname,'r')
content_length = None
response = dict()
progress()
c.put_object(url, token, 'myobjects', oname, f,
content_length, None, None, "application/octet-stream",
None, None, None, None, response)
f.close()
status = response.get('status')
assert (status >= 200 and status < 300)
headers = response.get('headers')
segment = dict()
segment['path'] = 'myobjects/%s' % oname
segment['size_bytes'] = 1048576
segment['etag'] = headers['etag']
assembly.append(segment)
content_length = None
response = dict()
headers = {'x-object-meta-prop1' : 'val1'}
progress()
c.put_object(url, token, 'myobjects', 'assembly', json.dumps(assembly),
content_length=None, etag=None, chunk_size=None, headers=headers,
query_string='multipart-manifest=put', response_dict=response)
status = response.get('status')
assert (status >= 200 and status < 300)
progress_ln()
def create_local_chunks():
for i in range(1,10):
progress()
oname = '/tmp/slo_chunk_%d' % i
f = open(oname,'w')
f.write(''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(1048576)))
f.close()
progress_ln()
def create_local_chunks():
for i in range(1, 10):
progress()
oname = '/tmp/slo_chunk_%d' % i
f = open(oname, 'w')
f.write(''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(1048576)))
f.close()
progress_ln()
def deploy_sloidentity_storlet(url, token):
progress()
response = dict()
c.put_container(url, token, 'mysloobject', None, None, response)
status = response.get('status')
assert (status >= 200 or status < 300)
progress()
put_storlet_object(url, token, SLOIDENTITY_STORLET_NAME,
SLOIDENTITY_PATH_TO_BUNDLE, '',
'com.ibm.storlet.sloidentity.SLOIdentityStorlet')
progress_ln()
def deploy_sloidentity_storlet(url, token):
progress()
response = dict()
c.put_container(url, token, 'mysloobject', None, None, response)
status = response.get('status')
assert (status >= 200 or status < 300)
progress()
put_storlet_object( url, token,
SLOIDENTITY_STORLET_NAME,
SLOIDENTITY_PATH_TO_BUNDLE,
'',
'com.ibm.storlet.sloidentity.SLOIdentityStorlet')
progress_ln()
def get_SLO(url, token):
response = dict()
headers, body = c.get_object(url, token, 'myobjects', 'assembly', http_conn=None,
resp_chunk_size=1048576, query_string=None, response_dict=response, headers=None)
i = 1
for chunk in body:
oname = '/tmp/slo_chunk_%d' % i
f = open(oname,'r')
file_content=f.read()
#print '%s %s' % (chunk[:10], file_content[:10])
#print '%d %d' % (len(chunk), len(file_content))
progress()
assert(chunk == file_content)
f.close()
i=i+1
progress_ln()
def invoke_identity_on_partial_get_SLO(url, token):
metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME }
for i in range(5):
progress()
response = dict()
headers, body = c.get_object(url, token,
'myobjects',
'assembly',
query_string = None,
response_dict=response,
resp_chunk_size=1048576,
headers=metadata)
j = 1
for chunk in body:
j = j + 1
if j == 5:
break
progress_ln()
def invoke_identity_on_partial_get_SLO(url, token):
metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME}
for i in range(5):
progress()
response = dict()
headers, body = c.get_object(url,
token,
'myobjects',
'assembly',
query_string=None,
response_dict=response,
resp_chunk_size=1048576,
headers=metadata)
j = 1
for chunk in body:
j = j + 1
if j == 5:
break
progress_ln()
def put_SLO(url, token):
# Create temp files
assembly = []
for i in range(1, 10):
oname = '/tmp/slo_chunk_%d' % i
f = open(oname, 'r')
content_length = None
response = dict()
progress()
c.put_object(url, token, 'myobjects', oname, f, content_length, None,
None, "application/octet-stream", None, None, None, None,
response)
f.close()
status = response.get('status')
assert (status >= 200 and status < 300)
headers = response.get('headers')
segment = dict()
segment['path'] = 'myobjects/%s' % oname
segment['size_bytes'] = 1048576
segment['etag'] = headers['etag']
assembly.append(segment)
content_length = None
response = dict()
headers = {'x-object-meta-prop1': 'val1'}
progress()
c.put_object(url,
token,
'myobjects',
'assembly',
json.dumps(assembly),
content_length=None,
etag=None,
chunk_size=None,
headers=headers,
query_string='multipart-manifest=put',
response_dict=response)
status = response.get('status')
assert (status >= 200 and status < 300)
progress_ln()
def invoke_identity_on_get_SLO_double(url, token):
metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME}
response = dict()
headers, body = c.get_object(url,
token,
'myobjects',
'assembly',
query_string='double=true',
response_dict=response,
resp_chunk_size=2048,
headers=metadata)
i = 1
progress()
oname = '/tmp/slo_chunk_%d' % i
f = open(oname, 'r')
file_content = f.read()
j = 0 # Count chunks in file 1...1024
for chunk in body:
file_fragment = file_content[j * 1024:(j + 1) * 1024]
chunk_framgment_low = chunk[0:1024]
chunk_framgment_high = chunk[1024:2048]
assert (chunk_framgment_low == file_fragment)
assert (chunk_framgment_high == file_fragment)
j = j + 1
if j == 1024:
i = i + 1
if i == 10:
break
f.close()
progress()
oname = '/tmp/slo_chunk_%d' % i
f = open(oname, 'r')
file_content = f.read()
j = 0
assert i == 10
progress_ln()
def compare_slo_to_chunks(body):
i = 1
for chunk in body:
if chunk:
if i<10:
progress()
oname = '/tmp/slo_chunk_%d' % i
f = open(oname,'r')
file_content=f.read()
#print '%s %s' % (chunk[:10], file_content[:10])
#print '%d %d' % (len(chunk), len(file_content))
assert(chunk == file_content)
f.close()
i=i+1
else:
aux_content = ''
for j in range(1,4):
oname = '/tmp/aux_file%d' % j
f = open(oname,'r')
aux_content+=f.read()
f.close()
assert(chunk == aux_content)
progress_ln()
def compare_slo_to_chunks(body):
i = 1
for chunk in body:
if chunk:
if i < 10:
progress()
oname = '/tmp/slo_chunk_%d' % i
f = open(oname, 'r')
file_content = f.read()
#print '%s %s' % (chunk[:10], file_content[:10])
#print '%d %d' % (len(chunk), len(file_content))
assert (chunk == file_content)
f.close()
i = i + 1
else:
aux_content = ''
for j in range(1, 4):
oname = '/tmp/aux_file%d' % j
f = open(oname, 'r')
aux_content += f.read()
f.close()
assert (chunk == aux_content)
progress_ln()
def invoke_identity_on_get_SLO_double(url, token):
metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME }
response = dict()
headers, body = c.get_object(url, token,
'myobjects',
'assembly',
query_string = 'double=true',
response_dict=response,
resp_chunk_size=2048,
headers=metadata)
i = 1
progress()
oname = '/tmp/slo_chunk_%d' % i
f = open(oname,'r')
file_content=f.read()
j = 0 # Count chunks in file 1...1024
for chunk in body:
file_fragment = file_content[j*1024:(j+1)*1024]
chunk_framgment_low = chunk[0:1024]
chunk_framgment_high = chunk[1024:2048]
assert(chunk_framgment_low == file_fragment)
assert(chunk_framgment_high == file_fragment)
j = j +1
if j == 1024:
i = i + 1
if i == 10:
break
f.close()
progress()
oname = '/tmp/slo_chunk_%d' % i
f = open(oname,'r')
file_content=f.read()
j = 0
assert i == 10
progress_ln()
def get_SLO(url, token):
response = dict()
headers, body = c.get_object(url,
token,
'myobjects',
'assembly',
http_conn=None,
resp_chunk_size=1048576,
query_string=None,
response_dict=response,
headers=None)
i = 1
for chunk in body:
oname = '/tmp/slo_chunk_%d' % i
f = open(oname, 'r')
file_content = f.read()
#print '%s %s' % (chunk[:10], file_content[:10])
#print '%d %d' % (len(chunk), len(file_content))
progress()
assert (chunk == file_content)
f.close()
i = i + 1
progress_ln()
def invoke_storlet_on_1GB_file(url, token):
GBFile = open('/tmp/1GB_file','w')
for _ in range(128):
progress()
uploaded_content = ''.join('1' for _ in range(8*1024*1024))
GBFile.write(uploaded_content)
GBFile.close()
headers = {'X-Run-Storlet': IDENTITY_STORLET_NAME }
GBFile = open('/tmp/1GB_file','r')
response=dict()
progress()
c.put_object(url, token, 'myobjects', '1GBFile', GBFile,
1024*1024*1024, None, None, "application/octet-stream",
headers, None, None, None, response)
progress()
status = response.get('status')
assert (status == 200 or status == 201)
progress()
os.remove('/tmp/1GB_file')
progress_ln()
def invoke_storlet_on_1GB_file(url, token):
GBFile = open('/tmp/1GB_file', 'w')
for _ in range(128):
progress()
uploaded_content = ''.join('1' for _ in range(8 * 1024 * 1024))
GBFile.write(uploaded_content)
GBFile.close()
headers = {'X-Run-Storlet': IDENTITY_STORLET_NAME}
GBFile = open('/tmp/1GB_file', 'r')
response = dict()
progress()
c.put_object(url, token, 'myobjects', '1GBFile', GBFile,
1024 * 1024 * 1024, None, None, "application/octet-stream",
headers, None, None, None, response)
progress()
status = response.get('status')
assert (status == 200 or status == 201)
progress()
os.remove('/tmp/1GB_file')
progress_ln()