def uploadStorageObject(endpoint, container='compute_images', authtoken=None, filename=None, splitsize=4000, poolsize=4, authendpoint=None, user=None, password=None, extractarchive=None, **kwargs):
basepath = container
imgbasepath = basepath
splitbasepath = basepath + '_segments'
headers = None
params = None
if extractarchive is not None:
if params is None:
params = {}
params['extract-archive'] = extractarchive
data = None
files = None
jsonResponse = ''
if filename is not None and os.path.exists(filename):
#md5hash = md5(filename)
filesize = os.path.getsize(filename)
filesize /= (1024 * 1024)
if filesize > splitsize:
print('Splitting : ' + filename)
filelist = splitfile(filename, str(splitsize) + 'MB')
print('Into ' + str(len(filelist)) + ' segments')
basepath = splitbasepath + '/' + os.path.split(filename)[-1] + '/_segment_'
pool = multiprocessing.Pool(poolsize)
# Build tupal list
workerdata = []
for fn in filelist:
workerdata.append([endpoint, basepath, authtoken, fn, authendpoint, user, password, headers, params])
#print(workerdata)
# Start processes
pool.map(uploadfile, workerdata)
# Upload manifest file to point to parts
manifest = basepath + '/' + getsplitprefix(filename)
resourcename = os.path.split(filename)[-1]
headers = {'Content-Length': "0", 'X-Object-Manifest': manifest}
printJSON(headers)
data = None
basepath = imgbasepath
try:
response = callRESTApi(endpoint, basepath, resourcename, method='PUT', authtoken=authtoken, headers=headers, params=params, data=data, files=files)
except REST401Exception as e:
# Reauthenticate and retry
if authendpoint is not None and user is not None and password is not None:
authtoken, endpoint = authenticate(authendpoint, user, password)
response = callRESTApi(endpoint, basepath, resourcename, method='PUT', authtoken=authtoken, headers=headers, params=params, data=data, files=files)
else:
raise
# Remove splitfiles
splitdir = getsplitdir(filename)
shutil.rmtree(splitdir)
else:
# Simple single file upload
basepath = imgbasepath
# Upload file
print('Uploading : ' + filename)
resourcename = os.path.split(filename)[-1]
with closing(open(filename, 'rb')) as f:
response = callRESTApi(endpoint, basepath, resourcename, method='PUT', authtoken=authtoken, headers=headers, params=params, data=f, files=files)
print('Uploaded : ' + filename)
jsonResponse = response.text
return jsonResponse
def authenticate(endpoint, user, password):
clearHTTPSession()
basepath = '/auth/v1.0/'
resourcename = ''
authtoken = None
headers = {"X-Storage-Pass": password, "X-Storage-User": user}
data = None
params = None
files = None
response = callRESTApi(endpoint,
basepath,
resourcename,
method='GET',
authtoken=authtoken,
headers=headers,
params=params,
data=data,
files=files)
if response is not None and 'X-Auth-Token' in response.headers:
authtoken = response.headers['X-Auth-Token']
else:
authtoken = ''
if response is not None and 'X-Storage-Url' in response.headers:
storageurl = response.headers['X-Storage-Url']
else:
storageurl = ''
return authtoken, storageurl
def uploadfile((endpoint, basepath, authtoken, filename, authendpoint, user, password, headers, params)):
print('Uploading : ' + filename)
files = None
resourcename = os.path.split(filename)[-1]
try:
with closing(open(filename, 'rb')) as f:
response = callRESTApi(endpoint, basepath, resourcename, method='PUT', authtoken=authtoken, headers=headers, params=params, data=f, files=files)
except REST401Exception as e:
# Reauthenticate and retry
if authendpoint is not None and user is not None and password is not None:
authtoken, endpoint = authenticate(authendpoint, user, password)
with closing(open(filename, 'rb')) as f:
response = callRESTApi(endpoint, basepath, resourcename, method='PUT', authtoken=authtoken, headers=headers, params=params, data=f, files=files)
else:
raise
print('Uploaded : ' + filename)
return
def createStorageContainer(endpoint, container, authtoken, **kwargs):
basepath = ''
headers = None
params = None
data = None
files = None
response = callRESTApi(endpoint, basepath, container, method='PUT', authtoken=authtoken, headers=headers, params=params, data=data, files=files)
jsonResponse = response.text
return jsonResponse
def deleteStorageObject(endpoint, resourcename, authtoken, **kwargs):
basepath = 'compute_images'
headers = None
params = None
data = None
files = None
response = callRESTApi(endpoint, basepath, resourcename, method='DELETE', authtoken=authtoken, headers=headers, params=params, data=data, files=files)
jsonResponse = response.text
return jsonResponse
def listStorageObjects(endpoint, resourcename, authtoken, limit=None, marker=None, end_marker=None, format=None, prefix=None, delimiter=None, **kwargs):
basepath = ''
headers = None
params = {}
if limit is not None:
params['limit'] = limit
if marker is not None:
params['marker'] = marker
if end_marker is not None:
params['end_marker'] = end_marker
if format is not None:
params['format'] = format
if prefix is not None:
params['prefix'] = prefix
if delimiter is not None:
params['delimiter'] = delimiter
data = None
files = None
response = callRESTApi(endpoint, basepath, resourcename, method='GET', authtoken=authtoken, headers=headers, params=params, data=data, files=files)
jsonResponse = response.text
return jsonResponse