def upload_part(self, bucket_name, object_name, upload_id, part_number,
data):
'''
Upload a multipart upload part
:param bucket_name:
:param object_name:
:param upload_id:
:param part_number:
:param data:
:return:
'''
uri = '%s%s/%s?%s%s' % (self._config.get_base_uri(), bucket_name,
object_name, "uploadId=" + upload_id,
"&partNumber=" + str(part_number))
response = self._request.put(uri, auth=self._auth, data=data)
if response.status_code == requests.codes.ok:
result = UploadPartResult(json.loads(response.content))
return result
else:
headers = ""
if self._config.debug:
headers = ' headers=%s' % response.headers
message = 'Upload part failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def list_next_batch_of_objects(self, previous):
'''
List objects in a iterative manner
:param previous: The FDSObjectListing returned by previous call or list_objects
:return: FDSObjectListing contains FDSObject list and other metadata, 'None'
if all objects returned by previous calls
'''
if not previous.is_truncated:
return None
bucket_name = previous.bucket_name
prefix = previous.prefix
delimiter = previous.delimiter
marker = previous.next_marker
uri = "%s%s?prefix=%s&delimiter=%s&marker=%s" % \
(self._config.get_base_uri(), bucket_name, prefix, delimiter, marker)
response = self._request.get(uri, auth=self._auth)
if response.status_code == requests.codes.ok:
objects_list = FDSObjectListing(json.loads(response.content))
return objects_list
else:
headers = ""
if self._config.debug:
headers = ' header=%s' % response.headers
message = 'List next batch of objects under bucket %s with prefix %s ' \
'and marker %s failed, status=%s, reason=%s%s' % \
(bucket_name, prefix, marker, response.status_code, response.content,
headers)
raise GalaxyFDSClientException(message)
def complete_multipart_upload(self, bucket_name, object_name, upload_id,
metadata, upload_part_result_list):
'''
Complete a multipart upload
:param bucket_name:
:param object_name:
:param upload_id:
:param metadata:
:param upload_part_result_list:
:return:
'''
uri = '%s%s/%s?%s' % (self._config.get_base_uri(), bucket_name,
object_name, "uploadId=" + upload_id)
if metadata is None:
metadata = FDSObjectMetadata()
response = self._request.put(uri,
auth=self._auth,
data=upload_part_result_list,
headers=metadata.metadata)
if response.status_code == requests.codes.ok:
result = PutObjectResult(json.loads(response.content))
return result
else:
headers = ""
if self._config.debug:
headers = ' headers=%s' % response.headers
message = 'Complete multipart upload failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def post_object(self, bucket_name, data, metadata=None):
'''
Post the object to a specified bucket. The object name will be generated
by the server uniquely.
:param bucket_name: The name of the bucket to whom the object is put
:param data: The data to put, bytes or a file like object
:param metadata: The metadata of the object
:return: The result of posting action server returns
'''
uri = '%s%s/' % (self._config.get_upload_base_uri(), bucket_name)
if metadata is None:
metadata = FDSObjectMetadata()
if self._config.enable_md5_calculate:
digest = hashlib.md5()
digest.update(data)
metadata.add_header(Common.CONTENT_MD5, digest.hexdigest())
response = self._request.post(uri,
data=data,
auth=self._auth,
headers=metadata.metadata)
if response.status_code == requests.codes.ok:
return PutObjectResult(json.loads(response.content))
headers = ""
if self._config.debug:
headers = ' header=%s' % response.headers
message = 'Post object failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def list_objects(self, bucket_name, prefix='', delimiter=None):
'''
List all objects in a specified bucket with prefix. If the number of objects
in the bucket is larger than a threshold, you would get a FDSObjectListing
contains no FDSObjects. In this scenario, you should call
list_next_batch_of_objects with the returned value
:param bucket_name: The name of the bucket to whom the object is put
:param prefix: The prefix of the object to list
:param delimiter: The delimiter used in listing, using '/' if 'None' given
:return: FDSObjectListing contains FDSObject list and other metadata
'''
if delimiter is None:
delimiter = self._delimiter
uri = '%s%s?prefix=%s&delimiter=%s' % \
(self._config.get_base_uri(), bucket_name, prefix, delimiter)
response = self._request.get(uri, auth=self._auth)
if response.status_code == requests.codes.ok:
objects_list = FDSObjectListing(json.loads(response.content))
return objects_list
else:
headers = ""
if self._config.debug:
headers = ' header=%s' % response.headers
message = 'List objects under bucket %s with prefix %s failed, ' \
'status=%s, reason=%s%s' % \
(bucket_name, prefix, response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def get_object(self,
bucket_name,
object_name,
position=0,
size=4096,
stream=None):
'''
Get a specified object from a bucket.
:param bucket_name: The name of the bucket from whom to get the object
:param object_name: The name of the object to get
:param position: The start index of object to get
:param size: The maximum size of each piece when return streaming is on
:param stream: Set True to enable streaming, otherwise, whole object content is read to memory
:return: The FDS object
'''
if position < 0:
raise GalaxyFDSClientException(
"Seek position should be no less than 0")
uri = '%s%s/%s' % (self._config.get_download_base_uri(), bucket_name,
object_name)
if position > 0:
header = {Common.RANGE: 'bytes=%d-' % position}
response = self._request.get(uri,
auth=self._auth,
headers=header,
stream=stream)
else:
response = self._request.get(uri, auth=self._auth, stream=stream)
if response.status_code == requests.codes.ok or \
response.status_code == requests.codes.partial:
obj = FDSObject()
obj.stream = response.iter_content(chunk_size=size)
summary = FDSObjectSummary()
summary.bucket_name = bucket_name
summary.object_name = object_name
summary.size = int(response.headers['content-length'])
obj.summary = summary
obj.metadata = self._parse_object_metadata_from_headers(
response.headers)
return obj
else:
headers = ""
if self._config.debug:
headers = ' header=%s' % response.headers
message = 'Get object failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def create_bucket(self, bucket_name):
'''
Create a bucket with the specified name.
:param bucket_name: The name of the bucket to create
'''
uri = '%s%s' % (self._config.get_base_uri(), bucket_name)
response = self._request.put(uri, auth=self._auth)
if response.status_code != requests.codes.ok:
headers = ""
if self._config.debug:
headers = ' header=%s' % response.headers
message = 'Create bucket failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def delete_bucket(self, bucket_name):
'''
Delete a bucket of a specified name.
:param bucket_name: The name of the bucket to delete
'''
uri = '%s%s' % (self._config.get_base_uri(), bucket_name)
response = self._request.delete(uri, auth=self._auth)
if (response.status_code != requests.codes.ok
and response.status_code != requests.codes.not_found):
headers = ""
if self._config.debug:
headers = ' header=%s' % response.headers
message = 'Delete bucket failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def restore_object(self, bucket_name, object_name):
'''
Restore a specified object from trash.
:param bucket_name: The name of the bucket
:param object_name: The name of the object
'''
uri = '%s%s/%srestore=' % (self._config.get_base_uri(), bucket_name,
object_name)
response = self._request.put(uri, auth=self._auth)
if response.status_code != requests.codes.ok:
headers = ""
if self._config.debug:
headers = ' header=%s' % response.headers
message = 'Restore object failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def refresh_object(self, bucket_name, object_name):
'''
Refresh the cache of the object in CDN
:param bucket_name: The name of the bucket
:param object_name: The name of the object
:return: void
'''
uri = '%s%s/%s?%s' % (self._config.get_base_uri(), bucket_name,
object_name, "refresh")
response = self._request.put(uri, auth=self._auth, data="")
if response.status_code != requests.codes.ok:
headers = ""
if self._config.debug:
headers = ' header=%s' % response.headers
message = 'Refresh object failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def abort_multipart_upload(self, bucket_name, object_name, upload_id):
'''
Abort a multipart upload
:param bucket_name:
:param object_name:
:param upload_id:
:return:
'''
uri = '%s%s/%s?%s' % (self._config.get_base_uri(), bucket_name,
object_name, "uploadId=" + upload_id)
response = self._request.delete(uri, auth=self._auth, data='')
if response.status_code != requests.codes.ok:
headers = ""
if self._config.debug:
headers = ' headers=%s' % response.headers
message = 'Abort multipart upload failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def rename_object(self, bucket_name, src_object_name, dst_object_name):
'''
Rename a specified object to a new name.
:param bucket_name: The name of the bucket
:param src_object_name: The original name of the object
:param dst_object_name: The target name of the object to rename to
'''
uri = '%s%s/%s?renameTo=%s' % (self._config.get_base_uri(),
bucket_name, src_object_name,
dst_object_name)
response = self._request.put(uri, auth=self._auth)
if response.status_code != requests.codes.ok:
headers = ""
if self._config.debug:
headers = ' header=%s' % response.headers
message = 'Rename object failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def does_bucket_exist(self, bucket_name):
'''
Check the existence of a specified bucket.
:param bucket_name: The bucket name of the bucket to check
:return: True if the bucket exists, otherwise False
'''
uri = '%s%s' % (self._config.get_base_uri(), bucket_name)
response = self._request.head(uri, auth=self._auth)
if response.status_code == requests.codes.ok:
return True
elif response.status_code == requests.codes.not_found:
return False
else:
headers = ""
if self._config.debug:
headers = ' header=%s' % response.headers
message = 'Check bucket existence failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def set_bucket_acl(self, bucket_name, acl):
'''
Add grant(ACL) for specified bucket.
:param bucket_name: The name of the bucket to add grant
:param acl: The grant(ACL) to add
'''
uri = '%s%s?%s' % (self._config.get_base_uri(), bucket_name,
SubResource.ACL)
acp = self._acl_to_acp(acl)
response = self._request.put(uri,
auth=self._auth,
data=json.dumps(
acp, default=lambda x: x.to_string()))
if response.status_code != requests.codes.ok:
headers = ""
if self._config.debug:
headers = ' header=%s' % response.headers
message = 'Set bucket acl failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def get_bucket_acl(self, bucket_name):
'''
Get the ACL of a specified bucket.
:param bucket_name: The name of the bucket to get ACL
:return: The got access control list
'''
uri = '%s%s?%s' % (self._config.get_base_uri(), bucket_name,
SubResource.ACL)
response = self._request.get(uri, auth=self._auth)
if response.status_code == requests.codes.ok:
acp = AccessControlPolicy(json.loads(response.content))
acl = self._acp_to_acl(acp)
return acl
else:
headers = ""
if self._config.debug:
headers = ' header=%s' % response.headers
message = 'Get bucket acl failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def init_multipart_upload(self, bucket_name, object_name):
'''
Init a multipart upload session
:param bucket_name:
:param object_name:
:return:
'''
uri = '%s%s/%s?%s' % (self._config.get_base_uri(), bucket_name,
object_name, "uploads")
response = self._request.put(uri, auth=self._auth, data="")
if response.status_code == requests.codes.ok:
result = InitMultipartUploadResult(json.loads(response.content))
return result
else:
headers = ""
if self._config.debug:
headers = ' headers=%s' % response.headers
message = 'Init multipart upload failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def generate_presigned_uri(self,
base_uri,
bucket_name,
object_name,
expiration,
http_method="GET",
content_type=None):
'''
Generate a pre-signed uri to share object with the public
:param base_uri: The base uri of rest server. Use client's default if 'None' pass
:param bucket_name: The name of the bucket
:param object_name: The name of the object
:param expiration: The expiration time of the uri: milliseconds from the Epoch
:param http_method: The http method used in uri
:return: The pre-signed uri string
'''
if not base_uri or base_uri == '':
if http_method == 'PUT' or http_method == 'POST':
base_uri = self._config.get_upload_base_uri()
elif http_method == 'DELETE':
base_uri = self._config.get_base_uri()
else:
base_uri = self._config.get_download_base_uri()
try:
uri = '%s%s/%s?%s=%s&%s=%s&' % \
(base_uri, bucket_name, object_name, \
Common.GALAXY_ACCESS_KEY_ID, self._auth._app_key, \
Common.EXPIRES, str(int(expiration)))
headers = None
if content_type != None and isinstance(content_type, basestring):
headers = {Common.CONTENT_TYPE: content_type}
signature = str(self._auth._sign_to_base64(http_method, headers, uri, \
self._auth._app_secret))
return '%s%s/%s?%s=%s&%s=%s&%s=%s' % \
(base_uri, quote(bucket_name), quote(object_name), \
Common.GALAXY_ACCESS_KEY_ID, self._auth._app_key, \
Common.EXPIRES, str(int(expiration)), Common.SIGNATURE, signature)
except Exception as e:
message = 'Wrong expiration given. ' \
'Milliseconds since January 1, 1970 should be used. ' + str(e)
raise GalaxyFDSClientException(message)
def get_object_metadata(self, bucket_name, object_name):
'''
Get the metadata of a specified object.
:param bucket_name: The name of the bucket
:param object_name: The name of the object
:return: The got object metadata
'''
uri = '%s%s/%s?%s' % (self._config.get_base_uri(), bucket_name,
object_name, SubResource.METADATA)
response = self._request.get(uri, auth=self._auth)
if response.status_code == requests.codes.ok:
metadata = self._parse_object_metadata_from_headers(
response.headers)
return metadata
else:
headers = ""
if self._config.debug:
headers = ' header=%s' % response.headers
message = 'Get object metadata failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
def list_authorized_buckets(self):
'''
List all the authorized buckets of the current developer.
:return: A list of FDSBucket which contains name and owner of the bucket.
'''
uri = self._config.get_base_uri() + '?authorizedBuckets'
response = self._request.get(uri, auth=self._auth)
if response.status_code != requests.codes.ok:
headers = ""
if self._config.debug:
headers = ' header=%s' % response.headers
message = 'List buckets failed, status=%s, reason=%s%s' % (
response.status_code, response.content, headers)
raise GalaxyFDSClientException(message)
elif response.content:
buckets_list = []
json_response = json.loads(response.content)
buckets = json_response['buckets']
for bucket in buckets:
buckets_list.append(FDSBucket(bucket['name'], ''))
return buckets_list
else:
return list()