def _make_retrying_upload_file_call(self):
self.kwargs["retry_strategy"] = DEFAULT_RETRY_STRATEGY
upload_manager = UploadManager(self.object_storage_client,
allow_multipart_uploads=False)
return upload_manager.upload_file(self.namespace_name,
self.bucket_name, self.object_name,
self.file_path, **self.kwargs)
def upload_objects_to_os_bucket(namespace):
bucket_name = input("\n\tEnter the bucket name to upload objects: ")
request = CreateBucketDetails(name=bucket_name,
compartment_id=compartment_dets)
try:
bucket = object_storage.create_bucket(namespace, request)
#print(bucket.data)
print("\n\t" + bucket.data.etag)
except Exception as e:
print("\n\t" + e.message)
directory = input(
"\n\tEnter the path to move files to OCI object storage: ")
user_input = directory
assert os.path.exists(
user_input), "I did not find the directory at, " + str(user_input)
print("\n\tFiles in directory " + str(directory) + " will be uploaded")
user_input = directory
files_to_process = [
file for file in os.listdir(directory) if file.endswith('tar.gz')
]
for upload_file in files_to_process:
print('\n\tUploading file {}'.format(upload_file))
print('\n\t' + upload_file)
partsize = 1000 * MEBIBYTE
object_name = upload_file
filename = os.path.join(directory, upload_file)
upload_manager = UploadManager(object_storage,
allow_parallel_uploads=True,
allow_multipart_uploads=True)
response = upload_manager.upload_file(
namespace,
bucket_name,
object_name,
filename,
part_size=partsize,
progress_callback=progresscallback)
if str(response.data) == 'None':
print("\n\tUpload Complete")
return
def upload_object(self, file_properties, user=None):
self.user = user
file, file_name = file_properties
self.set_config()
object_name = "{}_{}".format(self.user, file_name)
part_size = 2 * MEBIBYTE # part size (in bytes)
upload_manager = UploadManager(self.object_storage,
allow_parallel_uploads=True,
parallel_process_count=3)
file_path = self.get_media_file_path(file)
response = upload_manager.upload_file(
self.namespace,
self.bucket_name,
object_name,
file_path,
part_size=part_size,
progress_callback=self.progress_callback)
if response.status == 200:
os.remove(file_path)
return response
def upload_data_Object_storage ():
directory = os.path.dirname(os.path.abspath(__file__))
directory +='\\civil-aviation-authority-of-the-philippines-passenger-movement-data'
object_storage = oci.object_storage.ObjectStorageClient(config)
files_to_process = [file for file in os.listdir(directory) if file.endswith('csv')]
try:
for upload_file in files_to_process:
print('Uploading file {}'.format(upload_file))
print(upload_file)
partsize = 1000 * MEBIBYTE
print(partsize)
object_name=upload_file
filename=os.path.join(directory,upload_file)
upload_manager=UploadManager(object_storage,allow_parallel_uploads=True,allow_multipart_uploads=True)
response=upload_manager.upload_file(namespace,bucket_name,object_name,filename,part_size=partsize,progress_callback=progresscallback)
print(response.data)
except Exception as e:
print(e.message)
file_size_in_mebibytes = 10
sample_content = b'a'
with open(filename, 'wb') as f:
while f.tell() < MEBIBYTE * file_size_in_mebibytes:
f.write(sample_content * MEBIBYTE)
print("Uploading new object {!r}".format(object_name))
# upload manager will automatically use mutlipart uploads if the part size is less than the file size
part_size = 2 * MEBIBYTE # part size (in bytes)
upload_manager = UploadManager(object_storage,
allow_parallel_uploads=True,
parallel_process_count=3)
response = upload_manager.upload_file(namespace,
bucket_name,
object_name,
filename,
part_size=part_size,
progress_callback=progress_callback)
# To force single part uploads, set "allow_multipart_uploads=False" when creating the UploadManager.
# upload_manager = UploadManager(object_storage, allow_multipart_uploads=False)
# response = upload_manager.upload_file(
# namespace, bucket_name, object_name, filename, part_size=part_size, progress_callback=progress_callback)
# remove file to clean up
os.remove(filename)
print("Deleting object {}".format(object_name))
object_storage.delete_object(namespace, bucket_name, object_name)
print("Deleting bucket {}".format(bucket_name))
def _make_retrying_upload_file_call(self):
upload_manager = UploadManager(self.object_storage_client,
allow_multipart_uploads=False)
return upload_manager.upload_file(self.namespace_name,
self.bucket_name, self.object_name,
self.file_path, **self.kwargs)
class OCIObjectStorageClient():
'''
Object Storage Client
'''
def __init__(self, config_file, config_section, logger=None):
'''
Create ObjectStorageClient for OCI
config_file : OCI Configuration File
config_section : OCI Config File Section
logger : Logger, if not given one will be created
'''
config = from_file(config_file, config_section)
self.object_storage_client = ObjectStorageClient(config, retry_strategy=DEFAULT_RETRY_STRATEGY)
self.upload_manager = UploadManager(self.object_storage_client)
if logger is None:
self.logger = setup_logger("oci_client", 10)
else:
self.logger = logger
def object_list(self, namespace_name, bucket_name):
'''
Return list of object storage objects
namespace_name : Object Storage Namespace
bucket_name : Bucket Name
'''
self.logger.info("Retrieving object list from namespace %s and bucket %s",
namespace_name, bucket_name)
all_objects_response = list_call_get_all_results(self.object_storage_client.list_objects,
namespace_name, bucket_name, fields='name,md5,size,timeCreated')
return [to_dict(obj) for obj in all_objects_response.data.objects]
def object_put(self, namespace_name, bucket_name, object_name, file_name, md5_sum=None, resume_upload=False):
'''
Upload object to object storage
namespace_name : Object Storage Namespace
bucket_name : Bucket name
object_name : Name of uploaded object
file_name : Name of local file to upload
md5_sum : Md5 sum of local file
'''
self.logger.info(f'Starting upload of file "{file_name}" to namespace "{namespace_name}" '
f'bucket "{bucket_name}" and object name "{object_name}"')
upload_resumed = False
if resume_upload:
self.logger.debug(f'Checking if object name "{object_name}" is in list of pending uploads')
multipart_uploads = list_call_get_all_results(self.object_storage_client.list_multipart_uploads,
namespace_name, bucket_name)
for multipart_upload in multipart_uploads.data:
# Assume namespace and bucket are the same
if multipart_upload.object == object_name:
self.logger.debug(f'Resuming file upload {multipart_upload.upload_id} for object "{object_name}"')
response = self.upload_manager.resume_upload_file(namespace_name, bucket_name, object_name, file_name, multipart_upload.upload_id)
upload_resumed = True
break
if not upload_resumed:
response = self.upload_manager.upload_file(namespace_name, bucket_name, object_name, file_name, content_md5=md5_sum)
if response.status != 200:
raise ObjectStorageException(f'Error uploading object, Reponse code {str(response.status)}')
self.logger.info(f'File "{file_name}" uploaded to object storage with object name "{object_name}"')
return True
def object_get(self, namespace_name, bucket_name, object_name, file_name, set_restore=False):
'''
Download object from object storage
namespace_name : Object Storage Namespace
bucket_name : Bucket name
object_name : Name of object to download
file_name : Name of local file where object will be downloaded
set_restore : If object is archived, run "set_restore"
'''
self.logger.info(f'Downloading object "{object_name}" from namespace "{namespace_name}" and bucket "{bucket_name}" to file "{file_name}"')
with open(file_name, 'wb') as writer:
try:
get_response = self.object_storage_client.get_object(namespace_name,
bucket_name,
object_name)
except ServiceError as error:
self.logger.exception(f'Service Error when attempting to download object: {str(error)}')
if set_restore and "'code': 'NotRestored'" in str(error):
self.logger.debug(f'Object "{object_name}" in bucket "{bucket_name}" and namepsace '
f'"{namespace_name}" is archived, will mark for restore')
restore_details = RestoreObjectsDetails(object_name=object_name)
restore_response = self.object_storage_client.restore_objects(namespace_name, bucket_name, restore_details)
if restore_response.status != 202:
raise ObjectStorageException('Error restoring object, ' # pylint:disable=raise-missing-from
f'Response code {str(restore_response.status)}')
self.logger.info(f'Set restore on object "{object_name}" in bucket "{bucket_name}" and namespace "{namespace_name}"')
return False
if get_response.status != 200:
raise ObjectStorageException(f'Error downloading object, Response code {str(get_response.status)}')
self.logger.debug(f'Writing object "{object_name}" to file "{file_name}"')
shutil.copyfileobj(get_response.data.raw, writer)
return True
def object_delete(self, namespace_name, bucket_name, object_name):
'''
Delete object in object storage
namespace_name : Object Storage Namespace
bucket_name : Bucket name
object_name : Name of object to delete
'''
self.logger.info(f'Deleting object "{object_name}" from namespace "{namespace_name}" and bucket "{bucket_name}"')
response = self.object_storage_client.delete_object(namespace_name,
bucket_name,
object_name)
if response.status != 204:
raise ObjectStorageException(f'Error deleting object, Reponse code {str(response.status)}')
return True
