def _make_retrying_upload_file_call(self): self.kwargs["retry_strategy"] = DEFAULT_RETRY_STRATEGY upload_manager = UploadManager(self.object_storage_client, allow_multipart_uploads=False) return upload_manager.upload_file(self.namespace_name, self.bucket_name, self.object_name, self.file_path, **self.kwargs)
def run_job(self): self.status = SUCCESS self.log_callback.info('Uploading image.') timestamp = None build_time = self.status_msg.get('build_time', 'unknown') if self.use_build_time and (build_time != 'unknown'): timestamp = timestamp_from_epoch(build_time) elif self.use_build_time and (build_time == 'unknown'): raise MashUploadException( 'use_build_time set for job but build time is unknown.' ) self.cloud_image_name = format_string_with_date( self.base_cloud_image_name, timestamp=timestamp ) self.request_credentials([self.account]) credentials = self.credentials[self.account] config = { 'user': self.oci_user_id, 'key_content': credentials['signing_key'], 'fingerprint': credentials['fingerprint'], 'tenancy': self.tenancy, 'region': self.region } object_storage = ObjectStorageClient(config) namespace = object_storage.get_namespace().data upload_manager = UploadManager( object_storage, allow_parallel_uploads=True, parallel_process_count=self.upload_process_count ) object_name = ''.join([self.cloud_image_name, '.qcow2']) self._image_size = stat(self.status_msg['image_file']).st_size with open(self.status_msg['image_file'], 'rb') as image_stream: upload_manager.upload_stream( namespace, self.bucket, object_name, image_stream, progress_callback=self._progress_callback ) self.status_msg['cloud_image_name'] = self.cloud_image_name self.status_msg['object_name'] = object_name self.status_msg['namespace'] = namespace self.log_callback.info( 'Uploaded image: {0}, to the bucket named: {1}'.format( object_name, self.bucket ) )
def __init__(self, config_file, config_section, logger=None): ''' Create ObjectStorageClient for OCI config_file : OCI Configuration File config_section : OCI Config File Section logger : Logger, if not given one will be created ''' config = from_file(config_file, config_section) self.object_storage_client = ObjectStorageClient(config, retry_strategy=DEFAULT_RETRY_STRATEGY) self.upload_manager = UploadManager(self.object_storage_client) if logger is None: self.logger = setup_logger("oci_client", 10) else: self.logger = logger
def upload_object(self, callbacks_container, namespace_name, bucket_name, object_name, file_path, file_size, **kwargs): if not self._config.use_multipart_uploads: upload_task = SimpleSingleUploadTask(self._client, namespace_name, bucket_name, object_name, file_path, callbacks_container) return self._object_storage_request_pool.submit(upload_task) part_size = self._config.multipart_part_size if 'part_size' in kwargs: part_size = kwargs['part_size'] kwargs.pop('part_size') if not UploadManager._use_multipart(file_size, part_size=part_size): if 'multipart_part_completion_callback' in kwargs: kwargs.pop('multipart_part_completion_callback') upload_task = SimpleSingleUploadTask(self._client, namespace_name, bucket_name, object_name, file_path, callbacks_container, **kwargs) return self._object_storage_request_pool.submit(upload_task) else: multipart_upload_processor_task = MultipartUploadProcessorTask( self._client, namespace_name, bucket_name, object_name, file_path, callbacks_container, self._object_storage_multipart_request_pool, part_size, **kwargs) return self._multipart_upload_processor_pool.submit( multipart_upload_processor_task)
def upload_objects_to_os_bucket(namespace): bucket_name = input("\n\tEnter the bucket name to upload objects: ") request = CreateBucketDetails(name=bucket_name, compartment_id=compartment_dets) try: bucket = object_storage.create_bucket(namespace, request) #print(bucket.data) print("\n\t" + bucket.data.etag) except Exception as e: print("\n\t" + e.message) directory = input( "\n\tEnter the path to move files to OCI object storage: ") user_input = directory assert os.path.exists( user_input), "I did not find the directory at, " + str(user_input) print("\n\tFiles in directory " + str(directory) + " will be uploaded") user_input = directory files_to_process = [ file for file in os.listdir(directory) if file.endswith('tar.gz') ] for upload_file in files_to_process: print('\n\tUploading file {}'.format(upload_file)) print('\n\t' + upload_file) partsize = 1000 * MEBIBYTE object_name = upload_file filename = os.path.join(directory, upload_file) upload_manager = UploadManager(object_storage, allow_parallel_uploads=True, allow_multipart_uploads=True) response = upload_manager.upload_file( namespace, bucket_name, object_name, filename, part_size=partsize, progress_callback=progresscallback) if str(response.data) == 'None': print("\n\tUpload Complete") return
def upload_object(self, file_properties, user=None): self.user = user file, file_name = file_properties self.set_config() object_name = "{}_{}".format(self.user, file_name) part_size = 2 * MEBIBYTE # part size (in bytes) upload_manager = UploadManager(self.object_storage, allow_parallel_uploads=True, parallel_process_count=3) file_path = self.get_media_file_path(file) response = upload_manager.upload_file( self.namespace, self.bucket_name, object_name, file_path, part_size=part_size, progress_callback=self.progress_callback) if response.status == 200: os.remove(file_path) return response
def upload_data_Object_storage (): directory = os.path.dirname(os.path.abspath(__file__)) directory +='\\civil-aviation-authority-of-the-philippines-passenger-movement-data' object_storage = oci.object_storage.ObjectStorageClient(config) files_to_process = [file for file in os.listdir(directory) if file.endswith('csv')] try: for upload_file in files_to_process: print('Uploading file {}'.format(upload_file)) print(upload_file) partsize = 1000 * MEBIBYTE print(partsize) object_name=upload_file filename=os.path.join(directory,upload_file) upload_manager=UploadManager(object_storage,allow_parallel_uploads=True,allow_multipart_uploads=True) response=upload_manager.upload_file(namespace,bucket_name,object_name,filename,part_size=partsize,progress_callback=progresscallback) print(response.data) except Exception as e: print(e.message)
bucket = object_storage.create_bucket(namespace, request) # create example file to upload filename = 'multipart_object_content.txt' file_size_in_mebibytes = 10 sample_content = b'a' with open(filename, 'wb') as f: while f.tell() < MEBIBYTE * file_size_in_mebibytes: f.write(sample_content * MEBIBYTE) print("Uploading new object {!r}".format(object_name)) # upload manager will automatically use mutlipart uploads if the part size is less than the file size part_size = 2 * MEBIBYTE # part size (in bytes) upload_manager = UploadManager(object_storage, allow_parallel_uploads=True, parallel_process_count=3) response = upload_manager.upload_file(namespace, bucket_name, object_name, filename, part_size=part_size, progress_callback=progress_callback) # To force single part uploads, set "allow_multipart_uploads=False" when creating the UploadManager. # upload_manager = UploadManager(object_storage, allow_multipart_uploads=False) # response = upload_manager.upload_file( # namespace, bucket_name, object_name, filename, part_size=part_size, progress_callback=progress_callback) # remove file to clean up os.remove(filename)
def put_object(object_storage_client, module): namespace = module.params["namespace_name"] bucket = module.params["bucket_name"] obj = module.params["object_name"] src = module.params["src"] content_type = module.params["content_type"] content_length = module.params["content_length"] content_md5 = module.params["content_md5"] content_language = module.params["content_language"] content_encoding = module.params["content_encoding"] opc_meta = dict() if module.params["opc_meta"] is not None: opc_meta = module.params["opc_meta"] result = dict(changed=False) # Check if the object exists with same checksum. remote_object = head_object(object_storage_client, module) src_md5 = base64.b64encode(base64.b16decode(module.md5(src), True)).decode("ascii") # ENHANCEMENT_OVER_SDK: This is a EoU enhancement to make it easier for an Ansible user to provide # content for their object if remote_object is not None and ( src_md5 == remote_object.headers.get("Content-MD5", None) or src_md5 == remote_object.headers.get("opc-multipart-md5", None) ): changed = False elif module.params.get("multipart_upload"): # Note: If the file size is less than 128 MB, UploadManager will automatically chose the upload strategy # to be single-part upload. upload_manager = UploadManager( object_storage_client, allow_parallel_uploads=module.params.get("parallel_uploads"), ) response = oci_utils.call_with_backoff( upload_manager.upload_file, namespace_name=namespace, bucket_name=bucket, object_name=obj, file_path=src, ) changed = True result["object"] = dict(response.headers) else: with open(to_bytes(src), "rb") as src_file: put_object_body = src_file.read() try: response = oci_utils.call_with_backoff( object_storage_client.put_object, namespace_name=namespace, bucket_name=bucket, object_name=obj, put_object_body=put_object_body, content_encoding=content_encoding, content_language=content_language, content_length=content_length, content_md5=content_md5, content_type=content_type, opc_meta=opc_meta, ) changed = True result["object"] = dict(response.headers) except ServiceError as ex: module.fail_json(msg=ex.message) result["changed"] = changed return result
def upload_object(self, src, force): if not os.path.isfile(to_bytes(src)): self.module.fail_json(msg="The source path %s must be a file." % src) if not os.access(to_bytes(src), os.R_OK): self.module.fail_json( msg= "Failed to access %s. Make sure the file exists and that you have " "read access." % src) object_exists = False try: response = self.get_resource() object_exists = True except ServiceError as ex: if ex.status != 404: self.module.fail_json(msg=ex.message) if object_exists and force is False: return self.prepare_result( changed=False, resource_type=self.resource_type, resource=to_dict(response.data), msg= "Object %s already present in bucket. Use force option to overwrite." % self.module.params.get("object_name"), ) src_md5 = base64.b64encode(base64.b16decode(self.module.md5(src), True)).decode("ascii") if object_exists and ( src_md5 == response.headers.get("Content-MD5", None) or src_md5 == response.headers.get("opc-multipart-md5", None)): return self.prepare_result( changed=False, resource_type=self.resource_type, resource=to_dict(response.data), ) try: # sdk complains if the value is None self.module.params["opc_meta"] = ( self.module.params.get("opc_meta") or dict()) upload_manager = UploadManager(self.client, ) oci_common_utils.call_with_backoff( upload_manager.upload_file, namespace_name=self.module.params.get("namespace_name"), bucket_name=self.module.params.get("bucket_name"), object_name=self.module.params.get("object_name"), content_encoding=self.module.params.get("content_encoding"), content_language=self.module.params.get("content_language"), content_length=self.module.params.get("content_length"), content_md5=self.module.params.get("content_md5"), content_type=self.module.params.get("content_type"), content_disposition=self.module.params.get( "content_disposition"), cache_control=self.module.params.get("cache_control"), opc_sse_customer_algorithm=self.module.params.get( "opc_sse_customer_algorithm"), opc_sse_customer_key=self.module.params.get( "opc_sse_customer_key"), opc_sse_customer_key_sha256=self.module.params.get( "opc_sse_customer_key_sha256"), opc_meta=self.module.params.get("opc_meta"), file_path=src, ) uploaded_object_response = self.get_resource() return self.prepare_result( changed=True, resource_type=self.resource_type, resource=to_dict(uploaded_object_response.data), ) except ServiceError as ex: self.module.fail_json(msg=ex.message)
def _make_retrying_upload_file_call(self): upload_manager = UploadManager(self.object_storage_client, allow_multipart_uploads=False) return upload_manager.upload_file(self.namespace_name, self.bucket_name, self.object_name, self.file_path, **self.kwargs)
class OCIObjectStorageClient(): ''' Object Storage Client ''' def __init__(self, config_file, config_section, logger=None): ''' Create ObjectStorageClient for OCI config_file : OCI Configuration File config_section : OCI Config File Section logger : Logger, if not given one will be created ''' config = from_file(config_file, config_section) self.object_storage_client = ObjectStorageClient(config, retry_strategy=DEFAULT_RETRY_STRATEGY) self.upload_manager = UploadManager(self.object_storage_client) if logger is None: self.logger = setup_logger("oci_client", 10) else: self.logger = logger def object_list(self, namespace_name, bucket_name): ''' Return list of object storage objects namespace_name : Object Storage Namespace bucket_name : Bucket Name ''' self.logger.info("Retrieving object list from namespace %s and bucket %s", namespace_name, bucket_name) all_objects_response = list_call_get_all_results(self.object_storage_client.list_objects, namespace_name, bucket_name, fields='name,md5,size,timeCreated') return [to_dict(obj) for obj in all_objects_response.data.objects] def object_put(self, namespace_name, bucket_name, object_name, file_name, md5_sum=None, resume_upload=False): ''' Upload object to object storage namespace_name : Object Storage Namespace bucket_name : Bucket name object_name : Name of uploaded object file_name : Name of local file to upload md5_sum : Md5 sum of local file ''' self.logger.info(f'Starting upload of file "{file_name}" to namespace "{namespace_name}" ' f'bucket "{bucket_name}" and object name "{object_name}"') upload_resumed = False if resume_upload: self.logger.debug(f'Checking if object name "{object_name}" is in list of pending uploads') multipart_uploads = list_call_get_all_results(self.object_storage_client.list_multipart_uploads, namespace_name, bucket_name) for multipart_upload in multipart_uploads.data: # Assume namespace and bucket are the same if multipart_upload.object == object_name: self.logger.debug(f'Resuming file upload {multipart_upload.upload_id} for object "{object_name}"') response = self.upload_manager.resume_upload_file(namespace_name, bucket_name, object_name, file_name, multipart_upload.upload_id) upload_resumed = True break if not upload_resumed: response = self.upload_manager.upload_file(namespace_name, bucket_name, object_name, file_name, content_md5=md5_sum) if response.status != 200: raise ObjectStorageException(f'Error uploading object, Reponse code {str(response.status)}') self.logger.info(f'File "{file_name}" uploaded to object storage with object name "{object_name}"') return True def object_get(self, namespace_name, bucket_name, object_name, file_name, set_restore=False): ''' Download object from object storage namespace_name : Object Storage Namespace bucket_name : Bucket name object_name : Name of object to download file_name : Name of local file where object will be downloaded set_restore : If object is archived, run "set_restore" ''' self.logger.info(f'Downloading object "{object_name}" from namespace "{namespace_name}" and bucket "{bucket_name}" to file "{file_name}"') with open(file_name, 'wb') as writer: try: get_response = self.object_storage_client.get_object(namespace_name, bucket_name, object_name) except ServiceError as error: self.logger.exception(f'Service Error when attempting to download object: {str(error)}') if set_restore and "'code': 'NotRestored'" in str(error): self.logger.debug(f'Object "{object_name}" in bucket "{bucket_name}" and namepsace ' f'"{namespace_name}" is archived, will mark for restore') restore_details = RestoreObjectsDetails(object_name=object_name) restore_response = self.object_storage_client.restore_objects(namespace_name, bucket_name, restore_details) if restore_response.status != 202: raise ObjectStorageException('Error restoring object, ' # pylint:disable=raise-missing-from f'Response code {str(restore_response.status)}') self.logger.info(f'Set restore on object "{object_name}" in bucket "{bucket_name}" and namespace "{namespace_name}"') return False if get_response.status != 200: raise ObjectStorageException(f'Error downloading object, Response code {str(get_response.status)}') self.logger.debug(f'Writing object "{object_name}" to file "{file_name}"') shutil.copyfileobj(get_response.data.raw, writer) return True def object_delete(self, namespace_name, bucket_name, object_name): ''' Delete object in object storage namespace_name : Object Storage Namespace bucket_name : Bucket name object_name : Name of object to delete ''' self.logger.info(f'Deleting object "{object_name}" from namespace "{namespace_name}" and bucket "{bucket_name}"') response = self.object_storage_client.delete_object(namespace_name, bucket_name, object_name) if response.status != 204: raise ObjectStorageException(f'Error deleting object, Reponse code {str(response.status)}') return True