class MakeBucket(BaseOperator): """ Create a Bucket in OCI object store :param bucket_name: Name of bucket :type bucket_name: str :param compartment_ocid: Compartment ID :type compartment_id: str :param namespace_name: Object storage namespace :type namespace_name: str :param oci_conn_id: Airflow connection ID :type oci_conn_id: str """ @apply_defaults def __init__(self, bucket_name: str, compartment_ocid: str, namespace_name: Optional[str] = None, oci_conn_id: Optional[str] = "oci_default", *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.bucket_name = bucket_name self.compartment_id = compartment_ocid self.namespace_name = namespace_name self.oci_conn_id = oci_conn_id self._oci_hook = None self.oci_client = oci.object_storage.ObjectStorageClient def execute(self, context, **kwargs): self._oci_hook = OCIObjectStorageHook( compartment_id=self.compartment_id, bucket_name=self.bucket_name, oci_conn_id=self.oci_conn_id, namespace_name=self.namespace_name) client = self._oci_hook.get_client(self.oci_client) self.log.info("Validating OCI Config") self._oci_hook.validate_config() if not self.namespace_name: self.namespace_name = self._oci_hook.get_namespace() details = oci.object_storage.models.CreateBucketDetails( compartment_id=self.compartment_id, name=self.bucket_name) self.log.info("Checking if Bucket {} exists".format(self.bucket_name)) bucket_exists = self._oci_hook.check_for_bucket( namespace_name=self.namespace_name, bucket_name=self.bucket_name) if bucket_exists is True: self.log.info("Bucket {0} exists, skipping creation".format( self.bucket_name)) else: self.log.info("Creating Bucket {0} in {1}".format( self.bucket_name, self.namespace_name)) client.create_bucket(namespace_name=self.namespace_name, create_bucket_details=details, **kwargs) self.log.info("Create bucket complete")
class CopyFileToOCIObjectStorageOperator(BaseOperator): """ Copy local file to OCI object store :param bucket_name: Name of bucket :type bucket_name: str :param compartment_ocid: Compartment ID :type compartment_id: str :param object_name: Object name - must match local file :type object_name: str :param local_file_path: Path to local file :type local_file_path: str :param namespace_name: Object storage namespace :type namespace_name: str :param oci_conn_id: Airflow connection ID :type oci_conn_id: str :param overwrite: Overwrite files if they exist :type overwrite: bool """ @apply_defaults def __init__(self, bucket_name: str, compartment_ocid: str, object_name: str, local_file_path: str, namespace_name: Optional[str] = None, oci_conn_id: Optional[str] = "oci_default", overwrite: Optional[bool] = False, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.bucket_name = bucket_name self.compartment_id = compartment_ocid self.namespace_name = namespace_name self.object_name = object_name self.local_file_path = local_file_path self.oci_conn_id = oci_conn_id self.overwrite = overwrite self._oci_hook = None self.oci_client = oci.object_storage.ObjectStorageClient def execute(self, context, **kwargs): self._oci_hook = OCIObjectStorageHook( compartment_id=self.compartment_id, bucket_name=self.bucket_name, oci_conn_id=self.oci_conn_id) client = self._oci_hook.get_client(self.oci_client) self.log.info("Validating OCI Config") self._oci_hook.validate_config() if not self.namespace_name: self.namespace_name = self._oci_hook.get_namespace() details = oci.object_storage.models.CreateBucketDetails( compartment_id=self.compartment_id, name=self.bucket_name) self.log.info("Checking if Bucket {} exists".format(self.bucket_name)) bucket_exists = self._oci_hook.check_for_bucket( namespace_name=self.namespace_name, bucket_name=self.bucket_name) if bucket_exists is True: self.log.info("Bucket {0} exists, skipping creation".format( self.bucket_name)) else: self.log.info("Creating Bucket {0} in {1}".format( self.bucket_name, self.namespace_name)) client.create_bucket(namespace_name=self.namespace_name, create_bucket_details=details) self.log.info("Create bucket complete") self.log.info("Checking if {0} exists in {1}".format( self.object_name, self.bucket_name)) object_exists = self._oci_hook.check_for_object( namespace_name=self.namespace_name, bucket_name=self.bucket_name, object_name=self.object_name) if object_exists is True: if self.overwrite is True: self.log.info("Validating local file {0} exists".format( self.object_name)) if path.exists(self.local_file_path) is True: self.local_file = self.local_file_path + self.object_name if path.exists(self.local_file) is True: self.log.info("Copying {0} to {1}".format( self.local_file, self.bucket_name)) self.put_object_body = open(self.local_file, 'rb') self._oci_hook.copy_to_bucket( bucket_name=self.bucket_name, namespace_name=self.namespace_name, object_name=self.object_name, put_object_body=self.put_object_body, **kwargs) else: self.log.error("Local file {0} does not exist".format( self.local_file)) else: self.log.error("Local file path {0} does not exist".format( self.local_file_path)) else: self.log.info("Object {0} exists already in {1}".format( self.object_name, self.bucket_name)) else: self.log.info("Validating local file {0} exists".format( self.object_name)) if path.exists(self.local_file_path) is True: self.local_file = self.local_file_path + self.object_name if path.exists(self.local_file) is True: self.log.info("Copying {0} to {1}".format( self.local_file, self.bucket_name)) self.put_object_body = open(self.local_file, 'rb') self._oci_hook.copy_to_bucket( bucket_name=self.bucket_name, namespace_name=self.namespace_name, object_name=self.object_name, put_object_body=self.put_object_body, **kwargs) else: self.log.error("Local file {0} does not exist".format( self.local_file)) else: self.log.error("Local file path {0} does not exist".format( self.local_file_path))
class CopyToOCIObjectStorageOperator(BaseOperator): """ Copy data to OCI object store :param bucket_name: Name of target bucket :type bucket_name: str :param compartment_ocid: Compartment ID :type compartment_id: str :param object_name: Object name to create in object store :type object_name: str :param put_object_body: Contents of object_name :type put_object_body: stream :param oci_conn_id: Airflow connection ID :type oci_conn_id: str """ @apply_defaults def __init__(self, bucket_name: str, compartment_ocid: str, object_name: str, put_object_body: str, oci_conn_id: Optional[str] = "oci_default", *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.bucket_name = bucket_name self.compartment_id = compartment_ocid self.object_name = object_name self.put_object_body = put_object_body self.oci_conn_id = oci_conn_id self._oci_hook = None self.oci_client = oci.object_storage.ObjectStorageClient def execute(self, context, **kwargs): self._oci_hook = OCIObjectStorageHook( compartment_id=self.compartment_id, bucket_name=self.bucket_name, oci_conn_id=self.oci_conn_id) client = self._oci_hook.get_client(self.oci_client) self.log.info("Validating OCI Config") self._oci_hook.validate_config() namespace = self._oci_hook.get_namespace() details = oci.object_storage.models.CreateBucketDetails( compartment_id=self.compartment_id, name=self.bucket_name) self.log.info("Checking if Bucket {} exists".format(self.bucket_name)) bucket_exists = self._oci_hook.check_for_bucket( namespace_name=namespace, bucket_name=self.bucket_name) if bucket_exists is True: self.log.info("Bucket {0} exists, skipping creation".format( self.bucket_name)) else: self.log.info("Creating Bucket {0} in {1}".format( self.bucket_name, namespace)) client.create_bucket(namespace_name=namespace, create_bucket_details=details) self.log.info("Create bucket complete") self.log.info("Checking if {0} exists in {1}".format( self.object_name, self.bucket_name)) object_exists = self._oci_hook.check_for_object( namespace_name=namespace, bucket_name=self.bucket_name, object_name=self.object_name) if object_exists is True: self.log.info("Object {0} exists already in {1}".format( self.object_name, self.bucket_name)) else: self.log.info("Copying {0} to {1}".format(self.object_name, self.bucket_name)) self._oci_hook.copy_to_bucket(bucket_name=self.bucket_name, namespace_name=namespace, object_name=self.object_name, put_object_body=self.put_object_body, **kwargs)