def put_object(self, bucket: str, pvc: str, file_location: str, object_key: str) -> str:
"""Start a job to transfer an object from a PVC to the named bucket.
This will transfer a file from the the PVC to the S3 bucket and maintain any directory
structure of the file relative to the root of the PVC.
:param bucket: The name of the bucket to which the file will be copied.
:param pvc: The name of the Persistent Volume Claim where the file will be copied from.
:param file_location: The path and name of the source file to copy.
:param object_key: The value of the object's key in the bucket.
:return: The name of the job created.
"""
if self.verify_certificates:
verify_flag = ""
else:
verify_flag = "--insecure"
operation = "put-object"
command = f"mc cp {verify_flag} {self.data_volume_path}/{file_location} {self.s3_alias}/{bucket}/{object_key}"
job_spec = self.job_spec
job_spec.template = self._get_pod_template_spec(container_command=command, pvc=pvc, operation=operation)
job_metadata = V1ObjectMeta(generate_name="s3mover-{}-".format(operation),
namespace=self.namespace,
labels=_get_labels(operation=operation))
job = self.create_job(job_metadata=job_metadata, job_spec=job_spec)
return job.metadata.name
def put_bucket(self, bucket: str, pvc: str, pvc_dir: str = None) -> str:
"""Start a job to transfer all files from a PVC to the named bucket.
This will transfer all files recursively from the PVC to the S3 bucket and maintain any directory structure
of the files in the bucket.
:param bucket: The name of the bucket to which the files will be copied.
:param pvc: The name of the Persistent Volume Claim where files will be copied from.
:param pvc_dir: An optional path and directory name to specify the directory to use as the
base for uploading objects to the S3 bucket. If this is not specified then the root of
the PVC is used as the base directory.
:return: The name of the job created.
"""
if self.verify_certificates:
verify_flag = ""
else:
verify_flag = "--insecure"
if pvc_dir:
sub_dir = pvc_dir
else:
sub_dir = ""
operation = "put-bucket"
# If we don't change directories the cp will copy files in to a 'data' directory within the bucket
command = f"cd {self.data_volume_path}/{sub_dir};mc cp {verify_flag} -r * {self.s3_alias}/{bucket}"
job_spec = self.job_spec
job_spec.template = self._get_pod_template_spec(container_command=command, pvc=pvc, operation=operation)
job_metadata = V1ObjectMeta(generate_name="s3mover-{}-".format(operation),
namespace=self.namespace,
labels=_get_labels(operation=operation))
job = self.create_job(job_metadata=job_metadata, job_spec=job_spec)
return job.metadata.name
def get_object(self, bucket: str, pvc: str, object_key: str, file_location: str = None) -> str:
"""Start a job to transfer an object from a bucket to a PVC.
This will transfer the specified object from the S3 bucket to the PVC.
:param bucket: The name of the bucket where the object is located.
:param pvc: The name of the Persistent Volume Claim where the object will be copied to.
:param object_key: The value of the object key to copy from the bucket to the PVC.
:param file_location: The location within the PVC to save the file, including the file name.
If no location is specified the object will be copied to the PVC relative to the root of the PVC
with any pathing retained from the object's key name. The default is None.
:return: The name of the job created to transfer data.
"""
if self.verify_certificates:
verify_flag = ""
else:
verify_flag = "--insecure"
operation = "get-object"
if not file_location:
file_location = object_key
command = f"mc cp {verify_flag} {self.s3_alias}/{bucket}/{object_key} {self.data_volume_path}/{file_location}"
job_spec = self.job_spec
job_spec.template = self._get_pod_template_spec(container_command=command, pvc=pvc, operation=operation)
job_metadata = V1ObjectMeta(generate_name="s3mover-{}-".format(operation),
namespace=self.namespace,
labels=_get_labels(operation=operation))
job = self.create_job(job_metadata=job_metadata, job_spec=job_spec)
return job.metadata.name
def get_bucket(self, bucket: str, pvc: str, pvc_dir: str = None) -> str:
"""Start a job to transfer the contents of a bucket to a PVC.
This will transfer the entire contents of the bucket to the PVC maintaining the relative
directory structure of the files as they exist in the bucket.
:param bucket: The name of the bucket that will be the source of the data transfer.
:param pvc: The name of the Persistent Volume Claim that will be the destination of the
data transfer.
:param pvc_dir: An optional directory path to use as the base directory within the PVC
for the destination of the files to be transferred. If no directory is specified here
the root of the PVC is the base path used.
:return: The name of the job created to transfer data.
"""
if self.verify_certificates:
verify_flag = ""
else:
verify_flag = "--insecure"
if pvc_dir:
sub_dir = pvc_dir
else:
sub_dir = ""
operation = "get-bucket"
command = f"mc cp {verify_flag} -r {self.s3_alias}/{bucket}/ {self.data_volume_path}/{sub_dir}"
job_spec = self.job_spec
job_spec.template = self._get_pod_template_spec(container_command=command, pvc=pvc, operation=operation)
job_metadata = V1ObjectMeta(generate_name="s3mover-{}-".format(operation),
namespace=self.namespace,
labels=_get_labels(operation=operation))
job = self.create_job(job_metadata=job_metadata, job_spec=job_spec)
return job.metadata.name
def _get_pod_template_spec(self, container_command: str, pvc: str,
operation: str,
metadata: V1ObjectMeta = None) -> V1PodTemplateSpec:
"""Get a K8s PodTemplateSpec to use with the given command.
:param container_command: The command to use in the container. This is passed on.
:param pvc: The name of the Kubernetes persistent volume claim to use in the data movement.
:param operation: The name of the operation being used. This is used in the creation of the
labels for the default pod metadata. If metadata is explicitly provided that will be used
and won't contain the 'created-by-operation' label in the metadata.
:param metadata: Optional metadata for the PodTemplateSpec.
:return: The pod template spec definition for the provided command.
"""
if metadata is None:
metadata = V1ObjectMeta(
labels=_get_labels(operation=operation)
)
return V1PodTemplateSpec(
metadata=metadata,
spec=self._get_pod_spec(container_command=container_command, pvc=pvc)
)
def create(self):
"""Create the secret with the provided data."""
labels = _get_labels(operation="s3configsecret-create")
create_k8s_opaque_secret(name=self.name, data=self.secret_data,
namespace=self.namespace, labels=labels,
print_output=self.print_output)