def read(
bucket: s3.Bucket,
name: str,
) -> bytes:
f = io.BytesIO()
bucket.download_fileobj(Key=name, Fileobj=f)
return f.getvalue()
def download_files(
bucket: Bucket,
patient_hash: str,
export_date: str,
) -> bool:
"""
GET all files associated with the known record.
NOTE: S3 folder association is symbolic, so a need to pull down data through a nested loop.
"""
folder_path = Path(config.storage_vol) / f"{patient_hash}"
# 'raw' and 'files' are 2nd level top folders
for prefix in ["raw", "files"]:
sub_folder = folder_path / prefix
sub_folder.mkdir(parents=True, exist_ok=True)
# filter to limit returned results to just this patient
for obj in bucket.objects.filter(
Prefix=f"{export_date}/{prefix}/{patient_hash}"):
file_name = obj.key.rsplit("/", 1)[1]
bucket.download_file(obj.key,
str(folder_path / prefix / file_name))
# added method to dmpy service
dmpy.zip_folder_and_rm_local(folder_path)
return True
def _annotate_bucket(self, bucket: Bucket) -> AnnotatedBucket:
"""Annotate bucket objects with 'team', 'uri' and 'public' properties"""
bucket.team = self.team.pk
bucket.uri = urljoin(settings.CEPH_HOST, bucket.name)
try:
policy = bucket.Policy().policy
if policy == json.dumps(get_public_bucket_policy(bucket.name)):
bucket.is_public = True
except self.s3_client.exceptions.ClientError as exc:
if exc.response["Error"]["Code"] == "NoSuchBucketPolicy":
bucket.is_public = False
else:
raise
return bucket
def upload_image_to_s3(
bucket: Bucket,
file_name: str,
content_type: Optional[str],
user_profile: UserProfile,
contents: bytes,
) -> None:
key = bucket.Object(file_name)
metadata = {
"user_profile_id": str(user_profile.id),
"realm_id": str(user_profile.realm_id),
}
content_disposition = ""
if content_type is None:
content_type = ""
if content_type not in INLINE_MIME_TYPES:
content_disposition = "attachment"
key.put(
Body=contents,
Metadata=metadata,
ContentType=content_type,
ContentDisposition=content_disposition,
)
def put(
bucket: s3.Bucket,
source: pathlib.Path | bytes,
target: pathlib.Path, # directory
*,
name: str = "",
cache: bool = False,
content_type: str = "",
) -> s3.Object:
ctx: ContextManager[Any]
if isinstance(source, pathlib.Path):
ctx = open(source, "rb")
name = name or source.name
elif not name:
raise ValueError(f"Name not given for target {target}")
else:
ctx = contextlib.nullcontext(source)
if not content_type:
ct, _ = mimetypes.guess_type(name)
if ct is not None and "/" in ct:
content_type = ct
print("put", name, bucket, target)
with ctx as body:
result = bucket.put_object(
Key=str(target / name),
Body=body,
CacheControl=CACHE if cache else NO_CACHE,
ContentType=content_type,
ACL="public-read",
)
print(result)
return result
def delete_file_from_s3(self, path_id: str, bucket: Bucket) -> bool:
key = bucket.Object(path_id)
try:
key.load()
except botocore.exceptions.ClientError:
file_name = path_id.split("/")[-1]
logging.warning(
"%s does not exist. Its entry in the database will be removed.", file_name
)
return False
key.delete()
return True
def upload_png_from_dir(bucket: Bucket, png_files_dir: Path) -> None:
for p in sorted(png_files_dir.glob('*.png')):
with p.open(mode='rb') as f:
bucket.put_object(Key=f'custom_label/{p.parent}/{p.name}',
Body=f.read())