def s3_download_file(
bucket: str,
s3_client: boto3.Session.client,
filename: str,
out_dir: Path,
prefix: Optional[str] = "",
) -> None:
"""Downloads file with given prefix from s3 bucket.
:param bucket: The name of the s3 bucket to download file from.
:pram s3_client: The s3 client to download the files.
:param filename: The name of a file in s3 bucket.
:param prefix: The prefix to s3 file structure.
:param out_dir: The output directory to store the the file.
:return:
None. The file is downloaded to out_dir.
:raises:
FileNotFoundError: If `filename` is not in s3 bucket.
"""
paginator = s3_client.get_paginator("list_objects_v2", )
for _page in paginator.paginate(Bucket=bucket, Prefix=prefix):
for content in _page["Contents"]:
_key = content["Key"]
if Path(_key).name == filename:
s3_client.download_file(
bucket, _key,
out_dir.joinpath(Path(_key).name).as_posix())
return
raise FileNotFoundError(f"{filename} not found in s3 {bucket}")
def s3_upload_file(
up_file: Union[str, Path],
s3_client: boto3.Session.client,
bucket: str,
prefix: Optional[str] = "",
) -> None:
"""Uploads file into s3_bucket in a prefix folder.
:param up_file: The full path to a file to be uploaded.
:param s3_client: s3 client to upload the files.
:param bucket: The name of s3 bucket to upload the files into.
:param prefix: The output directory to put file in s3 bucket.
:raises:
ValueError if file fails to upload to s3 bucket.
"""
filename = Path(up_file).name
s3_path = f"{prefix}/{filename}"
try:
s3_client.head_object(Bucket=s3_bucket, Key=s3_path)
logger.info(f"{filename} exists in {bucket}/{prefix}, skipped upload")
except:
try:
s3_client.upload_file(up_file.as_posix(), bucket, Key=s3_path)
except ClientError as e:
raise ValueError(
f"failed to upload {filename} at {bucket}/{prefix}")
def ingest_data(write_client: boto3.Session.client, device: str, r_min: float,
r_max: float, curr_time: str):
# curr_time = get_curr_time_milli()
# Initialize dimensions (metadata)
dimensions = [
{
"Name": "region",
"Value": "us-east-1"
},
{
"Name": "device_group",
"Value": "ah-sms"
},
{
"Name": "device",
"Value": device
},
]
# Initialize measures (data)
temperature = {
"Dimensions": dimensions,
"MeasureName": "temperature",
"MeasureValue": str(round(random.uniform(r_min, r_max))),
"MeasureValueType": "DOUBLE",
"Time": curr_time
}
# Store all measures as a list of records
records = [temperature]
# Write data
try:
result = write_client.write_records(DatabaseName="Iot-Test",
TableName="TestTable",
Records=records,
CommonAttributes={})
except write_client.exceptions.RejectedRecordsException as err:
print(f"Rejected Records: {err}")
except Exception as err:
print(f"Error: {err}")
def s3_upload(
up_file: Union[str, Path],
s3_client: boto3.Session.client,
bucket: str,
prefix: Optional[str] = "",
replace: Optional[bool] = True,
):
"""Uploads file into s3_bucket in a prefix folder.
:param up_file: Full path to file that will be uploaded to s3.
:param s3_client: s3 client to upload the files.
:param bucket: The name of s3 bucket to upload the files into.
:param prefix: The output directory to put file in s3 bucket.
:param replace: Optional parameter to specify if files in s3 bucket
is to be replaced if it exists.
:return:
True if uploaded, else False
:raises:
ValueError if file fails to upload to s3 bucket.
"""
filename = Path(up_file).name
s3_path = f"{prefix}/{filename}"
try:
s3_client.head_object(Bucket=bucket, Key=s3_path)
if replace:
s3_client.upload_file(up_file.as_posix(), bucket, Key=s3_path)
return True
else:
STATUS_LOGGER.info(
f"{filename} exists in {bucket}/{prefix}, skipped upload")
return False
except ClientError:
try:
s3_client.upload_file(up_file.as_posix(), bucket, Key=s3_path)
return True
except ClientError:
raise ValueError(
f"failed to upload {filename} at {bucket}/{prefix}")
def get_current_account_id(sts_client: boto3.Session.client) -> str:
"""Get the current account ID"""
response = sts_client.get_caller_identity()
current_account_id: str = response.get("Account", "")
return current_account_id
def _get_key_id_with_alias(self, name: str,
client: boto3.Session.client) -> str:
"""Given an alias, return the key ID"""
response = client.describe_key(KeyId=name)
key_id = response.get("KeyMetadata").get("KeyId")
return key_id
def get_current_account_id(sts_client: boto3.Session.client) -> str:
response = sts_client.get_caller_identity()
current_account_id = response.get("Account")
return current_account_id