def handler(event, context):
query_id = event["QueryId"]
results = paginate(athena,
athena.get_query_results, ["ResultSet.Rows"],
QueryExecutionId=query_id)
rows = [result for result in results]
header_row = rows.pop(0)
path_field_index = next((index
for (index, d) in enumerate(header_row["Data"])
if d["VarCharValue"] == "$path"), None)
paths = [row["Data"][path_field_index]["VarCharValue"] for row in rows]
messages = []
for p in paths:
msg = {
"JobId": event["JobId"],
"Object": p,
"Columns": event["Columns"],
"RoleArn": event.get("RoleArn", None),
"DeleteOldVersions": event.get("DeleteOldVersions", True),
}
messages.append({k: v for k, v in msg.items() if v is not None})
batch_sqs_msgs(queue, messages)
return paths
def get_data_mappers():
results = paginate(ddb_client,
ddb_client.scan,
"Items",
TableName=data_mapper_table_name)
for result in results:
yield deserialize_item(result)
def handler(event, context):
query_id = event["QueryId"]
results = paginate(athena,
athena.get_query_results, ["ResultSet.Rows"],
QueryExecutionId=query_id)
rows = [result for result in results]
header_row = rows.pop(0)
path_field_index = next(
(index for (index, d) in enumerate(header_row["Data"])
if d["VarCharValue"] == "$path"),
None,
)
paths = [row["Data"][path_field_index]["VarCharValue"] for row in rows]
messages = []
for p in paths:
msg = {
"AllFiles": event["AllFiles"],
"JobId": event["JobId"],
"Object": p,
"QueryBucket": event["Bucket"],
"QueryKey": event["Key"],
"RoleArn": event.get("RoleArn", None),
"DeleteOldVersions": event.get("DeleteOldVersions", True),
"Format": event.get("Format"),
}
messages.append({k: v for k, v in msg.items() if v is not None})
btached_msgs = [
messages[i:i + NUM_OF_MESSAGES_IN_BATCH]
for i in range(0, len(messages), NUM_OF_MESSAGES_IN_BATCH)
]
for batch in btached_msgs:
batch_sqs_msgs(queue, batch)
return None
def get_deletion_queue():
results = paginate(ddb_client,
ddb_client.scan,
"Items",
TableName=deletion_queue_table_name)
for result in results:
yield deserialize_item(result)
def delete_old_versions(client, input_bucket, input_key, new_version):
try:
resp = list(paginate(client, client.list_object_versions, ["Versions", "DeleteMarkers"],
Bucket=input_bucket, Prefix=input_key, VersionIdMarker=new_version, KeyMarker=input_key))
versions = [el[0] for el in resp if el[0] is not None]
delete_markers = [el[1] for el in resp if el[1] is not None]
versions.extend(delete_markers)
sorted_versions = sorted(versions, key=lambda x: x["LastModified"])
version_ids = [v["VersionId"] for v in sorted_versions]
errors = []
max_deletions = 1000
for i in range(0, len(version_ids), max_deletions):
resp = client.delete_objects(
Bucket=input_bucket,
Delete={
'Objects': [
{
'Key': input_key,
'VersionId': version_id
} for version_id in version_ids[i:i + max_deletions]
],
'Quiet': True
}
)
errors.extend(resp.get("Errors", []))
if len(errors) > 0:
raise DeleteOldVersionsError(errors=[
"Delete object {} version {} failed: {}".format(e["Key"], e["VersionId"], e["Message"])
for e in errors
])
except ClientError as e:
raise DeleteOldVersionsError(errors=[str(e)])
def get_partitions(db, table_name):
return paginate(
glue_client,
glue_client.get_partitions,
["Partitions"],
DatabaseName=db,
TableName=table_name,
)
def test_it_supports_single_iter_key():
client = MagicMock()
client.get_paginator.return_value = client
client.some_method.__name__ = "some_method"
client.paginate.return_value = iter([{"Test": ["val"]}])
result = paginate(client, client.some_method, "Test")
assert isinstance(result, types.GeneratorType)
assert ["val"] == list(result)
def handler(event, context):
results = paginate(ddb_client, ddb_client.scan, "Items", TableName=event["TableName"])
items = [deserialize_item(result) for result in results]
return {
"Items": items,
"Count": len(items)
}
def delete(event, context):
props = event['ResourceProperties']
repository = props["Repository"]
images = list(paginate(ecr_client, ecr_client.list_images, ["imageIds"], repositoryName=repository))
if images:
ecr_client.batch_delete_image(
imageIds=images, repositoryName=repository)
return None
def test_it_supports_multiple_iter_keys_of_varying_lengths():
client = MagicMock()
client.get_paginator.return_value = client
client.some_method.__name__ = "some_method"
client.paginate.return_value = iter([{
"A": [
"valA1",
"valA2",
],
"B": [
"valB1",
]
}])
result = paginate(client, client.some_method, ["A", "B"])
assert isinstance(result, types.GeneratorType)
assert [("valA1", "valB1"), ("valA2", None)] == list(result)
def test_it_supports_nested_iter_keys():
client = MagicMock()
client.get_paginator.return_value = client
client.some_method.__name__ = "some_method"
client.paginate.return_value = iter([{
"A": {
"B": [
"valAB1",
"valAB2",
],
},
"C": [
"valC1",
]
}])
result = paginate(client, client.some_method, ["A.B", "C"])
assert isinstance(result, types.GeneratorType)
assert [("valAB1", "valC1"), ("valAB2", None)] == list(result)
def revert_last(client, input_bucket, input_key):
try:
resp = list(
paginate(client,
client.list_object_versions,
["Versions", "DeleteMarkers"],
Bucket=input_bucket,
Prefix=input_key))
versions = [el[0] for el in resp if el[0] is not None]
sorted_versions = sorted(versions,
key=lambda x: x["LastModified"],
reverse=True)
delete_markers = [el[1] for el in resp if el[1] is not None]
sorted_deletions = sorted(delete_markers,
key=lambda x: x["LastModified"],
reverse=True)
latest_version = sorted_versions[0]
if not latest_version["IsLatest"]:
raise RevertLastVersionError(errors=[
"Delete object {} version {} failed: {}".format(
input_key, latest_version,
"IsLatest and LastModified don't match")
])
if len(sorted_versions) > 1 and (len(sorted_deletions) == 0
or sorted_deletions[0]["LastModified"]
< sorted_versions[1]["LastModified"]):
try:
client.delete_object(Bucket=input_bucket,
Key=input_key,
VersionId=latest_version["VersionId"])
logger.info("File {}/{} was Reverted Successfully".format(
input_bucket, input_key))
except Exception as e:
raise RevertLastVersionError(errors=[
"Delete object {} version {} failed: {}".format(
input_key, latest_version, str(e))
])
else:
logger.info(
"Skipped Reversion for File: {}/{} since there are no prior versions"
.format(input_bucket, input_key))
except Exception as e:
raise RevertLastVersionError(errors=[str(e)])
