def handler(event, context): query_id = event["QueryId"] results = paginate(athena, athena.get_query_results, ["ResultSet.Rows"], QueryExecutionId=query_id) rows = [result for result in results] header_row = rows.pop(0) path_field_index = next((index for (index, d) in enumerate(header_row["Data"]) if d["VarCharValue"] == "$path"), None) paths = [row["Data"][path_field_index]["VarCharValue"] for row in rows] messages = [] for p in paths: msg = { "JobId": event["JobId"], "Object": p, "Columns": event["Columns"], "RoleArn": event.get("RoleArn", None), "DeleteOldVersions": event.get("DeleteOldVersions", True), } messages.append({k: v for k, v in msg.items() if v is not None}) batch_sqs_msgs(queue, messages) return paths
def get_data_mappers(): results = paginate(ddb_client, ddb_client.scan, "Items", TableName=data_mapper_table_name) for result in results: yield deserialize_item(result)
def handler(event, context): query_id = event["QueryId"] results = paginate(athena, athena.get_query_results, ["ResultSet.Rows"], QueryExecutionId=query_id) rows = [result for result in results] header_row = rows.pop(0) path_field_index = next( (index for (index, d) in enumerate(header_row["Data"]) if d["VarCharValue"] == "$path"), None, ) paths = [row["Data"][path_field_index]["VarCharValue"] for row in rows] messages = [] for p in paths: msg = { "AllFiles": event["AllFiles"], "JobId": event["JobId"], "Object": p, "QueryBucket": event["Bucket"], "QueryKey": event["Key"], "RoleArn": event.get("RoleArn", None), "DeleteOldVersions": event.get("DeleteOldVersions", True), "Format": event.get("Format"), } messages.append({k: v for k, v in msg.items() if v is not None}) btached_msgs = [ messages[i:i + NUM_OF_MESSAGES_IN_BATCH] for i in range(0, len(messages), NUM_OF_MESSAGES_IN_BATCH) ] for batch in btached_msgs: batch_sqs_msgs(queue, batch) return None
def get_deletion_queue(): results = paginate(ddb_client, ddb_client.scan, "Items", TableName=deletion_queue_table_name) for result in results: yield deserialize_item(result)
def delete_old_versions(client, input_bucket, input_key, new_version): try: resp = list(paginate(client, client.list_object_versions, ["Versions", "DeleteMarkers"], Bucket=input_bucket, Prefix=input_key, VersionIdMarker=new_version, KeyMarker=input_key)) versions = [el[0] for el in resp if el[0] is not None] delete_markers = [el[1] for el in resp if el[1] is not None] versions.extend(delete_markers) sorted_versions = sorted(versions, key=lambda x: x["LastModified"]) version_ids = [v["VersionId"] for v in sorted_versions] errors = [] max_deletions = 1000 for i in range(0, len(version_ids), max_deletions): resp = client.delete_objects( Bucket=input_bucket, Delete={ 'Objects': [ { 'Key': input_key, 'VersionId': version_id } for version_id in version_ids[i:i + max_deletions] ], 'Quiet': True } ) errors.extend(resp.get("Errors", [])) if len(errors) > 0: raise DeleteOldVersionsError(errors=[ "Delete object {} version {} failed: {}".format(e["Key"], e["VersionId"], e["Message"]) for e in errors ]) except ClientError as e: raise DeleteOldVersionsError(errors=[str(e)])
def get_partitions(db, table_name): return paginate( glue_client, glue_client.get_partitions, ["Partitions"], DatabaseName=db, TableName=table_name, )
def test_it_supports_single_iter_key(): client = MagicMock() client.get_paginator.return_value = client client.some_method.__name__ = "some_method" client.paginate.return_value = iter([{"Test": ["val"]}]) result = paginate(client, client.some_method, "Test") assert isinstance(result, types.GeneratorType) assert ["val"] == list(result)
def handler(event, context): results = paginate(ddb_client, ddb_client.scan, "Items", TableName=event["TableName"]) items = [deserialize_item(result) for result in results] return { "Items": items, "Count": len(items) }
def delete(event, context): props = event['ResourceProperties'] repository = props["Repository"] images = list(paginate(ecr_client, ecr_client.list_images, ["imageIds"], repositoryName=repository)) if images: ecr_client.batch_delete_image( imageIds=images, repositoryName=repository) return None
def test_it_supports_multiple_iter_keys_of_varying_lengths(): client = MagicMock() client.get_paginator.return_value = client client.some_method.__name__ = "some_method" client.paginate.return_value = iter([{ "A": [ "valA1", "valA2", ], "B": [ "valB1", ] }]) result = paginate(client, client.some_method, ["A", "B"]) assert isinstance(result, types.GeneratorType) assert [("valA1", "valB1"), ("valA2", None)] == list(result)
def test_it_supports_nested_iter_keys(): client = MagicMock() client.get_paginator.return_value = client client.some_method.__name__ = "some_method" client.paginate.return_value = iter([{ "A": { "B": [ "valAB1", "valAB2", ], }, "C": [ "valC1", ] }]) result = paginate(client, client.some_method, ["A.B", "C"]) assert isinstance(result, types.GeneratorType) assert [("valAB1", "valC1"), ("valAB2", None)] == list(result)
def revert_last(client, input_bucket, input_key): try: resp = list( paginate(client, client.list_object_versions, ["Versions", "DeleteMarkers"], Bucket=input_bucket, Prefix=input_key)) versions = [el[0] for el in resp if el[0] is not None] sorted_versions = sorted(versions, key=lambda x: x["LastModified"], reverse=True) delete_markers = [el[1] for el in resp if el[1] is not None] sorted_deletions = sorted(delete_markers, key=lambda x: x["LastModified"], reverse=True) latest_version = sorted_versions[0] if not latest_version["IsLatest"]: raise RevertLastVersionError(errors=[ "Delete object {} version {} failed: {}".format( input_key, latest_version, "IsLatest and LastModified don't match") ]) if len(sorted_versions) > 1 and (len(sorted_deletions) == 0 or sorted_deletions[0]["LastModified"] < sorted_versions[1]["LastModified"]): try: client.delete_object(Bucket=input_bucket, Key=input_key, VersionId=latest_version["VersionId"]) logger.info("File {}/{} was Reverted Successfully".format( input_bucket, input_key)) except Exception as e: raise RevertLastVersionError(errors=[ "Delete object {} version {} failed: {}".format( input_key, latest_version, str(e)) ]) else: logger.info( "Skipped Reversion for File: {}/{} since there are no prior versions" .format(input_bucket, input_key)) except Exception as e: raise RevertLastVersionError(errors=[str(e)])