def rotate_encryption_key(bucket_name, blob_name, base64_encryption_key,
base64_new_encryption_key):
"""Performs a key rotation by re-writing an encrypted blob with a new
encryption key."""
storage_client = storage.Client()
bucket = storage_client.get_bucket(bucket_name)
current_encryption_key = base64.b64decode(base64_encryption_key)
new_encryption_key = base64.b64decode(base64_new_encryption_key)
# Both source_blob and destination_blob refer to the same storage object,
# but destination_blob has the new encryption key.
source_blob = Blob(
blob_name, bucket, encryption_key=current_encryption_key)
destination_blob = Blob(
blob_name, bucket, encryption_key=new_encryption_key)
token = None
while True:
token, bytes_rewritten, total_bytes = destination_blob.rewrite(
source_blob, token=token)
if token is None:
break
print('Key rotation complete for Blob {}'.format(blob_name))
def rotate_encryption_key(bucket_name, blob_name, base64_encryption_key,
base64_new_encryption_key):
"""Performs a key rotation by re-writing an encrypted blob with a new
encryption key."""
storage_client = storage.Client()
bucket = storage_client.get_bucket(bucket_name)
current_encryption_key = base64.b64decode(base64_encryption_key)
new_encryption_key = base64.b64decode(base64_new_encryption_key)
# Both source_blob and destination_blob refer to the same storage object,
# but destination_blob has the new encryption key.
source_blob = Blob(blob_name,
bucket,
encryption_key=current_encryption_key)
destination_blob = Blob(blob_name,
bucket,
encryption_key=new_encryption_key)
token = None
while True:
token, bytes_rewritten, total_bytes = destination_blob.rewrite(
source_blob, token=token)
if token is None:
break
print('Key rotation complete for Blob {}'.format(blob_name))
def main():
# Parse command-line arguments
logging.basicConfig(level='DEBUG')
logging.info("Parsing command-line arguments.")
args = parse_args(sys.argv[1:])
logging.info("Command-line arguments: {}.".format(args))
project = args.gcp_project
json_file = args.json_file
target_bucket_name = args.target_bucket
meta_outfile = args.meta_outfile
delete_source = args.delete_source
# Load data from json file
logging.info("Loading blobs to transfer from {}.".format(json_file))
with open(json_file, 'r') as file_handle:
data = json.load(file_handle)
logging.info("Found {} blobs to transfer".format(len(data)))
logging.info("Creating storage client.")
client = storage.Client(project=project)
target_bucket = client.get_bucket(target_bucket_name)
for entry in data:
source_bucket_name = entry['node']['bucket']
source_path = entry['node']['path']
db_id = entry['id']
logging.info("Beginning transfer of {} from {} to {}".format(
source_path, source_bucket_name, target_bucket_name))
source_bucket = client.get_bucket(source_bucket_name)
# Trim "data/bina-deliverables" prefix for wgs-2000 data
# data/bina-deliverables/401593083/069303ad-4a99-4554-9da0-cfb324c9060a/Recalibration/alignments.bam
if source_bucket.name == "gbsc-gcp-project-mvp-phase-2-data":
elements = source_path.split('/')
target_path = '/'.join(elements[2:])
logging.info("Trimming first 2 elements of directory path.")
else:
target_path = source_path
# Get source blob
logging.info("Getting soure blob from {}".format(source_path))
source_blob = source_bucket.get_blob(source_path)
# Get source metadata
source_metadata = get_blob_metadata(source_blob)
logging.info("Source metadata: {}".format(source_metadata))
# Get target blob
logging.info("Rewriting source blob to gs://{}/{}.".format(
target_bucket, target_path))
target_blob = Blob(target_path, target_bucket)
token = None
while True:
result = target_blob.rewrite(source_blob, token=token)
token = result[0]
if token:
logging.info(
"Rewrite status: {} of {} total bytes written.".format(
result[1], result[2]))
else:
logging.info(
"Rewrite complete. {} of {} total bytes written. ".format(
result[1], result[2]))
break
# Get target blob metadata
logging.info("Getting target metadata.")
target_metadata = get_blob_metadata(target_blob)
# Add metadata changes to a dictionary that mirrors input
output_metadata = {'id': db_id, 'node': target_metadata}
# Compare source & target metadatas
diffs = list(dictdiffer.diff(source_metadata, target_metadata))
for diff in diffs:
logging.info("Changed metadata: {}".format(diff))
for diff in diffs:
meta_property = diff[1]
# Check that size, md5, crc23c, content-type are unchanged
if meta_property in ['md5_hash', 'crc23c', 'content-type']:
logging.error('Content changed: {}'.format(diff))
sys.exit()
# Delete source blob
#if delete_source:
# source_bucket.delete_blob(source_path)
with open(meta_outfile, 'a') as file_handle:
file_handle.write(json.dumps(target_metadata))
