def test_copy(_crc32c):
chunk = b"DEADBEEF"
helper = google_crc32c.Checksum(chunk)
clone = helper.copy()
before = helper._crc
helper.update(b"FACEDACE")
assert clone._crc == before
def access_secret_version(project_id, secret_id, version_id):
"""
Access the payload for the given secret version if one exists. The version
can be a version number as a string (e.g. "5") or an alias (e.g. "latest").
"""
# Import the Secret Manager client library.
from google.cloud import secretmanager
# Create the Secret Manager client.
client = secretmanager.SecretManagerServiceClient()
# Build the resource name of the secret version.
name = f"projects/{project_id}/secrets/{secret_id}/versions/{version_id}"
# Access the secret version.
response = client.access_secret_version(request={"name": name})
# Verify payload checksum.
crc32c = google_crc32c.Checksum()
crc32c.update(response.payload.data)
if response.payload.data_crc32c != int(crc32c.hexdigest(), 16):
print("Data corruption detected.")
return response
# Print the secret payload.
#
# WARNING: Do not print the secret in a production environment - this
# snippet is showing how to access the secret material.
payload = response.payload.data.decode("UTF-8")
print("Plaintext: {}".format(payload))
# [END secretmanager_access_secret_version]
return response
def get_crc32c(initial_data=b''):
"""Returns an instance of Hashlib-like helper for CRC32C operations.
Args:
initial_data (bytes): The CRC32C object will be initialized with the
checksum of the data.
Returns:
The google_crc32c.Checksum instance
if google-crc32c (https://github.com/googleapis/python-crc32c) is
available. If not, returns the predefined.Crc instance from crcmod library.
Usage:
# Get the instance.
crc = get_crc32c()
# Update the instance with data. If your data is available in chunks,
# you can update each chunk so that you don't have to keep everything in
# memory.
for chunk in chunks:
crc.update(data)
# Get the digest.
crc_digest = crc.digest()
"""
if IS_FAST_GOOGLE_CRC32C_AVAILABLE:
crc = google_crc32c.Checksum()
else:
crc = crcmod.predefined.Crc('crc-32c')
if initial_data:
crc.update(initial_data)
return crc
def __init__(self, wrapped_io: BinaryIO, name: str) -> None:
super().__init__()
self._wrapped_io = wrapped_io
self._name = name
self._crc32c = google_crc32c.Checksum()
self._eof = False
def test_update_w_multiple_chunks(_crc32c):
helper = google_crc32c.Checksum()
for index in itertools.islice(range(ISCSI_LENGTH), 0, None, 7):
chunk = ISCSI_SCSI_READ_10_COMMAND_PDU[index:index + 7]
helper.update(bytes(chunk))
assert helper._crc == ISCSI_CRC
def test_consume_stream(_crc32c, chunksize):
helper = google_crc32c.Checksum()
expected = [bytes(chunk) for chunk in iscsi_chunks(chunksize)]
stream = mock.Mock(spec=["read"])
stream.read.side_effect = expected + [b""]
found = list(helper.consume(stream, chunksize))
assert helper._crc == ISCSI_CRC
assert found == expected
for call in stream.read.call_args_list:
assert call == mock.call(chunksize)
def load_secrets():
secret_client = secretmanager.SecretManagerServiceClient()
for key in AUTH_KEYS:
if key not in os.environ:
typer.secho(f"fetching secret {key}")
name = f"projects/cal-itp-data-infra/secrets/{key}/versions/latest"
response = secret_client.access_secret_version(
request={"name": name})
crc32c = google_crc32c.Checksum()
crc32c.update(response.payload.data)
if response.payload.data_crc32c != int(crc32c.hexdigest(), 16):
raise ValueError(
f"Data corruption detected for secret {name}.")
os.environ[key] = response.payload.data.decode("UTF-8").strip()
def _get_crc32c_object():
"""Get crc32c object
Attempt to use the Google-CRC32c package. If it isn't available, try
to use CRCMod. CRCMod might be using a 'slow' varietal. If so, warn...
"""
try:
import google_crc32c
crc_obj = google_crc32c.Checksum()
except ImportError:
try:
import crcmod
crc_obj = crcmod.predefined.Crc("crc-32c")
_is_fast_crcmod()
except ImportError:
raise ImportError("Failed to import either `google-crc32c` or `crcmod`")
return crc_obj
def add_secret_version(project_id, secret_id, payload):
"""
Add a new secret version to the given secret with the provided payload.
"""
# Import the Secret Manager client library.
from google.cloud import secretmanager
# Create the Secret Manager client.
client = secretmanager.SecretManagerServiceClient()
# Build the resource name of the parent secret.
parent = client.secret_path(project_id, secret_id)
# Convert the string payload into a bytes. This step can be omitted if you
# pass in bytes instead of a str for the payload argument.
payload = payload.encode("UTF-8")
# Calculate payload checksum. Passing a checksum in add-version request
# is optional.
crc32c = google_crc32c.Checksum()
crc32c.update(payload)
# Add the secret version.
response = client.add_secret_version(
request={
"parent": parent,
"payload": {
"data": payload,
"data_crc32c": int(crc32c.hexdigest(), 16)
},
})
# Print the new secret version name.
print("Added secret version: {}".format(response.name))
# [END secretmanager_add_secret_version]
return response
def test_digest_nonzero(_crc32c):
helper = google_crc32c.Checksum()
helper._crc = 0x01020304
assert helper.digest() == b"\x01\x02\x03\x04"
def _crc32c(data: bytes) -> str:
# Compute Google's wonky base64 encoded crc32c checksum
return base64.b64encode(
google_crc32c.Checksum(data).digest()).decode("utf-8")
def _crc32c(content):
if isinstance(content, str):
content = content.encode()
val = google_crc32c.Checksum(content)
return b64encode(val.digest()).decode("ascii")
def TestOneInput(data): val1 = google_crc32c.value(data) val2 = google_crc32c.Checksum(data)._crc assert val1 == val2
def test_ctor_defaults(_crc32c):
helper = google_crc32c.Checksum()
assert helper._crc == 0
def __init__(self, data: Optional[bytes] = None):
if data is not None:
self._checksum = google_crc32c.Checksum(data)
else:
self._checksum = google_crc32c.Checksum(b"")
def test_hexdigest_nonzero(_crc32c):
helper = google_crc32c.Checksum()
helper._crc = 0x091A3B2C
assert helper.hexdigest() == b"091a3b2c"
def test_hexdigest_zero(_crc32c):
helper = google_crc32c.Checksum()
assert helper.hexdigest() == b"00" * 4
def test_ctor_explicit(_crc32c):
chunk = b"DEADBEEF"
helper = google_crc32c.Checksum(chunk)
assert helper._crc == google_crc32c.value(chunk)
def test_digest_zero(_crc32c):
helper = google_crc32c.Checksum()
assert helper.digest() == b"\x00" * 4
def test_update(_crc32c):
chunk = b"DEADBEEF"
helper = google_crc32c.Checksum()
helper.update(chunk)
assert helper._crc == google_crc32c.value(chunk)
