def read(self, size: Optional[int] = None) -> bytes:
if size is not None:
raise Exception("size is not yet supported")
full_read = bytearray()
for part in self._blob.get_parts():
if part.get("bytesRef"):
bytes_ref_str: str = part["bytesRef"]
bytes_ref_blob: Optional[Blob] = self._fetcher.fetch_blob(
Ref.from_ref_str(bytes_ref_str))
if not bytes_ref_blob:
raise Exception(f"blob not found {bytes_ref_str}")
full_read += BytesReader(
blob=BytesSchema(schema=Schema.from_blob(bytes_ref_blob)),
fetcher=self._fetcher,
).read()
elif part.get("blobRef"):
blob_ref_str: str = part["blobRef"]
blob_ref_blob: Optional[Blob] = self._fetcher.fetch_blob(
Ref.from_ref_str(blob_ref_str))
if not blob_ref_blob:
raise Exception(f"blob not found {blob_ref_str}")
full_read += blob_ref_blob.get_bytes()
return bytes(full_read)
def test_is_utf8() -> None:
blob: Blob = Blob(
ref=Ref.from_ref_str(
"sha224-d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f"),
readall=lambda: "test".encode("utf-8"),
)
assert blob.is_utf8()
blob = Blob(
ref=Ref.from_ref_str(
"sha224-d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f"),
readall=lambda: b"\xdc",
)
assert not blob.is_utf8()
def get(blobserver: S3, *, ref: str, contents: bool) -> None:
ref_: Ref = Ref.from_ref_str(ref)
blob = blobserver.fetch_blob(ref_)
if contents:
schema: Schema = Schema.from_blob(blob)
schema_to_read: Union[BytesSchema, FileSchema]
if schema.get_type() == CamliType.FILE:
schema_to_read = schema.as_file()
elif schema.get_type() == CamliType.BYTES:
schema_to_read = schema.as_bytes()
else:
click.echo(f"Don't know how to read a {schema.get_type()} schema")
return
bytes_reader: BytesReader = BytesReader(
blob=schema_to_read,
fetcher=blobserver,
)
full_contents: bytes = bytes_reader.read()
click.echo(full_contents)
return
if blob.is_utf8():
click.echo(blob.get_bytes().decode("utf-8"))
else:
click.echo(base64.b64encode(blob.get_bytes()))
def test_from_str_sha224() -> None:
ref_sha224: Ref = Ref.from_ref_str(
"sha224-d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f")
assert ref_sha224.get_digest_name() == "sha224"
assert (ref_sha224.get_bytes().hex() ==
"d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f")
assert (ref_sha224.to_str() ==
"sha224-d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f")
def test_is_valid() -> None:
blob_content: bytes = "test".encode("utf-8")
hexdigest: str = hashlib.sha224(blob_content).hexdigest()
ref: Ref = Ref.from_ref_str(f"sha224-{hexdigest}")
blob: Blob = Blob(
ref=ref,
readall=lambda: blob_content,
)
assert blob.is_valid()
def fetch_blob(self, ref: Ref) -> Blob:
resp: S3GetObjectResponse = self.client.get_object(
Bucket=self.bucket,
Key=self.dirprefix + ref.to_str(),
)
blob: Blob = Blob(
ref=ref,
readall=lambda: resp["Body"].read(),
)
return blob
def verify_json_signature(
*,
signed_json_object: bytes,
fetcher: Fetcher,
gpg_signature_verifier: GPGSignatureVerifier,
) -> bool:
# Load the JSON object
json_obj: Any = json.loads(signed_json_object)
if not isinstance(json_obj, dict):
raise Exception(f"JSON must be an object, got {type(json_obj)}")
# Extract the signature
camli_signature: Any = json_obj.get("camliSig", None)
if not isinstance(camli_signature, str):
raise Exception(
f"camliSig must be a string, got {type(camli_signature)}"
)
camli_signature_armored: str = CamliSig.to_armored_gpg_signature(
camli_signature
)
# Find the camliSigner's public key
camli_signer: Any = json_obj.get("camliSigner", None)
if not isinstance(camli_signer, str):
raise Exception(
f"camliSigner must be a string, got {type(camli_signer)}"
)
camli_signer_ref: Ref = Ref.from_ref_str(camli_signer)
camli_signer_public_key_blob: Optional[Blob] = fetcher.fetch_blob(
camli_signer_ref
)
if not camli_signer_public_key_blob:
raise Exception(f"Could not fetch public key for signer {camli_signer}")
camli_signer_public_key: str = (
camli_signer_public_key_blob.get_bytes().decode()
)
# Isolate the signed content
signed_bytes_end_index = signed_json_object.rindex(_SIGNATURE_DELIMITER)
signed_bytes: bytes = signed_json_object[:signed_bytes_end_index]
# Verify the signature
result = gpg_signature_verifier.verify_signature(
data=signed_bytes,
armored_detached_signature=camli_signature_armored,
armored_public_key=camli_signer_public_key,
)
return result
def enumerate_blobs(self, after: Optional[Ref] = None) -> Iterator[Ref]:
while True:
after_str: str = self.dirprefix + after.to_str() if after else ""
resp: S3ListObjectsV2Response = self.client.list_objects_v2(
Bucket=self.bucket,
Prefix=self.dirprefix,
StartAfter=after_str,
)
if not resp.get("Contents"):
break
for s3_object in resp["Contents"]:
ref_str: str = s3_object["Key"].split("/")[-1]
ref = Ref.from_ref_str(ref_str)
yield ref
after = ref
def sign_json(
*,
unsigned_json_object: _SignableJSON,
gpg_signer: GPGSigner,
gpg_key_inspector: GPGKeyInspector,
fetcher: Fetcher,
) -> bytes:
# Prepare the JSON for signing
json_bytes: bytes = json.dumps(unsigned_json_object, indent=4).encode()
json_bytes = json_bytes.rstrip()
if not json_bytes.endswith(b"}"):
raise Exception("The json object should end with '}'")
json_bytes = json_bytes.removesuffix(b"}")
# Find the GPG key fingerprint corresponding to the camliSigner
camli_signer: str = unsigned_json_object["camliSigner"]
camli_signer_ref: Ref = Ref.from_ref_str(camli_signer)
camli_signer_public_key_blob: Optional[Blob] = fetcher.fetch_blob(
camli_signer_ref
)
if not camli_signer_public_key_blob:
raise Exception(f"Could not fetch public key for signer {camli_signer}")
camli_signer_key_fingerprint: str = gpg_key_inspector.get_key_fingerprint(
armored_key=camli_signer_public_key_blob.get_bytes().decode(),
)
# Sign
armored_signature: str = gpg_signer.sign_detached_armored(
fingerprint=camli_signer_key_fingerprint, data=json_bytes
)
camli_signature: str = CamliSig.from_armored_gpg_signature(
armored_signature
)
signed_json: bytes = (
json_bytes + _SIGNATURE_DELIMITER + camli_signature.encode() + b'"}\n'
)
return signed_json
def test_ref_equal() -> None:
assert Ref.from_ref_str(
"sha224-d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f"
) == Ref.from_ref_str(
"sha224-d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f")
def fetch_blob(self, ref: Ref) -> Optional[Blob]:
return self.blobs.get(ref.to_str())