def read(self, size: Optional[int] = None) -> bytes: if size is not None: raise Exception("size is not yet supported") full_read = bytearray() for part in self._blob.get_parts(): if part.get("bytesRef"): bytes_ref_str: str = part["bytesRef"] bytes_ref_blob: Optional[Blob] = self._fetcher.fetch_blob( Ref.from_ref_str(bytes_ref_str)) if not bytes_ref_blob: raise Exception(f"blob not found {bytes_ref_str}") full_read += BytesReader( blob=BytesSchema(schema=Schema.from_blob(bytes_ref_blob)), fetcher=self._fetcher, ).read() elif part.get("blobRef"): blob_ref_str: str = part["blobRef"] blob_ref_blob: Optional[Blob] = self._fetcher.fetch_blob( Ref.from_ref_str(blob_ref_str)) if not blob_ref_blob: raise Exception(f"blob not found {blob_ref_str}") full_read += blob_ref_blob.get_bytes() return bytes(full_read)
def test_is_utf8() -> None: blob: Blob = Blob( ref=Ref.from_ref_str( "sha224-d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f"), readall=lambda: "test".encode("utf-8"), ) assert blob.is_utf8() blob = Blob( ref=Ref.from_ref_str( "sha224-d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f"), readall=lambda: b"\xdc", ) assert not blob.is_utf8()
def get(blobserver: S3, *, ref: str, contents: bool) -> None: ref_: Ref = Ref.from_ref_str(ref) blob = blobserver.fetch_blob(ref_) if contents: schema: Schema = Schema.from_blob(blob) schema_to_read: Union[BytesSchema, FileSchema] if schema.get_type() == CamliType.FILE: schema_to_read = schema.as_file() elif schema.get_type() == CamliType.BYTES: schema_to_read = schema.as_bytes() else: click.echo(f"Don't know how to read a {schema.get_type()} schema") return bytes_reader: BytesReader = BytesReader( blob=schema_to_read, fetcher=blobserver, ) full_contents: bytes = bytes_reader.read() click.echo(full_contents) return if blob.is_utf8(): click.echo(blob.get_bytes().decode("utf-8")) else: click.echo(base64.b64encode(blob.get_bytes()))
def test_from_str_sha224() -> None: ref_sha224: Ref = Ref.from_ref_str( "sha224-d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f") assert ref_sha224.get_digest_name() == "sha224" assert (ref_sha224.get_bytes().hex() == "d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f") assert (ref_sha224.to_str() == "sha224-d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f")
def test_is_valid() -> None: blob_content: bytes = "test".encode("utf-8") hexdigest: str = hashlib.sha224(blob_content).hexdigest() ref: Ref = Ref.from_ref_str(f"sha224-{hexdigest}") blob: Blob = Blob( ref=ref, readall=lambda: blob_content, ) assert blob.is_valid()
def fetch_blob(self, ref: Ref) -> Blob: resp: S3GetObjectResponse = self.client.get_object( Bucket=self.bucket, Key=self.dirprefix + ref.to_str(), ) blob: Blob = Blob( ref=ref, readall=lambda: resp["Body"].read(), ) return blob
def verify_json_signature( *, signed_json_object: bytes, fetcher: Fetcher, gpg_signature_verifier: GPGSignatureVerifier, ) -> bool: # Load the JSON object json_obj: Any = json.loads(signed_json_object) if not isinstance(json_obj, dict): raise Exception(f"JSON must be an object, got {type(json_obj)}") # Extract the signature camli_signature: Any = json_obj.get("camliSig", None) if not isinstance(camli_signature, str): raise Exception( f"camliSig must be a string, got {type(camli_signature)}" ) camli_signature_armored: str = CamliSig.to_armored_gpg_signature( camli_signature ) # Find the camliSigner's public key camli_signer: Any = json_obj.get("camliSigner", None) if not isinstance(camli_signer, str): raise Exception( f"camliSigner must be a string, got {type(camli_signer)}" ) camli_signer_ref: Ref = Ref.from_ref_str(camli_signer) camli_signer_public_key_blob: Optional[Blob] = fetcher.fetch_blob( camli_signer_ref ) if not camli_signer_public_key_blob: raise Exception(f"Could not fetch public key for signer {camli_signer}") camli_signer_public_key: str = ( camli_signer_public_key_blob.get_bytes().decode() ) # Isolate the signed content signed_bytes_end_index = signed_json_object.rindex(_SIGNATURE_DELIMITER) signed_bytes: bytes = signed_json_object[:signed_bytes_end_index] # Verify the signature result = gpg_signature_verifier.verify_signature( data=signed_bytes, armored_detached_signature=camli_signature_armored, armored_public_key=camli_signer_public_key, ) return result
def enumerate_blobs(self, after: Optional[Ref] = None) -> Iterator[Ref]: while True: after_str: str = self.dirprefix + after.to_str() if after else "" resp: S3ListObjectsV2Response = self.client.list_objects_v2( Bucket=self.bucket, Prefix=self.dirprefix, StartAfter=after_str, ) if not resp.get("Contents"): break for s3_object in resp["Contents"]: ref_str: str = s3_object["Key"].split("/")[-1] ref = Ref.from_ref_str(ref_str) yield ref after = ref
def sign_json( *, unsigned_json_object: _SignableJSON, gpg_signer: GPGSigner, gpg_key_inspector: GPGKeyInspector, fetcher: Fetcher, ) -> bytes: # Prepare the JSON for signing json_bytes: bytes = json.dumps(unsigned_json_object, indent=4).encode() json_bytes = json_bytes.rstrip() if not json_bytes.endswith(b"}"): raise Exception("The json object should end with '}'") json_bytes = json_bytes.removesuffix(b"}") # Find the GPG key fingerprint corresponding to the camliSigner camli_signer: str = unsigned_json_object["camliSigner"] camli_signer_ref: Ref = Ref.from_ref_str(camli_signer) camli_signer_public_key_blob: Optional[Blob] = fetcher.fetch_blob( camli_signer_ref ) if not camli_signer_public_key_blob: raise Exception(f"Could not fetch public key for signer {camli_signer}") camli_signer_key_fingerprint: str = gpg_key_inspector.get_key_fingerprint( armored_key=camli_signer_public_key_blob.get_bytes().decode(), ) # Sign armored_signature: str = gpg_signer.sign_detached_armored( fingerprint=camli_signer_key_fingerprint, data=json_bytes ) camli_signature: str = CamliSig.from_armored_gpg_signature( armored_signature ) signed_json: bytes = ( json_bytes + _SIGNATURE_DELIMITER + camli_signature.encode() + b'"}\n' ) return signed_json
def test_ref_equal() -> None: assert Ref.from_ref_str( "sha224-d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f" ) == Ref.from_ref_str( "sha224-d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f")
def fetch_blob(self, ref: Ref) -> Optional[Blob]: return self.blobs.get(ref.to_str())