class GS(Base): _creds: service_account.Credentials = None _project: str = None _bucket: Bucket = None def __init__(self, bucket: str, creds_path: Optional[str] = None): super().__init__() if creds_path is not None: self._creds = service_account.Credentials.from_service_account_file( creds_path) with open(creds_path, 'rt') as f: self._project = json.loads(f.read())['project_id'] self._bucket = Client(self._project, self._creds).bucket(bucket) else: self._bucket = Client().bucket(bucket) def get(self, path: str) -> bytes: return self._bucket.get_blob(path).download_as_string() def put(self, path: str, content: bytes): self._bucket.blob(path).upload_from_string(content) def exists(self, path: str) -> bool: return self._bucket.get_blob(path) is not None def delete(self, path: str): blobs = self._bucket.list_blobs(prefix=path) for blob in blobs: blob.delete()
def get_encrypted_secret(secret_name, project_id, env): """ Fetch the encrypted secret stored in project_id's secrets bucket. Return the encrypted string. Bucket names are globally unique. The secrets bucket for a project is called "{project_id}-secrets". Inside the bucket are folders corresponding to different environments; currently either dev, staging or prod. Inside each folder are files that are encrypted by GCloud KMS keys that are specific to that secret. """ bucket_name = "{id}-secrets".format(id=project_id) loc = "{env}/{name}".format(env=env, name=secret_name) bucket = Client().get_bucket(bucket_name) try: ret = bucket.blob(loc).download_as_string() except AttributeError: logging.warning( "Secret '{name}' in env '{env}' does not exist! Defaulting to an empty string." .format(env=env, name=secret_name)) return ret
class GSPath: path: PurePath bucket: Bucket blob: Blob @classmethod def from_blob(cls, blob: Blob) -> GSPath: return cls(blob.bucket, blob.name) @classmethod def from_url(cls, url_str: str) -> GSPath: url = urlparse(url_str) if url.scheme != 'gs': raise ValueError('Wrong url scheme') return cls(url.netloc, url.path[1:]) def __init__(self, bucket: Union[str, Bucket], path: Union[PurePath, str]) -> None: self.path = PurePath(str(path)) if isinstance(bucket, str): bucket = Client().bucket(bucket) self.bucket = bucket self.blob = self.bucket.blob(str(self.path)) def __getstate__(self) -> Dict[str, Any]: return {'path': self.path, 'bucket': self.bucket.name} def __setstate__(self, data: Dict[str, Any]) -> None: self.path = data['path'] self.bucket = Client().bucket(data['bucket']) self.blob = self.bucket.blob(str(self.path)) # Otherwise pylint thinks GSPath is undefined # pylint: disable=undefined-variable def __truediv__(self, other: Union[str, PurePath]) -> GSPath: return GSPath(self.bucket, self.path / str(other)) def __repr__(self) -> str: url = f'gs://{self.bucket.name}/{self.path}' return f'GSPath.from_url({url!r})' @property def parent(self) -> GSPath: return GSPath(self.bucket, self.path.parent) def mkdir(self, mode: int = 0, parents: bool = True, exist_ok: bool = True) -> None: # no notion of 'directories' in GS pass def rmtree(self) -> None: for path in self.iterdir(): path.unlink() def exists(self) -> bool: # print(f'{self.blob.name} exists? {self.blob.exists()}') return self.blob.exists() def unlink(self) -> None: self.blob.delete() def iterdir(self) -> Iterable[GSPath]: for blob in self.bucket.list_blobs(prefix=f'{self.path!s}/'): yield GSPath.from_blob(blob) def open(self, mode: str = 'r', encoding: str = 'UTF-8', newline: str = '\n') -> IO[Any]: if 'w' in mode: if 'b' in mode: return WGSFile(self, mode) else: return io.TextIOWrapper(WGSFile(self, mode), encoding=encoding, newline=newline) elif 'r' in mode: if 'b' in mode: return io.BytesIO(self.blob.download_as_string()) else: return io.StringIO( self.blob.download_as_string().decode(encoding=encoding), newline) else: raise RuntimeError(f'Flag {mode} not supported') def public_path(self) -> str: return f'https://storage.googleapis.com/{self.bucket.name}/{self.path}'