class GS(Base):
_creds: service_account.Credentials = None
_project: str = None
_bucket: Bucket = None
def __init__(self, bucket: str, creds_path: Optional[str] = None):
super().__init__()
if creds_path is not None:
self._creds = service_account.Credentials.from_service_account_file(
creds_path)
with open(creds_path, 'rt') as f:
self._project = json.loads(f.read())['project_id']
self._bucket = Client(self._project, self._creds).bucket(bucket)
else:
self._bucket = Client().bucket(bucket)
def get(self, path: str) -> bytes:
return self._bucket.get_blob(path).download_as_string()
def put(self, path: str, content: bytes):
self._bucket.blob(path).upload_from_string(content)
def exists(self, path: str) -> bool:
return self._bucket.get_blob(path) is not None
def delete(self, path: str):
blobs = self._bucket.list_blobs(prefix=path)
for blob in blobs:
blob.delete()
def get_encrypted_secret(secret_name, project_id, env):
"""
Fetch the encrypted secret stored in project_id's secrets bucket. Return the encrypted string.
Bucket names are globally unique. The secrets bucket for a project is called "{project_id}-secrets".
Inside the bucket are folders corresponding to different environments; currently either dev, staging or
prod. Inside each folder are files that are encrypted by GCloud KMS keys that are specific to that
secret.
"""
bucket_name = "{id}-secrets".format(id=project_id)
loc = "{env}/{name}".format(env=env, name=secret_name)
bucket = Client().get_bucket(bucket_name)
try:
ret = bucket.blob(loc).download_as_string()
except AttributeError:
logging.warning(
"Secret '{name}' in env '{env}' does not exist! Defaulting to an empty string."
.format(env=env, name=secret_name))
return ret
class GSPath:
path: PurePath
bucket: Bucket
blob: Blob
@classmethod
def from_blob(cls, blob: Blob) -> GSPath:
return cls(blob.bucket, blob.name)
@classmethod
def from_url(cls, url_str: str) -> GSPath:
url = urlparse(url_str)
if url.scheme != 'gs':
raise ValueError('Wrong url scheme')
return cls(url.netloc, url.path[1:])
def __init__(self, bucket: Union[str, Bucket], path: Union[PurePath,
str]) -> None:
self.path = PurePath(str(path))
if isinstance(bucket, str):
bucket = Client().bucket(bucket)
self.bucket = bucket
self.blob = self.bucket.blob(str(self.path))
def __getstate__(self) -> Dict[str, Any]:
return {'path': self.path, 'bucket': self.bucket.name}
def __setstate__(self, data: Dict[str, Any]) -> None:
self.path = data['path']
self.bucket = Client().bucket(data['bucket'])
self.blob = self.bucket.blob(str(self.path))
# Otherwise pylint thinks GSPath is undefined
# pylint: disable=undefined-variable
def __truediv__(self, other: Union[str, PurePath]) -> GSPath:
return GSPath(self.bucket, self.path / str(other))
def __repr__(self) -> str:
url = f'gs://{self.bucket.name}/{self.path}'
return f'GSPath.from_url({url!r})'
@property
def parent(self) -> GSPath:
return GSPath(self.bucket, self.path.parent)
def mkdir(self,
mode: int = 0,
parents: bool = True,
exist_ok: bool = True) -> None:
# no notion of 'directories' in GS
pass
def rmtree(self) -> None:
for path in self.iterdir():
path.unlink()
def exists(self) -> bool:
# print(f'{self.blob.name} exists? {self.blob.exists()}')
return self.blob.exists()
def unlink(self) -> None:
self.blob.delete()
def iterdir(self) -> Iterable[GSPath]:
for blob in self.bucket.list_blobs(prefix=f'{self.path!s}/'):
yield GSPath.from_blob(blob)
def open(self,
mode: str = 'r',
encoding: str = 'UTF-8',
newline: str = '\n') -> IO[Any]:
if 'w' in mode:
if 'b' in mode:
return WGSFile(self, mode)
else:
return io.TextIOWrapper(WGSFile(self, mode),
encoding=encoding,
newline=newline)
elif 'r' in mode:
if 'b' in mode:
return io.BytesIO(self.blob.download_as_string())
else:
return io.StringIO(
self.blob.download_as_string().decode(encoding=encoding),
newline)
else:
raise RuntimeError(f'Flag {mode} not supported')
def public_path(self) -> str:
return f'https://storage.googleapis.com/{self.bucket.name}/{self.path}'
