async def ensure_gcp_resources(self):
"""
Ensure all the required GCP resources exist for this user
"""
with tempfile.NamedTemporaryFile() as f:
f.write(self.gcp_service_key.encode())
f.flush()
storage_client = storage.Client.from_service_account_json(f.name)
credentials = service_account.Credentials.from_service_account_file(
f.name,
scopes=['https://www.googleapis.com/auth/cloud-platform'])
service = googleapiclient.discovery.build('iam',
'v1',
credentials=credentials)
# Create bucket if it doesn't exist
# FIXME: Don't use a private method?
bucket_name = self._expand_all(self.storage_bucket_template)
bucket = await self.ensure_bucket(storage_client, bucket_name)
self.environment['SPARK_GCS_BUCKET'] = bucket_name
# This is how service account emails are formatted
# FIXME: Clip this to 30char
sa_name = self._expand_all(self.service_account_template)
sa_email = await self.ensure_gcp_serviceaccount(service, sa_name)
# Grant SA access to bucket if it isn't present
policy = await self.run_in_gcp_executor(bucket.get_iam_policy)
role = 'roles/storage.objectAdmin'
if Policy.service_account(sa_email) not in policy.get(role, set()):
policy[role].add(Policy.service_account(sa_email))
await self.run_in_gcp_executor(bucket.set_iam_policy, policy)
# Check if _key exists in bucket. This is where we store private key
# FIXME: Instead, consider *deleting all other keys* and creating a new one,
# each time.
key_blob = bucket.blob('__key__.json')
if not key_blob.exists():
key = await self.run_in_gcp_executor(service.projects(
).serviceAccounts().keys().create(
name=f'projects/{self.gcp_project}/serviceAccounts/{sa_email}',
body={}).execute)
else:
key = json.loads(await self.run_in_gcp_executor(
key_blob.download_as_string))
self.environment['SPARK_GCS_KEY'] = base64.b64decode(
key['privateKeyData']).decode()
def test_create_intake_bucket(monkeypatch):
policy = Policy()
bucket = MagicMock()
bucket.exists.return_value = False
bucket.get_iam_policy.return_value = policy
storage_client = MagicMock()
storage_client.bucket.return_value = bucket
storage_client.create_bucket.return_value = bucket
monkeypatch.setattr("cidc_api.shared.gcloud_client._get_storage_client",
lambda: storage_client)
create_intake_bucket(EMAIL)
# Bucket name should have structure:
# <intake bucket prefix>-<10 character email hash>
name, hash = storage_client.bucket.call_args[0][0].rsplit("-", 1)
assert name == GOOGLE_INTAKE_BUCKET
assert len(hash) == 10 and EMAIL not in hash
# The bucket gets created and permissions get granted
storage_client.create_bucket.assert_called_once_with(bucket)
bucket.get_iam_policy.assert_called_once()
bucket.set_iam_policy.assert_called_once_with(policy)
assert len(policy.bindings) == 1
assert policy.bindings[0]["role"] == GOOGLE_INTAKE_ROLE
assert policy.bindings[0]["members"] == {f"user:{EMAIL}"}
# If the bucket already exists, it doesn't get re-created
storage_client.create_bucket.reset_mock()
bucket.exists.return_value = True
create_intake_bucket(EMAIL)
storage_client.create_bucket.assert_not_called()
def __init__(self, etag=None, version=None):
BasePolicy.__init__(self,
etag=etag if etag is None else _to_bytes(etag),
version=version)
def __init__(self, etag=None, version=None):
BasePolicy.__init__(
self, etag=etag if etag is None else _to_bytes(etag), version=version
)