def _is_valid_service_account(sa_email, google_project_id):
"""
Validate the given registered service account and remove if invalid.
Args:
sa_email(str): service account email
google_project_id(str): google project id
"""
with GoogleCloudManager(google_project_id) as gcm:
google_project_number = get_google_project_number(
google_project_id, gcm)
has_access = bool(google_project_number)
if not has_access:
# if our monitor doesn't have access at this point, just don't return any
# information. When the project check runs, it will catch the monitor missing
# error and add it to the removal reasons
raise Unauthorized(
"Google Monitoring SA doesn't have access to Google Project: {}".
format(google_project_id))
try:
sa_validity = GoogleServiceAccountValidity(
sa_email,
google_project_id,
google_project_number=google_project_number)
if is_google_managed_service_account(sa_email):
sa_validity.check_validity(
early_return=True,
check_type=True,
check_policy_accessible=True,
check_external_access=False,
)
else:
sa_validity.check_validity(
early_return=True,
check_type=True,
check_policy_accessible=True,
check_external_access=True,
)
except Exception as exc:
# any issues, assume invalid
# TODO not sure if this is the right way to handle this...
logger.warning(
"Service Account {} determined invalid due to unhandled exception: {}. "
"Assuming service account is invalid.".format(sa_email, str(exc)))
traceback.print_exc()
sa_validity = None
return sa_validity
def remove_expired_google_service_account_keys(db):
cirrus_config.update(**config["CIRRUS_CFG"])
db = SQLAlchemyDriver(db)
with db.session as current_session:
client_service_accounts = current_session.query(
GoogleServiceAccount,
Client).filter(GoogleServiceAccount.client_id == Client.client_id)
current_time = int(time.time())
print("Current time: {}\n".format(current_time))
expired_sa_keys_for_users = current_session.query(
GoogleServiceAccountKey).filter(
GoogleServiceAccountKey.expires <= current_time)
with GoogleCloudManager() as g_mgr:
# handle service accounts with default max expiration
for service_account, client in client_service_accounts:
g_mgr.handle_expired_service_account_keys(
service_account.email)
# handle service accounts with custom expiration
for expired_user_key in expired_sa_keys_for_users:
sa = (current_session.query(GoogleServiceAccount).filter(
GoogleServiceAccount.id ==
expired_user_key.service_account_id).first())
response = g_mgr.delete_service_account_key(
account=sa.email, key_name=expired_user_key.key_id)
response_error_code = response.get("error", {}).get("code")
if not response_error_code:
current_session.delete(expired_user_key)
print(
"INFO: Removed expired service account key {} "
"for service account {} (owned by user with id {}).\n".
format(expired_user_key.key_id, sa.email, sa.user_id))
elif response_error_code == 404:
print(
"INFO: Service account key {} for service account {} "
"(owned by user with id {}) does not exist in Google. "
"Removing from database...\n".format(
expired_user_key.key_id, sa.email, sa.user_id))
current_session.delete(expired_user_key)
else:
print("ERROR: Google returned an error when attempting to "
"remove service account key {} "
"for service account {} (owned by user with id {}). "
"Error:\n{}\n".format(expired_user_key.key_id,
sa.email, sa.user_id,
response))
def remove_expired_google_accounts_from_proxy_groups(db):
cirrus_config.update(**config["CIRRUS_CFG"])
db = SQLAlchemyDriver(db)
with db.session as current_session:
current_time = int(time.time())
print("Current time: {}".format(current_time))
expired_accounts = current_session.query(UserGoogleAccountToProxyGroup).filter(
UserGoogleAccountToProxyGroup.expires <= current_time
)
with GoogleCloudManager() as g_mgr:
for expired_account_access in expired_accounts:
g_account = (
current_session.query(UserGoogleAccount)
.filter(
UserGoogleAccount.id
== expired_account_access.user_google_account_id
)
.first()
)
try:
response = g_mgr.remove_member_from_group(
member_email=g_account.email,
group_id=expired_account_access.proxy_group_id,
)
response_error_code = response.get("error", {}).get("code")
if not response_error_code:
current_session.delete(expired_account_access)
print(
"INFO: Removed {} from proxy group with id {}.\n".format(
g_account.email, expired_account_access.proxy_group_id
)
)
else:
print(
"ERROR: Google returned an error when attempting to "
"remove member {} from proxy group {}. Error:\n{}\n".format(
g_account.email,
expired_account_access.proxy_group_id,
response,
)
)
except Exception as exc:
print(
"ERROR: Google returned an error when attempting to "
"remove member {} from proxy group {}. Error:\n{}\n".format(
g_account.email, expired_account_access.proxy_group_id, exc
)
)
def delete(self, bucket, file_id):
try:
with GoogleCloudManager(creds='a') as gcm:
gcm.delete_data_file(bucket, file_id)
return ("", 204)
except Exception as e:
logger.error(e)
try:
status_code = e.resp.status
except Exception as exc:
logger.error(exc)
status_code = 500
return ("Failed to delete data file.", status_code)
def add_user_service_account_to_google(
session, to_add_project_ids, google_project_id, service_account
):
"""
Add service account to Google access groups
Args:
session(current_session): db session
to_add_project_ids (List(id)): list of project ids
google_project_id (str): google project id
service_account (UserServiceAccount): user service account
"""
logger.debug(
"attempting to add {} to groups for projects: {}".format(
service_account, to_add_project_ids
)
)
for project_id in to_add_project_ids:
access_groups = _get_google_access_groups(session, project_id)
logger.debug(
"google group(s) for project {}: {}".format(project_id, access_groups)
)
for access_group in access_groups:
try:
# TODO: Need to remove try/catch after major refactor
with GoogleCloudManager(
google_project_id, use_default=False
) as g_manager:
response = g_manager.add_member_to_group(
member_email=service_account.email, group_id=access_group.email
)
if response.get("email", None):
logger.debug(
"Successfully add member {} to Google group {}.".format(
service_account.email, access_group.email
)
)
else:
raise GoogleAPIError(
"Can not add {} to Google group {}".format(
service_account.email, access_group.email
)
)
except Exception as exc:
raise GoogleAPIError(
"Can not add {} to Google group {}. Detail {}".format(
service_account.email, access_group.email, exc
)
)
def get_test_cloud_manager():
project_id = "test_project"
manager = GoogleCloudManager(project_id)
manager._authed_session = MagicMock()
manager._admin_service = MagicMock()
manager._storage_client = MagicMock()
manager.credentials = MagicMock()
return manager
def create_google_access_key(client_id, user_id, username, proxy_group_id):
"""
Return an access key for current user and client.
NOTE: This will create a service account for the client if one does
not exist.
Returns:
JSON key in Google Credentials File format:
.. code-block:: JavaScript
{
"type": "service_account",
"project_id": "project-id",
"private_key_id": "some_number",
"private_key": "-----BEGIN PRIVATE KEY-----\n....
=\n-----END PRIVATE KEY-----\n",
"client_email": "<api-name>[email protected]",
"client_id": "...",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://accounts.google.com/o/oauth2/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/...<api-name>api%40project-id.iam.gserviceaccount.com"
}
"""
key = {}
service_account = get_or_create_service_account(
client_id=client_id,
user_id=user_id,
username=username,
proxy_group_id=proxy_group_id,
)
with GoogleCloudManager() as g_cloud:
key = g_cloud.get_access_key(service_account.email)
logger.info(
"Created key with id {} for service account {} in user {}'s "
"proxy group {} (user's id: {}).".format(
key.get("private_key_id"),
service_account.email,
username,
proxy_group_id,
user_id,
)
)
return key, service_account
def delete(self, bucket, file_id):
try:
with GoogleCloudManager(
creds=config["CIRRUS_CFG"]["GOOGLE_STORAGE_CREDS"]) as gcm:
gcm.delete_data_file(bucket, file_id)
return ("", 204)
except Exception as e:
logger.error(e)
try:
status_code = e.resp.status
except Exception as exc:
logger.error(exc)
status_code = 500
return ("Failed to delete data file.", status_code)
def get_google_project_membership(project_id):
"""
Returns GCM get_project_membership() result, which is a list of all
members on the projects IAM
Args:
project_id(str): unique id for project
Returns
List(GooglePolicyMember): list of members on project's IAM
"""
with GoogleCloudManager(project_id, use_default=False) as prj:
return prj.get_project_membership(project_id)
def get(self):
"""
List access keys for user
**Example:**
.. code-block:: http
POST /credentials/apis/ HTTP/1.1
Content-Type: application/json
Accept: application/json
Info from Google API /serviceAccounts/<account>/keys endpoint
TODO: In the future we should probably add in our expiration time, when
we start monitoring and deleting after x amount of time
.. code-block:: JavaScript
{
"access_keys":
[
{
"keyAlgorithm": enum(ServiceAccountKeyAlgorithm),
"validBeforeTime": string,
"name": string,
"validAfterTime": string,
},
...
]
}
"""
client_id = current_token.get("azp") or None
user_id = current_token["sub"]
username = current_token.get("context", {}).get("user", {}).get("name")
with GoogleCloudManager() as g_cloud_manager:
proxy_group_id = get_or_create_proxy_group_id()
service_account = get_or_create_service_account(
client_id=client_id,
user_id=user_id,
username=username,
proxy_group_id=proxy_group_id,
)
keys = g_cloud_manager.get_service_account_keys_info(
service_account.google_unique_id)
result = {"access_keys": keys}
return flask.jsonify(result)
def get_google_project_valid_users_and_service_accounts(project_id, membership=None):
"""
Gets google project members of type
USER or SERVICE_ACCOUNT and raises an error if it finds a member
that isn't one of those types.
Args:
project_id (str): Google project ID
membership (List(GooglePolicyMember): pre-calculated list of members,
Will make call to Google API if membership is None
Return:
List[cirrus.google_cloud.iam.GooglePolicyMember]: Members on the
google project
Raises:
NotSupported: Member is invalid type
"""
try:
with GoogleCloudManager(project_id, use_default=False) as prj:
members = membership or prj.get_project_membership(project_id)
for member in members:
if not (
member.member_type == GooglePolicyMember.SERVICE_ACCOUNT
or member.member_type == GooglePolicyMember.USER
):
raise NotSupported(
"Member {} has invalid type: {}".format(
member.email_id, member.member_type
)
)
users = [
member
for member in members
if member.member_type == GooglePolicyMember.USER
]
service_accounts = [
member
for member in members
if member.member_type == GooglePolicyMember.SERVICE_ACCOUNT
]
return users, service_accounts
except Exception as exc:
logger.error(
"validity of Google Project (id: {}) members "
"could not complete. Details: {}".format(project_id, exc)
)
raise
def email_users_without_access(db, auth_ids, user_emails, check_linking,
google_project_id):
"""
Build list of users without acess and send emails.
Args:
db (str): database instance
auth_ids (list(str)): list of project auth_ids to check access against
user_emails (list(str)): list of emails to check access for
check_linking (bool): flag to check for linked google email
Returns:
None
"""
users_without_access = _get_users_without_access(db, auth_ids, user_emails,
check_linking)
if len(users_without_access) == len(user_emails):
logger.warning(
"No user has proper access to provided projects. Contact project administrator. No emails will be sent"
)
return
elif len(users_without_access) > 0:
logger.info(
"Some user(s) do not have proper access to provided projects. Email(s) will be sent to user(s)."
)
with GoogleCloudManager(google_project_id) as gcm:
members = gcm.get_project_membership(google_project_id)
users = []
for member in members:
if member.member_type == GooglePolicyMember.USER:
users.append(member.email_id)
for user, projects in users_without_access.items():
logger.info(
"{} does not have access to the following datasets: {}.".
format(user, ",".join(projects)))
if user in users:
logger.info(
"{} is a member of google project: {}. User will be emailed."
.format(user, google_project_id))
email_user_without_access(user, projects, google_project_id)
else:
logger.info(
"{} is NOT a member of google project: {}. User will NOT be emailed."
.format(user, google_project_id))
else:
logger.info("All users have proper access to provided projects.")
def _remove_client_service_accounts(db_session, client):
client_service_accounts = db_session.query(GoogleServiceAccount).filter(
GoogleServiceAccount.client_id == client.client_id)
with GoogleCloudManager() as g_mgr:
for service_account in client_service_accounts:
print("Deleting client {}'s service account: {}".format(
client.name, service_account.email))
response = g_mgr.delete_service_account(service_account.email)
if not response.get("error"):
db_session.delete(service_account)
db_session.commit()
else:
print("ERROR - from Google: {}".format(response))
print("ERROR - Could not delete client service account: {}".
format(service_account.email))
def delete(self):
"""
.. http:get: /google/
Delete keypair(s) for user
?all=true must be specified
True values are y, yes, t, true, on and 1; false values are n, no, f, false, off and 0
:statuscode 204 Success
:statuscode 403 Forbidden to delete access key
:statuscode 405 Method Not Allowed if ?all=true is not included
"""
user_id = current_token["sub"]
try:
all_arg = strtobool(flask.request.args.get("all", "false").lower())
except ValueError:
all_arg = False
if not all_arg:
flask.abort(
405,
"Please include ?all=true to confirm deletion of ALL Google Service account keys.",
)
with GoogleCloudManager() as g_cloud:
client_id = current_token.get("azp") or None
service_account = get_service_account(client_id, user_id)
if service_account:
keys_for_account = g_cloud.get_service_account_keys_info(
service_account.email)
# Only delete the key if is owned by current client's SA
all_client_keys = [
key["name"].split("/")[-1] for key in keys_for_account
]
for key in all_client_keys:
_delete_service_account_key(g_cloud, service_account.email,
key)
else:
flask.abort(
404, "Could not find service account for current user.")
return "", 204
def create_service_account(client_id, user_id, username, proxy_group_id):
"""
Create a Google Service account for the current client and user.
Args:
g_cloud_manager (cirrus.GoogleCloudManager): instance of
cloud manager to use
Returns:
fence.models.GoogleServiceAccount: New service account
"""
if proxy_group_id:
if client_id:
service_account_id = get_valid_service_account_id_for_client(
client_id, user_id)
else:
service_account_id = get_valid_service_account_id_for_user(
user_id, username)
with GoogleCloudManager() as g_cloud:
new_service_account = g_cloud.create_service_account_for_proxy_group(
proxy_group_id, account_id=service_account_id)
service_account = GoogleServiceAccount(
google_unique_id=new_service_account["uniqueId"],
client_id=client_id,
user_id=user_id,
email=new_service_account["email"],
google_project_id=new_service_account["projectId"],
)
current_session.add(service_account)
current_session.commit()
flask.current_app.logger.info(
"Created service account {} for proxy group {}.".format(
new_service_account["email"], proxy_group_id))
return service_account
else:
flask.abort(
404,
"Could not find Google proxy group for current user in the "
"given token.",
)
def _register_new_service_account(self, sa):
"""
Add service account and related entries to database and add
service account to google bucket access groups
WARNING: this assumes that the project_access provided are all
valid Project.auth_ids, currently checked before this is called
in validity checking
Args:
sa (
fence.resources.google.service_account.GoogleServiceAccountRegistration
): the service account object with its email, project_access, a google project,
and optionally a user who is attempting to modify/add
Return:
(dict): dictionary representing service account object
"""
with GoogleCloudManager(sa.google_project_id) as google_project:
g_service_account = google_project.get_service_account(sa.email)
db_service_account = UserServiceAccount(
google_unique_id=g_service_account.get("uniqueId"),
email=g_service_account.get("email"),
google_project_id=sa.google_project_id,
)
current_session.add(db_service_account)
current_session.commit()
project_ids = get_project_ids_from_project_auth_ids(
current_session, sa.project_access)
add_user_service_account_to_db(current_session, project_ids,
db_service_account)
add_user_service_account_to_google(current_session, project_ids,
sa.google_project_id,
db_service_account)
return {
"service_account_email": g_service_account.get("email"),
"google_project_id": g_service_account.get("projectId"),
"project_access": sa.project_access,
}
def _create_google_bucket_access_group(db_session, google_bucket_name,
bucket_db_id, google_project_id,
privileges):
access_group = None
# use default creds for creating group and iam policies
with GoogleCloudManager(google_project_id) as g_mgr:
# create bucket access group
result = g_mgr.create_group(name=google_bucket_name + "_" +
"_".join(privileges) + "_gbag")
group_email = result["email"]
# add bucket group to db
access_group = GoogleBucketAccessGroup(bucket_id=bucket_db_id,
email=group_email,
privileges=privileges)
db_session.add(access_group)
db_session.commit()
return access_group
def force_remove_service_account_from_access(
service_account_email, google_project_id, db=None
):
"""
loop through ServiceAccountToBucket
remove from google group
delete entries from db
Args:
service_account_email (str): service account email
google_project_id (str): google project id
db (None, str): Optional db connection string
Returns:
None
"""
session = get_db_session(db)
service_account = (
session.query(UserServiceAccount).filter_by(email=service_account_email).first()
)
if not service_account:
raise fence.errors.NotFound(
"{} does not exist in DB".format(service_account_email)
)
access_groups = get_google_access_groups_for_service_account(service_account)
for bucket_access_group in access_groups:
try:
with GoogleCloudManager(google_project_id, use_default=False) as g_manager:
g_manager.remove_member_from_group(
member_email=service_account.email,
group_id=bucket_access_group.email,
)
except Exception as exc:
raise GoogleAPIError(
"Can not remove member {} from access group {}. Detail {}".format(
service_account.email, bucket_access_group.email, exc
)
)
_force_remove_service_account_from_access_db(service_account, access_groups, db)
def delete(self, access_key):
"""
.. http:get: /google/(string: access_key)
Delete a keypair for user
:param access_key: existing access key that belongs to this user
:statuscode 204 Success
:statuscode 403 Forbidden to delete access key
:statuscode 404 Access key doesn't exist
"""
user_id = current_token["sub"]
with GoogleCloudManager() as g_cloud:
client_id = current_token.get("azp") or None
service_account = get_service_account(client_id, user_id)
if service_account:
keys_for_account = g_cloud.get_service_account_keys_info(
service_account.google_unique_id)
# Only delete the key if is owned by current client's SA
all_client_keys = [
key["name"].split("/")[-1] for key in keys_for_account
]
if access_key in all_client_keys:
g_cloud.delete_service_account_key(
service_account.google_unique_id, access_key)
db_entry = (current_session.query(GoogleServiceAccountKey).
filter_by(key_id=access_key).first())
if db_entry:
current_session.delete(db_entry)
current_session.commit()
else:
flask.abort(
404,
"Could not delete key " + access_key +
". Not found for current user.",
)
else:
flask.abort(
404, "Could not find service account for current user.")
return "", 204
def get_google_project_parent_org(project_id):
"""
Checks if google project has parent org. Wraps
GoogleCloudManager.get_project_organization()
Args:
project_id(str): unique id for project
Returns:
str: The Google projects parent organization name or None if it does't have one
"""
try:
with GoogleCloudManager(project_id, use_default=False) as prj:
return prj.get_project_organization()
except Exception as exc:
logger.error(
"Could not determine if Google project (id: {}) has parent org"
"due to error (Details: {})".format(project_id, exc)
)
def _revoke_user_service_account_from_google(
session, to_delete_project_ids, google_project_id, service_account
):
"""
revoke service account from google access group
Args:
session(current_session): db session
to_delete_project_ids (List(str)): list of project ids
google_project_id (str): google project id
service_account (UserServiceAccount): user service account
Returns:
None
"""
for project_id in to_delete_project_ids:
access_groups = _get_google_access_groups(session, project_id)
for access_group in access_groups:
try:
# TODO: Need to remove outer try/catch after major refactor
with GoogleCloudManager(
google_project_id, use_default=False
) as g_manager:
if not g_manager.remove_member_from_group(
member_email=service_account.email, group_id=access_group.email
):
logger.debug(
"Removed {} from google group {}".format(
service_account.email, access_group.email
)
)
else:
raise GoogleAPIError("Can not remove {} from group {}")
except Exception as exc:
raise GoogleAPIError(
"Can not remove {} from group {}. Detail {}".format(
service_account.email, access_group.email, exc
)
)
def delete(self, access_key):
"""
.. http:get: /google/(string: access_key)
Delete keypair(s) for user
:param access_key: existing access key that belongs to this user
:statuscode 204 Success
:statuscode 403 Forbidden to delete access key
:statuscode 404 Access key doesn't exist
"""
user_id = current_token["sub"]
username = current_token.get("context", {}).get("user", {}).get("name")
with GoogleCloudManager() as g_cloud:
client_id = current_token.get("azp") or None
service_account = get_service_account(client_id, user_id, username=username)
if service_account:
keys_for_account = g_cloud.get_service_account_keys_info(
service_account.email
)
# Only delete the key if is owned by current client's SA
all_client_keys = [
key["name"].split("/")[-1] for key in keys_for_account
]
if access_key in all_client_keys:
_delete_service_account_key(
g_cloud, service_account.email, access_key
)
else:
flask.abort(
404,
"Could not delete key "
+ access_key
+ ". Not found for current user.",
)
else:
flask.abort(404, "Could not find service account for current user.")
return "", 204
def get_user(self, username):
"""
Get a user
Args:
username (str): An email address representing a User's Google
Proxy Group (e.g. a single Google Group to hold a single
user's diff identities).
Returns:
UserProxy: a UserProxy object if the user exists, else None
"""
user_proxy = None
with GoogleCloudManager(project_id=self.google_project_id) as g_mgr:
user_proxy_response = g_mgr.get_group(username)
if user_proxy_response.get("email"):
user_proxy = UserProxy(username=user_proxy_response.get("email"))
return user_proxy
def force_add_service_accounts_to_access(
service_account_emails, google_project_id, project_access, db=None
):
"""
service_account_emails(list(str)): list of account emails
google_project_id(str): google project id
project_access(list(str)): list of projects
db(str): db connection string
"""
session = get_db_session(db)
with GoogleCloudManager(google_project_id) as google_project:
for service_account_email in service_account_emails:
g_service_account = google_project.get_service_account(
service_account_email
)
sa = (
session.query(UserServiceAccount)
.filter_by(email=service_account_email)
.first()
)
if not sa:
sa = UserServiceAccount(
google_unique_id=g_service_account.get("uniqueId"),
email=service_account_email,
google_project_id=google_project_id,
)
session.add(sa)
session.commit()
project_ids = set()
for project in project_access:
project_db = session.query(Project).filter_by(auth_id=project).first()
if project_db:
project_ids.add(project_db.id)
add_user_service_account_to_db(session, project_ids, sa)
add_user_service_account_to_google(
session, project_ids, google_project_id, sa
)
def verify_bucket_access_group(DB):
"""
Go through all the google group members, remove them from Google group and Google
user service account if they are not in Fence
Args:
DB(str): db connection string
Returns:
None
"""
cirrus_config.update(**config["CIRRUS_CFG"])
driver = SQLAlchemyDriver(DB)
with driver.session as session:
access_groups = session.query(GoogleBucketAccessGroup).all()
with GoogleCloudManager() as manager:
for access_group in access_groups:
try:
members = manager.get_group_members(access_group.email)
logger.debug(f"google group members response: {members}")
except GoogleAuthError as e:
logger.error("ERROR: Authentication error!!!. Detail {}".format(e))
return
except Exception as e:
logger.error(
"ERROR: Could not list group members of {}. Detail {}".format(
access_group.email, e
)
)
return
for member in members:
if member.get("type") == "GROUP":
_verify_google_group_member(session, access_group, member)
elif member.get("type") == "USER":
_verify_google_service_account_member(
session, access_group, member
)
def _verify_google_service_account_member(session, access_group, member):
"""
Delete if the member which is a service account is not in Fence.
Args:
session(session): db session
access_group(GoogleBucketAccessGroup): access group
members(dict): service account member info
Returns:
None
"""
account_emails = [
account.service_account.email
for account in (
session.query(ServiceAccountToGoogleBucketAccessGroup)
.filter_by(access_group_id=access_group.id)
.all()
)
]
if not any([email for email in account_emails if email == member.get("email")]):
try:
with GoogleCloudManager() as manager:
manager.remove_member_from_group(
member.get("email"), access_group.email
)
logger.info(
"Removed {} from {}, not found in fence but found "
"in Google Group.".format(member.get("email"), access_group.email)
)
except Exception as e:
logger.error(
"ERROR: Could not remove service account memeber {} from access group {}. Detail {}".format(
member.get("email"), access_group.email, e
)
)
def delete_bucket_acl(self, bucket, user):
"""
Set quota for the entire bucket
Args:
bucket (str): Google Bucket Access Group email address. This should
be the address of a Google Group that has read access on a
single bucket. Access is controlled by adding members to this
group.
user (str): An email address of a member to add to the Google
Bucket Access Group.
"""
response = None
with GoogleCloudManager(project_id=self.google_project_id) as g_mgr:
try:
response = g_mgr.remove_member_from_group(
member_email=user, group_id=bucket
)
except Exception as exc:
raise RequestError("Google API Error: {}".format(exc), code=400)
return response
def bulk_update_google_groups(google_bulk_mapping):
"""
Update Google Groups based on mapping provided from Group -> Users.
Args:
google_bulk_mapping (dict): {"*****@*****.**": ["member1", "member2"]}
"""
google_project_id = (
config["STORAGE_CREDENTIALS"].get("google", {}).get("google_project_id")
)
with GoogleCloudManager(google_project_id) as gcm:
for group, expected_members in google_bulk_mapping.items():
expected_members = set(expected_members)
logger.debug(f"Starting diff for group {group}...")
# get members list from google
google_members = set(
member.get("email") for member in gcm.get_group_members(group)
)
logger.debug(f"Google membership for {group}: {google_members}")
logger.debug(f"Expected membership for {group}: {expected_members}")
# diff between expected group membership and actual membership
to_delete = set.difference(google_members, expected_members)
to_add = set.difference(expected_members, google_members)
no_update = set.intersection(google_members, expected_members)
logger.info(f"All already in group {group}: {no_update}")
# do add
for member_email in to_add:
logger.info(f"Adding to group {group}: {member_email}")
gcm.add_member_to_group(member_email, group)
# do remove
for member_email in to_delete:
logger.info(f"Removing from group {group}: {member_email}")
gcm.remove_member_from_group(member_email, group)
def _verify_google_group_member(session, access_group, member):
"""
Delete if the member which is a google group is not in Fence.
Args:
session(Session): db session
access_group(GoogleBucketAccessGroup): access group
member(dict): group member info
Returns:
None
"""
account_emails = [
granted_group.proxy_group.email
for granted_group in (
session.query(GoogleProxyGroupToGoogleBucketAccessGroup)
.filter_by(access_group_id=access_group.id)
.all()
)
]
if not any([email for email in account_emails if email == member.get("email")]):
try:
with GoogleCloudManager() as manager:
manager.remove_member_from_group(
member.get("email"), access_group.email
)
print(
"Removed {} from {}, not found in fence but found "
"in Google Group.".format(member.get("email"), access_group.email)
)
except Exception as e:
print(
"ERROR: Could not remove google group memeber {} from access group {}. Detail {}".format(
member.get("email"), access_group.email, e
)
)
def get_or_create_service_account(client_id, user_id, username,
proxy_group_id):
"""
Create a Google Service account for the current client and user.
This effectively handles conflicts in Google and will update our db
accordingly based on the newest information from Google.
Args:
g_cloud_manager (cirrus.GoogleCloudManager): instance of
cloud manager to use
Returns:
fence.models.GoogleServiceAccount: New service account
"""
if proxy_group_id:
if client_id:
service_account_id = get_valid_service_account_id_for_client(
client_id,
user_id,
prefix=config["GOOGLE_SERVICE_ACCOUNT_PREFIX"])
else:
service_account_id = get_valid_service_account_id_for_user(
user_id,
username,
prefix=config["GOOGLE_SERVICE_ACCOUNT_PREFIX"])
with GoogleCloudManager() as g_cloud:
new_service_account = g_cloud.create_service_account_for_proxy_group(
proxy_group_id, account_id=service_account_id)
return _update_service_account_db_entry(client_id, user_id,
proxy_group_id,
new_service_account)
else:
flask.abort(
404,
"Could not find Google proxy group for current user in the given token.",
)
