def _get_project_sa_validity_info(
self, service_account, google_project_number, early_return
):
service_account_id = str(service_account)
service_account_validity_info = GoogleServiceAccountValidity(
service_account,
self.google_project_id,
google_project_number=google_project_number,
google_cloud_manager=self.google_cloud_manager,
)
logger.debug(
"Google Project with id: {} and number: {}. "
"Beginning validation on project service account {}".format(
self.google_project_id, google_project_number, service_account_id
)
)
logger.debug(
"Determining if the service account {} is google-managed.".format(
service_account_id
)
)
# we do NOT need to check the service account type and external access
# for google-managed accounts.
if is_google_managed_service_account(service_account_id):
logger.debug(
"Service account {} IS google-managed. Therefore, "
"we only need to detemine if it belongs.".format(service_account_id)
)
service_account_validity_info.check_validity(
early_return=early_return,
check_type=False,
check_policy_accessible=False,
check_external_access=False,
)
else:
logger.debug(
"Service account {} is NOT google-managed. Therefore, "
"we need to run all validation checks against it.".format(
service_account_id
)
)
service_account_validity_info.check_validity(
early_return=early_return,
check_type=True,
check_policy_accessible=True,
check_external_access=True,
)
return service_account_validity_info
def _is_valid_service_account(sa_email, google_project_id):
"""
Validate the given registered service account and remove if invalid.
Args:
sa_email(str): service account email
google_project_id(str): google project id
"""
with GoogleCloudManager(google_project_id) as gcm:
google_project_number = get_google_project_number(google_project_id, gcm)
has_access = bool(google_project_number)
if not has_access:
# if our monitor doesn't have access at this point, just don't return any
# information. When the project check runs, it will catch the monitor missing
# error and add it to the removal reasons
raise Unauthorized(
"Google Monitoring SA doesn't have access to Google Project: {}".format(
google_project_id
)
)
try:
sa_validity = GoogleServiceAccountValidity(
sa_email, google_project_id, google_project_number=google_project_number
)
if is_google_managed_service_account(sa_email):
sa_validity.check_validity(
early_return=True,
check_type=True,
check_policy_accessible=True,
check_external_access=False,
)
else:
sa_validity.check_validity(
early_return=True,
check_type=True,
check_policy_accessible=True,
check_external_access=True,
)
except Exception as exc:
# any issues, assume invalid
# TODO not sure if this is the right way to handle this...
logger.warning(
"Service Account {} determined invalid due to unhandled exception: {}. "
"Assuming service account is invalid.".format(sa_email, str(exc))
)
traceback.print_exc()
sa_validity = None
return sa_validity
def is_service_account_from_google_project(
service_account_email, project_id, project_number, google_managed_sa_domains=None
):
"""
Checks if service account is among project's service acounts
Args:
service_account_email(str): service account email
project_id(str): uniqueId of Google Cloud Project
Return:
Bool: True iff the given service_account_email is from the
given Google Project
"""
try:
service_account_name = service_account_email.split("@")[0]
if is_google_managed_service_account(
service_account_email,
google_managed_service_account_domains=google_managed_sa_domains,
):
return (
service_account_name == "service-{}".format(project_number)
or service_account_name == "project-{}".format(project_number)
or service_account_name == project_number
or service_account_name == project_id
)
# if it's a user-created project SA, the id is in the domain, otherwise,
# attempt to parse it out of the name
domain = service_account_email.split("@")[-1]
if "iam.gserviceaccount.com" in domain:
return domain.split(".")[0] == project_id
return (
service_account_name == "{}-compute".format(project_number)
or service_account_name == project_id
)
except Exception as exc:
logger.error(
"Could not determine if service account (id: {} is from project"
" (id: {}) due to error. Details: {}".format(
service_account_email, project_id, exc
)
)
return False
def check_validity(self, early_return=True, db=None):
"""
Determine whether or not project is valid for registration. If
early_return is False, this object will store information about the
failure.
Args:
early_return (bool, optional): Description
"""
self.google_cloud_manager.open()
logger.debug(
"Google Project with id: {}, "
"new service account requested: {}, project access requested: {}, "
"user requesting: {}".format(
self.google_project_id,
self.new_service_account,
self.new_service_account_access,
self.user_id,
)
)
logger.debug(
"Attempting to get project number "
"for project id {}".format(self.google_project_id)
)
google_project_number = get_google_project_number(
self.google_project_id, self.google_cloud_manager
)
has_access = bool(google_project_number)
self.set("monitor_has_access", has_access)
# always early return if we can't access the project
if not has_access:
logger.warning(
"INVALID Fence's Monitoring service account does "
"NOT have access in project id {}. Monitor needs access to continue "
"checking project validity. Exiting early and determining invalid.".format(
self.user_id, self.google_project_id
)
)
return
logger.debug(
"Retrieving project membership "
"for project id {}".format(self.google_project_id)
)
membership = get_google_project_membership(
self.google_project_id, self.google_cloud_manager
)
logger.debug(
"Project Members: {}".format(
str(
[
getattr(member, "email_id", "unknown_member_email")
for member in membership
]
)
)
)
if self.user_id is not None:
logger.debug(
"Checking that user requesting, {}, is part of "
"the Google Project with id {}".format(
self.user_id, self.google_project_id
)
)
user_has_access = is_user_member_of_google_project(
self.user_id, self.google_cloud_manager, membership=membership, db=db
)
self.set("user_has_access", user_has_access)
if not user_has_access:
# always early return if user isn't a member on the project
logger.warning(
"INVALID User {} "
"for project id {}. User is not a member of the project so does not "
"have permission on the project.".format(
self.user_id, self.google_project_id
)
)
return
logger.debug(
"Retrieving Parent Organization "
"for project id {} to make sure it's valid".format(self.google_project_id)
)
parent_org = get_google_project_parent_org(self.google_cloud_manager)
valid_parent_org = True
if parent_org:
valid_parent_org = is_org_whitelisted(parent_org)
self.set("valid_parent_org", valid_parent_org)
if not valid_parent_org:
logger.warning(
"INVALID Parent Organization {} "
"for project id {}. No parent org is allowed unless it's explicitly "
"whitelisted in cfg.".format(parent_org, self.google_project_id)
)
if early_return:
return
logger.debug(
"Determining if other users and service accounts on "
"project id {} are valid.".format(self.google_project_id)
)
user_members = None
service_account_members = []
try:
user_members, service_account_members = get_google_project_valid_users_and_service_accounts(
self.google_project_id, self.google_cloud_manager, membership=membership
)
self.set("valid_member_types", True)
except Exception:
self.set("valid_member_types", False)
logger.warning(
"INVALID users and/or service accounts (SAs) on "
"project id {}.".format(self.google_project_id)
)
if early_return:
return
logger.debug(
"Determining if valid users exist in fence.".format(self.google_project_id)
)
# if we have valid members, we can check if they exist in fence
users_in_project = None
if user_members is not None:
try:
users_in_project = get_users_from_google_members(user_members, db=db)
self.set("members_exist_in_fence", True)
except Exception as e:
self.set("members_exist_in_fence", False)
logger.warning(
"INVALID user(s) do not exist in fence and thus, "
"we cannot determine their authZ info: {}.".format(e.message)
)
if early_return:
return
# use a generic validityinfo object to hold all the service accounts
# validity. then check all the service accounts. Top level will be
# invalid if any service accounts are invalid
new_service_account_validity = ValidityInfo()
if self.new_service_account:
service_account_validity_info = GoogleServiceAccountValidity(
self.new_service_account,
self.google_project_id,
google_project_number=google_project_number,
google_cloud_manager=self.google_cloud_manager,
)
service_account_id = str(self.new_service_account)
logger.debug(
"Google Project with id: {} and number: {}. "
"Beginning validation on service account for registration {}".format(
self.google_project_id, google_project_number, service_account_id
)
)
logger.debug(
"Determining if the service account {} is google-managed.".format(
service_account_id
)
)
# we do NOT need to check the service account type and external access
# for google-managed accounts.
if is_google_managed_service_account(service_account_id):
logger.debug(
"GCP SA Validity -Service account {} IS google-managed. Therefore, "
"we do NOT need to check the SA Type or if it has external access.".format(
service_account_id
)
)
service_account_validity_info.check_validity(
early_return=early_return,
check_type=True,
check_policy_accessible=True,
check_external_access=False,
)
else:
logger.debug(
"GCP SA Validity -Service account {} is NOT google-managed. Therefore, "
"we need to run all validation checks against it.".format(
service_account_id
)
)
service_account_validity_info.check_validity(
early_return=early_return,
check_type=True,
check_policy_accessible=True,
check_external_access=True,
)
# update project with error info from the service accounts
new_service_account_validity.set(
service_account_id, service_account_validity_info
)
if not service_account_validity_info:
logger.warning("INVALID service account {}.".format(service_account_id))
# if we need to return early for invalid SA, make sure to include
# error details and invalidate the overall validity
if early_return:
self.set("new_service_account", new_service_account_validity)
return
self.set("new_service_account", new_service_account_validity)
logger.debug(
"Google Project with id: {} and number: {}. "
"Beginning validation on project service accounts not requested for "
"registration.".format(self.google_project_id, google_project_number)
)
service_accounts = get_service_account_ids_from_google_members(
service_account_members
)
logger.debug("SAs on the project {}.".format(service_accounts))
remove_white_listed_service_account_ids(service_accounts)
# don't double check service account being registered
if self.new_service_account:
try:
service_accounts.remove(self.new_service_account)
except ValueError:
logger.debug(
"Service Account requested for registration is not a "
"member of the Google project."
)
# use a generic validityinfo object to hold all the service accounts
# validity. then check all the service accounts. Top level will be
# invalid if any service accounts are invalid
service_accounts_validity = ValidityInfo()
for service_account in service_accounts:
service_account_validity_info = self._get_project_sa_validity_info(
service_account, google_project_number, early_return
)
# update project with error info from the service accounts
service_accounts_validity.set(
service_account, service_account_validity_info
)
if not service_account_validity_info and early_return:
# if we need to return early for invalid SA, make sure to include
# error details and invalidate the overall validity
self.set("service_accounts", service_accounts_validity)
return
self.set("service_accounts", service_accounts_validity)
logger.debug(
"Checking data access for Google Project {}...".format(
self.google_project_id
)
)
# get the service accounts for the project to determine all the data
# the project can access through the service accounts
service_accounts = get_registered_service_accounts(
self.google_project_id, db=db
)
service_account_project_access = get_project_access_from_service_accounts(
service_accounts, db=db
)
logger.debug(
"Registered SAs {} current have project access: {}".format(
service_accounts, service_account_project_access
)
)
# use a generic validityinfo object to hold all the projects validity
project_access_validities = ValidityInfo()
# extend list with any provided access to test
for provided_access in self.new_service_account_access:
project = get_project_from_auth_id(provided_access)
# if provided access doesn't exist, set error in project_validity
if not project:
logger.warning(
"INVALID project access requested. "
"Data Access with auth_id {} does not exist.".format(
provided_access
)
)
project_validity = ValidityInfo()
project_validity.set("exists", False)
project_validity.set("all_users_have_access", None)
project_access_validities.set(str(provided_access), project_validity)
else:
service_account_project_access.append(project)
logger.debug(
"New project access requested (in addition to "
"previous access): {}.".format(service_account_project_access)
)
# make sure all the users of the project actually have access to all
# the data the service accounts have access to
for project in service_account_project_access:
project_validity = ValidityInfo()
project_validity.set("exists", True)
# if all the users exist in our db, we can check if they have valid
# access
logger.debug(
"Checking that all users in project have "
"access to project with id {}".format(
getattr(project, "id", "ERROR-could-not-get-project-id")
)
)
valid_access = None
if users_in_project:
valid_access = do_all_users_have_access_to_project(
users_in_project, project.id, db=db
)
if not valid_access:
logger.warning(
"INVALID Some users do NOT have "
"access to project with id {}. users in project: {}".format(
getattr(project, "id", "ERROR-could-not-get-project-id"),
[
getattr(user, "username", "unknown_user")
for user in users_in_project
],
)
)
project_validity.set("all_users_have_access", valid_access)
project_access_validities.set(str(project.auth_id), project_validity)
self.set("access", project_access_validities)
self.google_cloud_manager.close()
return
def check_validity(self, early_return=True, db=None, config=None):
"""
Determine whether or not project is valid for registration. If
early_return is False, this object will store information about the
failure.
Args:
early_return (bool, optional): Description
"""
google_project_number = get_google_project_number(
self.google_project_id)
has_access = bool(google_project_number)
self.set("monitor_has_access", has_access)
# always early return if we can't access the project
if not has_access:
return
membership = get_google_project_membership(self.google_project_id)
if self.user_id is not None:
user_has_access = is_user_member_of_all_google_projects(
self.user_id, [self.google_project_id],
membership=membership,
db=db)
self.set("user_has_access", user_has_access)
if not user_has_access:
# always early return if user isn't a member on the project
return
parent_org = get_google_project_parent_org(self.google_project_id)
valid_parent_org = True
# if there is an org, let's remove whitelisted orgs and then check validity
# again
white_listed_google_parent_orgs = (
config.get("WHITE_LISTED_GOOGLE_PARENT_ORGS") if config else None)
if parent_org:
valid_parent_org = is_org_whitelisted(
parent_org,
white_listed_google_parent_orgs=white_listed_google_parent_orgs,
)
self.set("valid_parent_org", valid_parent_org)
if not valid_parent_org and early_return:
return
user_members = None
service_account_members = []
try:
user_members, service_account_members = get_google_project_valid_users_and_service_accounts(
self.google_project_id, membership=membership)
self.set("valid_member_types", True)
except Exception:
self.set("valid_member_types", False)
if early_return:
return
# if we have valid members, we can check if they exist in fence
users_in_project = None
if user_members is not None:
try:
users_in_project = get_users_from_google_members(user_members,
db=db)
self.set("members_exist_in_fence", True)
except Exception:
self.set("members_exist_in_fence", False)
if early_return:
return
# use a generic validityinfo object to hold all the service accounts
# validity. then check all the service accounts. Top level will be
# invalid if any service accounts are invalid
new_service_account_validity = ValidityInfo()
if self.new_service_account:
service_account_validity_info = GoogleServiceAccountValidity(
self.new_service_account, self.google_project_id,
google_project_number)
service_account_id = str(self.new_service_account)
google_sa_domains = (
config.get("GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS")
if config else None)
# we do NOT need to check the service account type and external access
# for google-managed accounts.
if is_google_managed_service_account(
service_account_id,
google_managed_service_account_domains=google_sa_domains,
):
service_account_validity_info.check_validity(
early_return=early_return,
check_type_and_access=False,
config=config,
)
else:
service_account_validity_info.check_validity(
early_return=early_return,
check_type_and_access=True,
config=config)
# update project with error info from the service accounts
new_service_account_validity.set(service_account_id,
service_account_validity_info)
if not service_account_validity_info and early_return:
# if we need to return early for invalid SA, make sure to include
# error details and invalidate the overall validity
self.set("new_service_account", new_service_account_validity)
return
self.set("new_service_account", new_service_account_validity)
service_accounts = get_service_account_ids_from_google_members(
service_account_members)
white_listed_service_accounts = (
config.get("WHITE_LISTED_SERVICE_ACCOUNT_EMAILS")
if config else None)
app_creds_file = (config.get("GOOGLE_APPLICATION_CREDENTIALS")
if config else None)
remove_white_listed_service_account_ids(
service_accounts,
app_creds_file=app_creds_file,
white_listed_sa_emails=white_listed_service_accounts,
)
# use a generic validityinfo object to hold all the service accounts
# validity. then check all the service accounts. Top level will be
# invalid if any service accounts are invalid
service_accounts_validity = ValidityInfo()
for service_account in service_accounts:
service_account_id = str(service_account)
service_account_validity_info = GoogleServiceAccountValidity(
service_account, self.google_project_id, google_project_number)
google_sa_domains = (
config.get("GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS")
if config else None)
# we do NOT need to check the service account type and external access
# for google-managed accounts.
if is_google_managed_service_account(
service_account_id,
google_managed_service_account_domains=google_sa_domains,
):
service_account_validity_info.check_validity(
early_return=early_return,
check_type_and_access=False,
config=config,
)
else:
service_account_validity_info.check_validity(
early_return=early_return,
check_type_and_access=True,
config=config)
# update project with error info from the service accounts
service_accounts_validity.set(service_account_id,
service_account_validity_info)
if not service_account_validity_info and early_return:
# if we need to return early for invalid SA, make sure to include
# error details and invalidate the overall validity
self.set("service_accounts", service_accounts_validity)
return
self.set("service_accounts", service_accounts_validity)
# get the service accounts for the project to determine all the data
# the project can access through the service accounts
service_accounts = get_registered_service_accounts(
self.google_project_id, db=db)
service_account_project_access = get_project_access_from_service_accounts(
service_accounts, db=db)
# use a generic validityinfo object to hold all the projects validity
project_access_validities = ValidityInfo()
# extend list with any provided access to test
for provided_access in self.new_service_account_access:
project = get_project_from_auth_id(provided_access)
# if provided access doesn't exist, set error in project_validity
if not project:
project_validity = ValidityInfo()
project_validity.set("exists", False)
project_validity.set("all_users_have_access", None)
project_access_validities.set(str(provided_access),
project_validity)
else:
service_account_project_access.append(project)
# make sure all the users of the project actually have access to all
# the data the service accounts have access to
for project in service_account_project_access:
project_validity = ValidityInfo()
project_validity.set("exists", True)
# if all the users exist in our db, we can check if they have valid
# access
valid_access = None
if users_in_project:
valid_access = do_all_users_have_access_to_project(
users_in_project, project.id, db=db)
project_validity.set("all_users_have_access", valid_access)
project_access_validities.set(str(project.auth_id),
project_validity)
self.set("access", project_access_validities)
return