def check_validity(self, early_return=True, db=None):
"""
Determine whether or not project is valid for registration. If
early_return is False, this object will store information about the
failure.
Args:
early_return (bool, optional): Description
"""
self.google_cloud_manager.open()
logger.debug(
"Google Project with id: {}, "
"new service account requested: {}, project access requested: {}, "
"user requesting: {}".format(
self.google_project_id,
self.new_service_account,
self.new_service_account_access,
self.user_id,
)
)
logger.debug(
"Attempting to get project number "
"for project id {}".format(self.google_project_id)
)
google_project_number = get_google_project_number(
self.google_project_id, self.google_cloud_manager
)
has_access = bool(google_project_number)
self.set("monitor_has_access", has_access)
# always early return if we can't access the project
if not has_access:
logger.warning(
"INVALID Fence's Monitoring service account does "
"NOT have access in project id {}. Monitor needs access to continue "
"checking project validity. Exiting early and determining invalid.".format(
self.user_id, self.google_project_id
)
)
return
logger.debug(
"Retrieving project membership "
"for project id {}".format(self.google_project_id)
)
membership = get_google_project_membership(
self.google_project_id, self.google_cloud_manager
)
logger.debug(
"Project Members: {}".format(
str(
[
getattr(member, "email_id", "unknown_member_email")
for member in membership
]
)
)
)
if self.user_id is not None:
logger.debug(
"Checking that user requesting, {}, is part of "
"the Google Project with id {}".format(
self.user_id, self.google_project_id
)
)
user_has_access = is_user_member_of_google_project(
self.user_id, self.google_cloud_manager, membership=membership, db=db
)
self.set("user_has_access", user_has_access)
if not user_has_access:
# always early return if user isn't a member on the project
logger.warning(
"INVALID User {} "
"for project id {}. User is not a member of the project so does not "
"have permission on the project.".format(
self.user_id, self.google_project_id
)
)
return
logger.debug(
"Retrieving Parent Organization "
"for project id {} to make sure it's valid".format(self.google_project_id)
)
parent_org = get_google_project_parent_org(self.google_cloud_manager)
valid_parent_org = True
if parent_org:
valid_parent_org = is_org_whitelisted(parent_org)
self.set("valid_parent_org", valid_parent_org)
if not valid_parent_org:
logger.warning(
"INVALID Parent Organization {} "
"for project id {}. No parent org is allowed unless it's explicitly "
"whitelisted in cfg.".format(parent_org, self.google_project_id)
)
if early_return:
return
logger.debug(
"Determining if other users and service accounts on "
"project id {} are valid.".format(self.google_project_id)
)
user_members = None
service_account_members = []
try:
user_members, service_account_members = get_google_project_valid_users_and_service_accounts(
self.google_project_id, self.google_cloud_manager, membership=membership
)
self.set("valid_member_types", True)
except Exception:
self.set("valid_member_types", False)
logger.warning(
"INVALID users and/or service accounts (SAs) on "
"project id {}.".format(self.google_project_id)
)
if early_return:
return
logger.debug(
"Determining if valid users exist in fence.".format(self.google_project_id)
)
# if we have valid members, we can check if they exist in fence
users_in_project = None
if user_members is not None:
try:
users_in_project = get_users_from_google_members(user_members, db=db)
self.set("members_exist_in_fence", True)
except Exception as e:
self.set("members_exist_in_fence", False)
logger.warning(
"INVALID user(s) do not exist in fence and thus, "
"we cannot determine their authZ info: {}.".format(e.message)
)
if early_return:
return
# use a generic validityinfo object to hold all the service accounts
# validity. then check all the service accounts. Top level will be
# invalid if any service accounts are invalid
new_service_account_validity = ValidityInfo()
if self.new_service_account:
service_account_validity_info = GoogleServiceAccountValidity(
self.new_service_account,
self.google_project_id,
google_project_number=google_project_number,
google_cloud_manager=self.google_cloud_manager,
)
service_account_id = str(self.new_service_account)
logger.debug(
"Google Project with id: {} and number: {}. "
"Beginning validation on service account for registration {}".format(
self.google_project_id, google_project_number, service_account_id
)
)
logger.debug(
"Determining if the service account {} is google-managed.".format(
service_account_id
)
)
# we do NOT need to check the service account type and external access
# for google-managed accounts.
if is_google_managed_service_account(service_account_id):
logger.debug(
"GCP SA Validity -Service account {} IS google-managed. Therefore, "
"we do NOT need to check the SA Type or if it has external access.".format(
service_account_id
)
)
service_account_validity_info.check_validity(
early_return=early_return,
check_type=True,
check_policy_accessible=True,
check_external_access=False,
)
else:
logger.debug(
"GCP SA Validity -Service account {} is NOT google-managed. Therefore, "
"we need to run all validation checks against it.".format(
service_account_id
)
)
service_account_validity_info.check_validity(
early_return=early_return,
check_type=True,
check_policy_accessible=True,
check_external_access=True,
)
# update project with error info from the service accounts
new_service_account_validity.set(
service_account_id, service_account_validity_info
)
if not service_account_validity_info:
logger.warning("INVALID service account {}.".format(service_account_id))
# if we need to return early for invalid SA, make sure to include
# error details and invalidate the overall validity
if early_return:
self.set("new_service_account", new_service_account_validity)
return
self.set("new_service_account", new_service_account_validity)
logger.debug(
"Google Project with id: {} and number: {}. "
"Beginning validation on project service accounts not requested for "
"registration.".format(self.google_project_id, google_project_number)
)
service_accounts = get_service_account_ids_from_google_members(
service_account_members
)
logger.debug("SAs on the project {}.".format(service_accounts))
remove_white_listed_service_account_ids(service_accounts)
# don't double check service account being registered
if self.new_service_account:
try:
service_accounts.remove(self.new_service_account)
except ValueError:
logger.debug(
"Service Account requested for registration is not a "
"member of the Google project."
)
# use a generic validityinfo object to hold all the service accounts
# validity. then check all the service accounts. Top level will be
# invalid if any service accounts are invalid
service_accounts_validity = ValidityInfo()
for service_account in service_accounts:
service_account_validity_info = self._get_project_sa_validity_info(
service_account, google_project_number, early_return
)
# update project with error info from the service accounts
service_accounts_validity.set(
service_account, service_account_validity_info
)
if not service_account_validity_info and early_return:
# if we need to return early for invalid SA, make sure to include
# error details and invalidate the overall validity
self.set("service_accounts", service_accounts_validity)
return
self.set("service_accounts", service_accounts_validity)
logger.debug(
"Checking data access for Google Project {}...".format(
self.google_project_id
)
)
# get the service accounts for the project to determine all the data
# the project can access through the service accounts
service_accounts = get_registered_service_accounts(
self.google_project_id, db=db
)
service_account_project_access = get_project_access_from_service_accounts(
service_accounts, db=db
)
logger.debug(
"Registered SAs {} current have project access: {}".format(
service_accounts, service_account_project_access
)
)
# use a generic validityinfo object to hold all the projects validity
project_access_validities = ValidityInfo()
# extend list with any provided access to test
for provided_access in self.new_service_account_access:
project = get_project_from_auth_id(provided_access)
# if provided access doesn't exist, set error in project_validity
if not project:
logger.warning(
"INVALID project access requested. "
"Data Access with auth_id {} does not exist.".format(
provided_access
)
)
project_validity = ValidityInfo()
project_validity.set("exists", False)
project_validity.set("all_users_have_access", None)
project_access_validities.set(str(provided_access), project_validity)
else:
service_account_project_access.append(project)
logger.debug(
"New project access requested (in addition to "
"previous access): {}.".format(service_account_project_access)
)
# make sure all the users of the project actually have access to all
# the data the service accounts have access to
for project in service_account_project_access:
project_validity = ValidityInfo()
project_validity.set("exists", True)
# if all the users exist in our db, we can check if they have valid
# access
logger.debug(
"Checking that all users in project have "
"access to project with id {}".format(
getattr(project, "id", "ERROR-could-not-get-project-id")
)
)
valid_access = None
if users_in_project:
valid_access = do_all_users_have_access_to_project(
users_in_project, project.id, db=db
)
if not valid_access:
logger.warning(
"INVALID Some users do NOT have "
"access to project with id {}. users in project: {}".format(
getattr(project, "id", "ERROR-could-not-get-project-id"),
[
getattr(user, "username", "unknown_user")
for user in users_in_project
],
)
)
project_validity.set("all_users_have_access", valid_access)
project_access_validities.set(str(project.auth_id), project_validity)
self.set("access", project_access_validities)
self.google_cloud_manager.close()
return
def _get_project_service_accounts(self, google_project_ids):
"""
Return a list of service accounts for the given Google Cloud
Project IDs.
WARNING: NO AUTHORIZATION CHECK DONE HERE. This will blindly return
all service accounts for the given projects.
Args:
google_project_ids (List(str)): List of unique google project ids
Raises:
List(dict): List of service accounts
Example:
{
"service_accounts": [
{
"service_account_email": "string",
"google_project_id": "string",
"project_access": [
"string"
],
"project_access_exp": 0
}
]
}
"""
all_service_accounts = []
for google_project_id in google_project_ids:
output_service_accounts = []
project_service_accounts = get_registered_service_accounts(
google_project_id)
for project_sa in project_service_accounts:
project_access = get_project_access_from_service_accounts(
[project_sa])
# need to determine expiration by getting the access groups
# and then checking the expiration for each of them
bucket_access_groups = get_google_access_groups_for_service_account(
project_sa)
sa_to_gbags = []
for gbag in bucket_access_groups:
sa_to_gbags.extend(gbag.to_access_groups)
expirations = [
sa_to_gbag.expires for sa_to_gbag in sa_to_gbags
if sa_to_gbag.service_account_id == project_sa.id
]
output_sa = {
"service_account_email":
project_sa.email,
"google_project_id":
project_sa.google_project_id,
"project_access":
[project.auth_id for project in project_access],
"project_access_exp":
min(expirations or [0]),
}
output_service_accounts.append(output_sa)
all_service_accounts.extend(output_service_accounts)
return all_service_accounts
def check_validity(self, early_return=True, db=None, config=None):
"""
Determine whether or not project is valid for registration. If
early_return is False, this object will store information about the
failure.
Args:
early_return (bool, optional): Description
"""
google_project_number = get_google_project_number(
self.google_project_id)
has_access = bool(google_project_number)
self.set("monitor_has_access", has_access)
# always early return if we can't access the project
if not has_access:
return
membership = get_google_project_membership(self.google_project_id)
if self.user_id is not None:
user_has_access = is_user_member_of_all_google_projects(
self.user_id, [self.google_project_id],
membership=membership,
db=db)
self.set("user_has_access", user_has_access)
if not user_has_access:
# always early return if user isn't a member on the project
return
parent_org = get_google_project_parent_org(self.google_project_id)
valid_parent_org = True
# if there is an org, let's remove whitelisted orgs and then check validity
# again
white_listed_google_parent_orgs = (
config.get("WHITE_LISTED_GOOGLE_PARENT_ORGS") if config else None)
if parent_org:
valid_parent_org = is_org_whitelisted(
parent_org,
white_listed_google_parent_orgs=white_listed_google_parent_orgs,
)
self.set("valid_parent_org", valid_parent_org)
if not valid_parent_org and early_return:
return
user_members = None
service_account_members = []
try:
user_members, service_account_members = get_google_project_valid_users_and_service_accounts(
self.google_project_id, membership=membership)
self.set("valid_member_types", True)
except Exception:
self.set("valid_member_types", False)
if early_return:
return
# if we have valid members, we can check if they exist in fence
users_in_project = None
if user_members is not None:
try:
users_in_project = get_users_from_google_members(user_members,
db=db)
self.set("members_exist_in_fence", True)
except Exception:
self.set("members_exist_in_fence", False)
if early_return:
return
# use a generic validityinfo object to hold all the service accounts
# validity. then check all the service accounts. Top level will be
# invalid if any service accounts are invalid
new_service_account_validity = ValidityInfo()
if self.new_service_account:
service_account_validity_info = GoogleServiceAccountValidity(
self.new_service_account, self.google_project_id,
google_project_number)
service_account_id = str(self.new_service_account)
google_sa_domains = (
config.get("GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS")
if config else None)
# we do NOT need to check the service account type and external access
# for google-managed accounts.
if is_google_managed_service_account(
service_account_id,
google_managed_service_account_domains=google_sa_domains,
):
service_account_validity_info.check_validity(
early_return=early_return,
check_type_and_access=False,
config=config,
)
else:
service_account_validity_info.check_validity(
early_return=early_return,
check_type_and_access=True,
config=config)
# update project with error info from the service accounts
new_service_account_validity.set(service_account_id,
service_account_validity_info)
if not service_account_validity_info and early_return:
# if we need to return early for invalid SA, make sure to include
# error details and invalidate the overall validity
self.set("new_service_account", new_service_account_validity)
return
self.set("new_service_account", new_service_account_validity)
service_accounts = get_service_account_ids_from_google_members(
service_account_members)
white_listed_service_accounts = (
config.get("WHITE_LISTED_SERVICE_ACCOUNT_EMAILS")
if config else None)
app_creds_file = (config.get("GOOGLE_APPLICATION_CREDENTIALS")
if config else None)
remove_white_listed_service_account_ids(
service_accounts,
app_creds_file=app_creds_file,
white_listed_sa_emails=white_listed_service_accounts,
)
# use a generic validityinfo object to hold all the service accounts
# validity. then check all the service accounts. Top level will be
# invalid if any service accounts are invalid
service_accounts_validity = ValidityInfo()
for service_account in service_accounts:
service_account_id = str(service_account)
service_account_validity_info = GoogleServiceAccountValidity(
service_account, self.google_project_id, google_project_number)
google_sa_domains = (
config.get("GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS")
if config else None)
# we do NOT need to check the service account type and external access
# for google-managed accounts.
if is_google_managed_service_account(
service_account_id,
google_managed_service_account_domains=google_sa_domains,
):
service_account_validity_info.check_validity(
early_return=early_return,
check_type_and_access=False,
config=config,
)
else:
service_account_validity_info.check_validity(
early_return=early_return,
check_type_and_access=True,
config=config)
# update project with error info from the service accounts
service_accounts_validity.set(service_account_id,
service_account_validity_info)
if not service_account_validity_info and early_return:
# if we need to return early for invalid SA, make sure to include
# error details and invalidate the overall validity
self.set("service_accounts", service_accounts_validity)
return
self.set("service_accounts", service_accounts_validity)
# get the service accounts for the project to determine all the data
# the project can access through the service accounts
service_accounts = get_registered_service_accounts(
self.google_project_id, db=db)
service_account_project_access = get_project_access_from_service_accounts(
service_accounts, db=db)
# use a generic validityinfo object to hold all the projects validity
project_access_validities = ValidityInfo()
# extend list with any provided access to test
for provided_access in self.new_service_account_access:
project = get_project_from_auth_id(provided_access)
# if provided access doesn't exist, set error in project_validity
if not project:
project_validity = ValidityInfo()
project_validity.set("exists", False)
project_validity.set("all_users_have_access", None)
project_access_validities.set(str(provided_access),
project_validity)
else:
service_account_project_access.append(project)
# make sure all the users of the project actually have access to all
# the data the service accounts have access to
for project in service_account_project_access:
project_validity = ValidityInfo()
project_validity.set("exists", True)
# if all the users exist in our db, we can check if they have valid
# access
valid_access = None
if users_in_project:
valid_access = do_all_users_have_access_to_project(
users_in_project, project.id, db=db)
project_validity.set("all_users_have_access", valid_access)
project_access_validities.set(str(project.auth_id),
project_validity)
self.set("access", project_access_validities)
return