def extend_service_account_access(service_account_email, db=None):
"""
Extend the Google service accounts access to data by extending the
expiration time for each of the Google Bucket Access Groups it's in.
WARNING: This does NOT do any AuthZ, do before this.
Args:
service_account_email (str): service account email
db(str): db connection string
"""
session = get_db_session(db)
service_account = (
session.query(UserServiceAccount).filter_by(email=service_account_email).first()
)
if service_account:
bucket_access_groups = get_google_access_groups_for_service_account(
service_account
)
# timestamp at which the SA will lose bucket access
# by default: use configured time or 7 days
expiration_time = int(time.time()) + config.get(
"GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN", 604800
)
requested_expires_in = get_valid_expiration_from_request()
if requested_expires_in:
requested_expiration = int(time.time()) + requested_expires_in
expiration_time = min(expiration_time, requested_expiration)
logger.debug(
"Service Account ({}) access extended to {}.".format(
service_account.email, expiration_time
)
)
for access_group in bucket_access_groups:
bucket_access = (
session.query(ServiceAccountToGoogleBucketAccessGroup)
.filter_by(
service_account_id=service_account.id,
access_group_id=access_group.id,
)
.first()
)
if not bucket_access:
bucket_access = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_account.id,
access_group_id=access_group.id,
expires=expiration_time,
)
session.add(bucket_access)
bucket_access.expires = expiration_time
session.commit()
def test_delete_expired_service_accounts_with_one_fail_second(
cloud_manager, app, db_session
):
"""
Test the case that there is a failure of removing service account from google group
"""
from googleapiclient.errors import HttpError
import fence
fence.settings = MagicMock()
cloud_manager.return_value.__enter__.return_value.remove_member_from_group.side_effect = [
{},
HttpError(mock.Mock(status=403), bytes("Permission denied", "utf-8")),
]
_setup_service_account_to_google_bucket_access_group(db_session)
service_accounts = db_session.query(UserServiceAccount).all()
google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all()
current_time = int(time.time())
# Add two expired service account. They are both supposed to be deleted but
# only one is deleted due to a raise exception
service_account1 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_accounts[0].id,
expires=current_time - 3600,
access_group_id=google_bucket_access_grps[0].id,
)
service_account2 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_accounts[1].id,
expires=current_time - 3600,
access_group_id=google_bucket_access_grps[1].id,
)
db_session.add(service_account1)
db_session.add(service_account2)
db_session.commit()
# check database to make sure there are two records in DB
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 2
# call function to delete expired service account
delete_expired_service_accounts(config["DB"])
# check database to make sure only the first one is deleted, the second one
# still exists because of the raised exception
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 1
assert records[0].id == service_account2.id
def test_delete_expired_service_accounts_with_one_fail_first(
cloud_manager, app, db_session
):
"""
Test the case that there is a failure of removing service account from google group
"""
from googleapiclient.errors import HttpError
import fence
fence.settings = MagicMock()
cirrus.config.update = MagicMock()
cloud_manager.return_value.__enter__.return_value.remove_member_from_group.side_effect = [
HttpError(mock.Mock(status=403), bytes("Permission denied", "utf-8")),
{},
]
_setup_service_account_to_google_bucket_access_group(db_session)
service_accounts = db_session.query(UserServiceAccount).all()
google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all()
current_time = int(time.time())
# Add expired service account. This acccount is supposed to be deleted
db_session.add(
ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_accounts[0].id,
expires=current_time - 3600,
access_group_id=google_bucket_access_grps[0].id,
)
)
# Add non-expired service account.
db_session.add(
ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_accounts[1].id,
expires=current_time + 3600,
access_group_id=google_bucket_access_grps[1].id,
)
)
db_session.commit()
# check database to make sure all the service accounts exist
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 2
# call function to delete expired service account
delete_expired_service_accounts(config["DB"])
# check database again. Expect no service account is deleted
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 2
def test_delete_expired_service_accounts(cloud_manager, app, db_session):
"""
Test deleting all expired service accounts
"""
import fence
fence.settings = MagicMock()
cloud_manager.return_value.__enter__.return_value.remove_member_from_group.return_value = (
{}
)
_setup_service_account_to_google_bucket_access_group(db_session)
service_accounts = db_session.query(UserServiceAccount).all()
google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all()
current_time = int(time.time())
# Add 2 expired and 1 not expired accounts
service_account1 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_accounts[0].id,
expires=current_time - 3600,
access_group_id=google_bucket_access_grps[0].id,
)
service_account2 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_accounts[0].id,
expires=current_time - 3600,
access_group_id=google_bucket_access_grps[1].id,
)
service_account3 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_accounts[1].id,
expires=current_time + 3600,
access_group_id=google_bucket_access_grps[1].id,
)
db_session.add(service_account1)
db_session.add(service_account2)
db_session.add(service_account3)
db_session.commit()
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 3
# call function to delete expired service account
delete_expired_service_accounts(config["DB"])
# check database. Expect 2 deleted
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 1
assert records[0].id == service_account3.id
def test_delete_not_expired_service_account(app, db_session):
"""
Test the case that there is no expired service account
"""
import fence
fence.settings = MagicMock()
_setup_service_account_to_google_bucket_access_group(db_session)
service_account = db_session.query(UserServiceAccount).first()
google_bucket_access_grp1 = db_session.query(
GoogleBucketAccessGroup).first()
current_time = int(time.time())
# Add non-expired service account
service_account = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_account.id,
expires=current_time + 3600,
access_group_id=google_bucket_access_grp1.id,
)
db_session.add(service_account)
db_session.commit()
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 1
# call function to delete expired service account
delete_expired_service_accounts(app.config["DB"])
# check db again to make sure the record still exists
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 1
def extend_service_account_access(service_account_email, db=None):
"""
Extend the Google service accounts access to data by extending the
expiration time for each of the Google Bucket Access Groups it's in.
WARNING: This does NOT do any AuthZ, do before this.
Args:
service_account_email (str): service account email
db(str): db connection string
"""
session = get_db_session(db)
service_account = (
session.query(UserServiceAccount).filter_by(email=service_account_email).first()
)
if service_account:
bucket_access_groups = get_google_access_groups_for_service_account(
service_account
)
# use configured time or 7 days
expiration_time = int(time.time()) + flask.current_app.config.get(
"GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN", 604800
)
for access_group in bucket_access_groups:
bucket_access = (
session.query(ServiceAccountToGoogleBucketAccessGroup)
.filter_by(
service_account_id=service_account.id,
access_group_id=access_group.id,
)
.first()
)
if not bucket_access:
bucket_access = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_account.id,
access_group_id=access_group.id,
expires=expiration_time,
)
session.add(bucket_access)
bucket_access.expires = expiration_time
session.commit()
def add_user_service_account_to_db(session, to_add_project_ids, service_account):
"""
Add user service account to service account
access privilege and service account bucket access group
Args:
sess(current_session): db session
to_add_project_ids(List(int)): List of project id
service_account(UserServiceAccount): user service account
requested_expires_in(int): requested time (in seconds) during which
the SA has bucket access
Returns:
None
Contrains:
The service account is not in DB yet
"""
for project_id in to_add_project_ids:
session.add(
ServiceAccountAccessPrivilege(
project_id=project_id, service_account_id=service_account.id
)
)
access_groups = _get_google_access_groups(session, project_id)
# timestamp at which the SA will lose bucket access
# by default: use configured time or 7 days
expiration_time = int(time.time()) + config.get(
"GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN", 604800
)
requested_expires_in = (
get_valid_expiration_from_request()
) # requested time (in seconds)
if requested_expires_in:
# convert it to timestamp
requested_expiration = int(time.time()) + requested_expires_in
expiration_time = min(expiration_time, requested_expiration)
for access_group in access_groups:
sa_to_group = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_account.id,
expires=expiration_time,
access_group_id=access_group.id,
)
session.add(sa_to_group)
session.commit()
def add_user_service_account_to_db(session, to_add_project_ids,
service_account):
"""
Add user service account to service account
access privilege and service account bucket access group
Args:
sess(current_session): db session
to_add_project_ids(List(int)): List of project id
service_account(UserServiceAccount): user service account
requested_expires_in(int): requested time (in seconds) during which
the SA has bucket access
Returns:
None
Contrains:
The service account is not in DB yet
"""
for project_id in to_add_project_ids:
session.add(
ServiceAccountAccessPrivilege(
project_id=project_id, service_account_id=service_account.id))
access_groups = _get_google_access_groups(session, project_id)
# time until the SA will lose bucket access
# by default: use configured time or 7 days
default_expires_in = config.get(
"GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN", 604800)
# use expires_in from request query params if it was provided and
# it was not greater than the default
expires_in = get_valid_expiration_from_request(
max_limit=default_expires_in,
default=default_expires_in,
)
# convert expires_in to timestamp
expiration_time = int(time.time() + expires_in)
for access_group in access_groups:
sa_to_group = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_account.id,
expires=expiration_time,
access_group_id=access_group.id,
)
session.add(sa_to_group)
session.commit()
def test_service_account_relationsips(db_session):
"""
test service account tables have proper relationships/fields
"""
project = Project(id=1)
bucket = Bucket(id=1)
user_sa = UserServiceAccount(
id=1,
google_unique_id="guid",
email="*****@*****.**",
google_project_id="gpid",
)
sa_access_privilege = ServiceAccountAccessPrivilege(id=1,
project_id=1,
service_account_id=1)
gbag = GoogleBucketAccessGroup(id=1, bucket_id=1, email="*****@*****.**")
sa_to_gbag = ServiceAccountToGoogleBucketAccessGroup(id=1,
service_account_id=1,
expires=0,
access_group_id=1)
db_session.add(project)
db_session.add(bucket)
db_session.add(user_sa)
db_session.add(sa_access_privilege)
db_session.add(gbag)
db_session.add(sa_to_gbag)
db_session.commit()
assert project.sa_access_privileges[
0].__class__ == ServiceAccountAccessPrivilege
assert project.sa_access_privileges[0].id == 1
assert sa_access_privilege.project.__class__ == Project
assert sa_access_privilege.project.id == 1
assert sa_access_privilege.service_account.__class__ == UserServiceAccount
assert sa_access_privilege.service_account.id == 1
assert user_sa.access_privileges[
0].__class__ == ServiceAccountAccessPrivilege
assert user_sa.access_privileges[0].id == 1
assert (user_sa.to_access_groups[0].__class__ ==
ServiceAccountToGoogleBucketAccessGroup)
assert user_sa.to_access_groups[0].id == 1
assert sa_to_gbag.service_account.__class__ == UserServiceAccount
assert sa_to_gbag.service_account.id == 1
assert sa_to_gbag.access_group.__class__ == GoogleBucketAccessGroup
assert sa_to_gbag.access_group.id == 1
assert gbag.to_access_groups[
0].__class__ == ServiceAccountToGoogleBucketAccessGroup
assert gbag.to_access_groups[0].id == 1
def add_user_service_account_to_db(session, to_add_project_ids, service_account):
"""
Add user service account to service account
access privilege and service account bucket access group
Args:
sess(current_session): db session
to_add_project_ids(List(int)): List of project id
service_account(UserServiceAccount): user service account
Returns:
None
Contrains:
The service account is not in DB yet
"""
for project_id in to_add_project_ids:
session.add(
ServiceAccountAccessPrivilege(
project_id=project_id, service_account_id=service_account.id
)
)
access_groups = _get_google_access_groups(session, project_id)
# use configured time or 7 days
expiration_time = int(time.time()) + flask.current_app.config.get(
"GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN", 604800
)
for access_group in access_groups:
sa_to_group = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_account.id,
expires=expiration_time,
access_group_id=access_group.id,
)
session.add(sa_to_group)
session.commit()
def setup_test_data(db_session):
cp = CloudProvider(name="test", endpoint="http://test.endpt")
proxy_group_list = [
{
"id": "group1",
"email": "*****@*****.**"
},
{
"id": "group2",
"email": "*****@*****.**"
},
]
user_account_list = [
{
"google_unique_id": "test_id1",
"email": "*****@*****.**",
"google_project_id": "test",
},
{
"google_unique_id": "test_id2",
"email": "*****@*****.**",
"google_project_id": "test",
},
]
proxy_groups = []
for group in proxy_group_list:
proxy_groups.append(GoogleProxyGroup(**group))
db_session.add(proxy_groups[-1])
user_service_accounts = []
for user in user_account_list:
user_service_accounts.append(UserServiceAccount(**user))
db_session.add(user_service_accounts[-1])
db_session.commit()
bucket1 = Bucket(name="bucket1", provider_id=cp.id)
bucket2 = Bucket(name="bucket2", provider_id=cp.id)
bucket3 = Bucket(name="bucket3", provider_id=cp.id)
db_session.add(bucket1)
db_session.add(bucket2)
db_session.add(bucket3)
db_session.commit()
access_grp1 = GoogleBucketAccessGroup(bucket_id=bucket1.id,
email="*****@*****.**")
db_session.add(access_grp1)
db_session.commit()
db_session.add(
GoogleProxyGroupToGoogleBucketAccessGroup(
proxy_group_id=proxy_groups[0].id, access_group_id=access_grp1.id))
db_session.add(
ServiceAccountToGoogleBucketAccessGroup(
service_account_id=user_service_accounts[0].id,
access_group_id=access_grp1.id,
))
db_session.commit()
def invalid_service_account_not_exist(db_session):
invalid_service_account = "*****@*****.**"
user = UserServiceAccount(
google_unique_id="invalid_test_id",
email=invalid_service_account,
google_project_id="test",
)
db_session.add(user)
db_session.commit()
cp = db_session.query(CloudProvider).filter_by(name="test").first()
if not cp:
cp = CloudProvider(name="test", endpoint="http://test.endpt")
db_session.add(cp)
db_session.commit()
bucket1 = db_session.query(Bucket).filter_by(name="bucket1").first()
if not bucket1:
bucket1 = Bucket(name="bucket1", provider_id=cp.id)
db_session.add(bucket1)
db_session.commit()
project1 = db_session.query(Project).filter_by(name="test_1").first()
if not project1:
project1 = Project(name="test_1", auth_id="test_auth_1")
db_session.add(project1)
db_session.commit()
access_grp1 = (
db_session.query(GoogleBucketAccessGroup)
.filter_by(email="*****@*****.**")
.first()
)
if not access_grp1:
access_grp1 = GoogleBucketAccessGroup(
bucket_id=bucket1.id, email="*****@*****.**"
)
db_session.add(access_grp1)
db_session.commit()
db_session.add(
ServiceAccountAccessPrivilege(
project_id=project1.id, service_account_id=user.id
)
)
db_session.commit()
# expiration set to 0 for testing that it gets set
current_time = 0
service_account_grp1 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=user.id, access_group_id=access_grp1.id, expires=current_time
)
db_session.add(service_account_grp1)
db_session.commit()
def mock_is_valid(sa_email, *args, **kwargs):
if sa_email == invalid_service_account:
validity = GoogleServiceAccountValidity("account_id", "project_id")
# set overall validity to False
# set policy_accessible to False so the SA is removed from the DB
validity["policy_accessible"] = False
validity._valid = False
return validity
return True
patcher = patch(
"fence.scripting.google_monitor._is_valid_service_account", mock_is_valid
)
patcher.start()
yield {
"service_account": user,
"projects": [project1],
"bucket_access_groups": [access_grp1],
}
patcher.stop()
def register_user_service_account(db_session):
cp = db_session.query(CloudProvider).filter_by(name="test").first()
if not cp:
cp = CloudProvider(name="test", endpoint="http://test.endpt")
db_session.add(cp)
db_session.commit()
bucket1 = db_session.query(Bucket).filter_by(name="bucket1").first()
if not bucket1:
bucket1 = Bucket(name="bucket1", provider_id=cp.id)
db_session.add(bucket1)
db_session.commit()
bucket2 = db_session.query(Bucket).filter_by(name="bucket2").first()
if not bucket2:
bucket2 = Bucket(name="bucket2", provider_id=cp.id)
db_session.add(bucket2)
db_session.commit()
project1 = db_session.query(Project).filter_by(name="test_1").first()
if not project1:
project1 = Project(name="test_1", auth_id="test_auth_1")
db_session.add(project1)
db_session.commit()
project2 = db_session.query(Project).filter_by(name="test_2").first()
if not project2:
project2 = Project(name="test_2", auth_id="test_auth_2")
db_session.add(project2)
db_session.commit()
access_grp1 = (
db_session.query(GoogleBucketAccessGroup)
.filter_by(email="*****@*****.**")
.first()
)
if not access_grp1:
access_grp1 = GoogleBucketAccessGroup(
bucket_id=bucket1.id, email="*****@*****.**"
)
db_session.add(access_grp1)
db_session.commit()
access_grp2 = (
db_session.query(GoogleBucketAccessGroup)
.filter_by(email="*****@*****.**")
.first()
)
if not access_grp2:
access_grp2 = GoogleBucketAccessGroup(
bucket_id=bucket2.id, email="*****@*****.**"
)
db_session.add(access_grp2)
db_session.commit()
project_to_bucket1 = (
db_session.query(ProjectToBucket).filter_by(project_id=project1.id).first()
)
if not project_to_bucket1:
project_to_bucket1 = ProjectToBucket(
project_id=project1.id, bucket_id=bucket1.id
)
db_session.add(project_to_bucket1)
db_session.commit()
project_to_bucket2 = (
db_session.query(ProjectToBucket).filter_by(project_id=project2.id).first()
)
if not project_to_bucket2:
project_to_bucket2 = ProjectToBucket(
project_id=project2.id, bucket_id=bucket2.id
)
db_session.add(project_to_bucket2)
db_session.commit()
# new service account each time this is called
random_string = "".join(
random.choice(string.ascii_uppercase + string.digits) for _ in range(6)
)
user = UserServiceAccount(
google_unique_id="{}".format(random_string),
email="{}@test.iam.gserviceaccount.com".format(random_string),
google_project_id="test",
)
db_session.add(user)
db_session.commit()
db_session.add(
ServiceAccountAccessPrivilege(
project_id=project1.id, service_account_id=user.id
)
)
db_session.add(
ServiceAccountAccessPrivilege(
project_id=project2.id, service_account_id=user.id
)
)
# expiration set to 0 for testing that it gets set
current_time = 0
service_account_grp1 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=user.id, access_group_id=access_grp1.id, expires=current_time
)
service_account_grp2 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=user.id, access_group_id=access_grp2.id, expires=current_time
)
db_session.add(service_account_grp1)
db_session.add(service_account_grp2)
db_session.commit()
return {
"service_account": user,
"projects": [project1, project2],
"buckets": [bucket1, bucket2],
"bucket_access_groups": [access_grp1, access_grp2],
}
def setup_data(db_session):
cp = CloudProvider(name="test", endpoint="http://test.endpt")
user = UserServiceAccount(
google_unique_id="test_id", email="*****@*****.**", google_project_id="test"
)
user_1 = UserServiceAccount(
google_unique_id="test_id", email="*****@*****.**", google_project_id="test"
)
user_2 = UserServiceAccount(
google_unique_id="test_id", email="*****@*****.**", google_project_id="test"
)
user_3 = UserServiceAccount(
google_unique_id="test_id", email="*****@*****.**", google_project_id="test"
)
db_session.add(user)
db_session.add(user_1)
db_session.add(user_2)
db_session.add(user_3)
db_session.add(cp)
db_session.commit()
bucket = Bucket(name="bucket1", provider_id=cp.id)
bucket2 = Bucket(name="bucket2", provider_id=cp.id)
bucket3 = Bucket(name="bucket3", provider_id=cp.id)
db_session.add(bucket)
db_session.add(bucket2)
db_session.add(bucket3)
db_session.commit()
project1 = Project(name="test_1", auth_id="test_auth_1")
project2 = Project(name="test_2", auth_id="test_auth_2")
project3 = Project(name="test_3", auth_id="test_auth_3")
db_session.add(project1)
db_session.add(project2)
db_session.add(project3)
db_session.commit()
db_session.add(ProjectToBucket(project_id=project1.id, bucket_id=bucket.id))
db_session.add(ProjectToBucket(project_id=project2.id, bucket_id=bucket2.id))
db_session.add(ProjectToBucket(project_id=project3.id, bucket_id=bucket3.id))
db_session.add(
ServiceAccountAccessPrivilege(
project_id=project1.id, service_account_id=user.id
)
)
db_session.add(
ServiceAccountAccessPrivilege(
project_id=project2.id, service_account_id=user.id
)
)
db_session.add(
ServiceAccountAccessPrivilege(
project_id=project1.id, service_account_id=user_1.id
)
)
db_session.add(
ServiceAccountAccessPrivilege(
project_id=project1.id, service_account_id=user_2.id
)
)
db_session.add(
ServiceAccountAccessPrivilege(
project_id=project1.id, service_account_id=user_3.id
)
)
access_grp = GoogleBucketAccessGroup(
bucket_id=bucket.id, email="*****@*****.**"
)
access_grp2 = GoogleBucketAccessGroup(
bucket_id=bucket2.id, email="*****@*****.**"
)
access_grp3 = GoogleBucketAccessGroup(
bucket_id=bucket3.id, email="*****@*****.**"
)
db_session.add(access_grp)
db_session.add(access_grp2)
db_session.add(access_grp3)
db_session.commit()
service_account_grp1 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=user.id, access_group_id=access_grp.id
)
service_account_grp2 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=user.id, access_group_id=access_grp2.id
)
db_session.add(service_account_grp1)
db_session.add(service_account_grp2)
db_session.commit()