def create_providers(data, db_session):
s = db_session
providers = data["providers"]
for provider in providers:
prov = CloudProvider()
prov.name = provider["name"]
prov.backend = provider["backend"]
prov.service = provider["service"]
s.add(prov)
s.flush
for name, user in list(data["users"].items()):
new_user = User()
new_user.username = name
new_user.email = user["email"]
new_user.is_admin = user["is_admin"]
s.add(new_user)
user["id"] = new_user.id
for project in data["projects"]:
new_project = Project()
new_project.name = project["name"]
s.add(new_project)
for storage in project["storage_access"]:
provider = s.query(CloudProvider).filter_by(name=storage).first()
if provider:
new_storage_access = StorageAccess(provider_id=provider.id,
project_id=new_project.id)
s.add(new_storage_access)
for bucket in project["buckets"]:
new_bucket = Bucket()
new_bucket.name = bucket["name"]
provider = s.query(CloudProvider).filter_by(
name=bucket["provider"]).first()
new_bucket.provider_id = provider.id
s.add(new_bucket)
s.flush()
project_to_bucket = ProjectToBucket()
project_to_bucket.bucket_id = new_bucket.id
project_to_bucket.project_id = new_project.id
s.add(project_to_bucket)
s.flush()
for user in project["users"]:
access = AccessPrivilege()
access.user_id = data["users"][user["name"]]["id"]
access.project_id = new_project.id
s.add(access)
def create_provider(
current_session,
provider_name,
backend=None,
service=None,
endpoint=None,
description=None,
):
"""
Create a new provider on the table
"""
check = (current_session.query(CloudProvider).filter(
CloudProvider.name == provider_name).first())
if check:
msg = (
"provider name {} already in use; please choose a different name"
" and try again").format(provider_name)
raise UserError(msg)
provider = CloudProvider(
name=provider_name,
backend=backend,
service=service,
endpoint=endpoint,
description=description,
)
current_session.add(provider)
msg = {"result": "success"}
return msg
def test_create_projects(db_session):
# setup
project_1_id = "123"
project_1_name = "my-project-1"
project_2_id = "456"
project_2_name = "my-project-2"
provider_id = "789"
bucket_name = "my-bucket-2"
cp = CloudProvider(
id=provider_id,
name=provider_id,
endpoint="https://test.com",
backend="test_backend",
description="description",
service="service",
)
db_session.add(cp)
# only pre-create project 1
p = Project(id=project_1_id, name=project_1_name)
db_session.add(p)
# only pre-create a StorageAccess for project 1
sa = StorageAccess(project_id=project_1_id, provider_id=provider_id)
db_session.add(sa)
# only pre-create a Bucket for project 2
b = Bucket(name=bucket_name, provider_id=provider_id)
db_session.add(b)
# test "fence-create create" projects creation
data = {
"projects": [
{
"id": project_1_id,
"auth_id": "phs-project-1",
"name": project_1_name,
"storage_accesses": [{"name": provider_id, "buckets": ["my-bucket-1"]}],
},
{
"id": project_2_id,
"auth_id": "phs-project-2",
"name": project_2_name,
"storage_accesses": [{"name": provider_id, "buckets": [bucket_name]}],
},
]
}
create_projects(db_session, data)
projects_in_db = db_session.query(Project).all()
assert projects_in_db, "no projects were created"
assert len(projects_in_db) == len(data["projects"])
project_names = {p.name for p in projects_in_db}
assert project_1_name in project_names
assert project_2_name in project_names
def _setup_service_account_to_google_bucket_access_group(db_session):
"""
Setup some testing data.
"""
cloud_provider = CloudProvider(
name="test_provider",
endpoint="https://test.com",
backend="test_backend",
description="description",
service="service",
)
db_session.add(cloud_provider)
db_session.add(
UserServiceAccount(
google_unique_id="test_id1",
email="*****@*****.**",
google_project_id="efewf444",
)
)
db_session.add(
UserServiceAccount(
google_unique_id="test_id2",
email="*****@*****.**",
google_project_id="edfwf444",
)
)
db_session.commit()
bucket1 = Bucket(name="test_bucket1", provider_id=cloud_provider.id)
db_session.add(bucket1)
db_session.commit()
db_session.add(
GoogleBucketAccessGroup(
bucket_id=bucket1.id,
email="*****@*****.**",
privileges=["read-storage", "write-storage"],
)
)
db_session.add(
GoogleBucketAccessGroup(
bucket_id=bucket1.id,
email="*****@*****.**",
privileges=["read-storage"],
)
)
db_session.commit()
def setup_test_data(db_session):
cp = CloudProvider(name="test", endpoint="http://test.endpt")
proxy_group_list = [
{
"id": "group1",
"email": "*****@*****.**"
},
{
"id": "group2",
"email": "*****@*****.**"
},
]
user_account_list = [
{
"google_unique_id": "test_id1",
"email": "*****@*****.**",
"google_project_id": "test",
},
{
"google_unique_id": "test_id2",
"email": "*****@*****.**",
"google_project_id": "test",
},
]
proxy_groups = []
for group in proxy_group_list:
proxy_groups.append(GoogleProxyGroup(**group))
db_session.add(proxy_groups[-1])
user_service_accounts = []
for user in user_account_list:
user_service_accounts.append(UserServiceAccount(**user))
db_session.add(user_service_accounts[-1])
db_session.commit()
bucket1 = Bucket(name="bucket1", provider_id=cp.id)
bucket2 = Bucket(name="bucket2", provider_id=cp.id)
bucket3 = Bucket(name="bucket3", provider_id=cp.id)
db_session.add(bucket1)
db_session.add(bucket2)
db_session.add(bucket3)
db_session.commit()
access_grp1 = GoogleBucketAccessGroup(bucket_id=bucket1.id,
email="*****@*****.**")
db_session.add(access_grp1)
db_session.commit()
db_session.add(
GoogleProxyGroupToGoogleBucketAccessGroup(
proxy_group_id=proxy_groups[0].id, access_group_id=access_grp1.id))
db_session.add(
ServiceAccountToGoogleBucketAccessGroup(
service_account_id=user_service_accounts[0].id,
access_group_id=access_grp1.id,
))
db_session.commit()
def invalid_service_account_not_exist(db_session):
invalid_service_account = "*****@*****.**"
user = UserServiceAccount(
google_unique_id="invalid_test_id",
email=invalid_service_account,
google_project_id="test",
)
db_session.add(user)
db_session.commit()
cp = db_session.query(CloudProvider).filter_by(name="test").first()
if not cp:
cp = CloudProvider(name="test", endpoint="http://test.endpt")
db_session.add(cp)
db_session.commit()
bucket1 = db_session.query(Bucket).filter_by(name="bucket1").first()
if not bucket1:
bucket1 = Bucket(name="bucket1", provider_id=cp.id)
db_session.add(bucket1)
db_session.commit()
project1 = db_session.query(Project).filter_by(name="test_1").first()
if not project1:
project1 = Project(name="test_1", auth_id="test_auth_1")
db_session.add(project1)
db_session.commit()
access_grp1 = (
db_session.query(GoogleBucketAccessGroup)
.filter_by(email="*****@*****.**")
.first()
)
if not access_grp1:
access_grp1 = GoogleBucketAccessGroup(
bucket_id=bucket1.id, email="*****@*****.**"
)
db_session.add(access_grp1)
db_session.commit()
db_session.add(
ServiceAccountAccessPrivilege(
project_id=project1.id, service_account_id=user.id
)
)
db_session.commit()
# expiration set to 0 for testing that it gets set
current_time = 0
service_account_grp1 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=user.id, access_group_id=access_grp1.id, expires=current_time
)
db_session.add(service_account_grp1)
db_session.commit()
def mock_is_valid(sa_email, *args, **kwargs):
if sa_email == invalid_service_account:
validity = GoogleServiceAccountValidity("account_id", "project_id")
# set overall validity to False
# set policy_accessible to False so the SA is removed from the DB
validity["policy_accessible"] = False
validity._valid = False
return validity
return True
patcher = patch(
"fence.scripting.google_monitor._is_valid_service_account", mock_is_valid
)
patcher.start()
yield {
"service_account": user,
"projects": [project1],
"bucket_access_groups": [access_grp1],
}
patcher.stop()
def register_user_service_account(db_session):
cp = db_session.query(CloudProvider).filter_by(name="test").first()
if not cp:
cp = CloudProvider(name="test", endpoint="http://test.endpt")
db_session.add(cp)
db_session.commit()
bucket1 = db_session.query(Bucket).filter_by(name="bucket1").first()
if not bucket1:
bucket1 = Bucket(name="bucket1", provider_id=cp.id)
db_session.add(bucket1)
db_session.commit()
bucket2 = db_session.query(Bucket).filter_by(name="bucket2").first()
if not bucket2:
bucket2 = Bucket(name="bucket2", provider_id=cp.id)
db_session.add(bucket2)
db_session.commit()
project1 = db_session.query(Project).filter_by(name="test_1").first()
if not project1:
project1 = Project(name="test_1", auth_id="test_auth_1")
db_session.add(project1)
db_session.commit()
project2 = db_session.query(Project).filter_by(name="test_2").first()
if not project2:
project2 = Project(name="test_2", auth_id="test_auth_2")
db_session.add(project2)
db_session.commit()
access_grp1 = (
db_session.query(GoogleBucketAccessGroup)
.filter_by(email="*****@*****.**")
.first()
)
if not access_grp1:
access_grp1 = GoogleBucketAccessGroup(
bucket_id=bucket1.id, email="*****@*****.**"
)
db_session.add(access_grp1)
db_session.commit()
access_grp2 = (
db_session.query(GoogleBucketAccessGroup)
.filter_by(email="*****@*****.**")
.first()
)
if not access_grp2:
access_grp2 = GoogleBucketAccessGroup(
bucket_id=bucket2.id, email="*****@*****.**"
)
db_session.add(access_grp2)
db_session.commit()
project_to_bucket1 = (
db_session.query(ProjectToBucket).filter_by(project_id=project1.id).first()
)
if not project_to_bucket1:
project_to_bucket1 = ProjectToBucket(
project_id=project1.id, bucket_id=bucket1.id
)
db_session.add(project_to_bucket1)
db_session.commit()
project_to_bucket2 = (
db_session.query(ProjectToBucket).filter_by(project_id=project2.id).first()
)
if not project_to_bucket2:
project_to_bucket2 = ProjectToBucket(
project_id=project2.id, bucket_id=bucket2.id
)
db_session.add(project_to_bucket2)
db_session.commit()
# new service account each time this is called
random_string = "".join(
random.choice(string.ascii_uppercase + string.digits) for _ in range(6)
)
user = UserServiceAccount(
google_unique_id="{}".format(random_string),
email="{}@test.iam.gserviceaccount.com".format(random_string),
google_project_id="test",
)
db_session.add(user)
db_session.commit()
db_session.add(
ServiceAccountAccessPrivilege(
project_id=project1.id, service_account_id=user.id
)
)
db_session.add(
ServiceAccountAccessPrivilege(
project_id=project2.id, service_account_id=user.id
)
)
# expiration set to 0 for testing that it gets set
current_time = 0
service_account_grp1 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=user.id, access_group_id=access_grp1.id, expires=current_time
)
service_account_grp2 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=user.id, access_group_id=access_grp2.id, expires=current_time
)
db_session.add(service_account_grp1)
db_session.add(service_account_grp2)
db_session.commit()
return {
"service_account": user,
"projects": [project1, project2],
"buckets": [bucket1, bucket2],
"bucket_access_groups": [access_grp1, access_grp2],
}
def setup_data(db_session):
cp = CloudProvider(name="test", endpoint="http://test.endpt")
user = UserServiceAccount(
google_unique_id="test_id", email="*****@*****.**", google_project_id="test"
)
user_1 = UserServiceAccount(
google_unique_id="test_id", email="*****@*****.**", google_project_id="test"
)
user_2 = UserServiceAccount(
google_unique_id="test_id", email="*****@*****.**", google_project_id="test"
)
user_3 = UserServiceAccount(
google_unique_id="test_id", email="*****@*****.**", google_project_id="test"
)
db_session.add(user)
db_session.add(user_1)
db_session.add(user_2)
db_session.add(user_3)
db_session.add(cp)
db_session.commit()
bucket = Bucket(name="bucket1", provider_id=cp.id)
bucket2 = Bucket(name="bucket2", provider_id=cp.id)
bucket3 = Bucket(name="bucket3", provider_id=cp.id)
db_session.add(bucket)
db_session.add(bucket2)
db_session.add(bucket3)
db_session.commit()
project1 = Project(name="test_1", auth_id="test_auth_1")
project2 = Project(name="test_2", auth_id="test_auth_2")
project3 = Project(name="test_3", auth_id="test_auth_3")
db_session.add(project1)
db_session.add(project2)
db_session.add(project3)
db_session.commit()
db_session.add(ProjectToBucket(project_id=project1.id, bucket_id=bucket.id))
db_session.add(ProjectToBucket(project_id=project2.id, bucket_id=bucket2.id))
db_session.add(ProjectToBucket(project_id=project3.id, bucket_id=bucket3.id))
db_session.add(
ServiceAccountAccessPrivilege(
project_id=project1.id, service_account_id=user.id
)
)
db_session.add(
ServiceAccountAccessPrivilege(
project_id=project2.id, service_account_id=user.id
)
)
db_session.add(
ServiceAccountAccessPrivilege(
project_id=project1.id, service_account_id=user_1.id
)
)
db_session.add(
ServiceAccountAccessPrivilege(
project_id=project1.id, service_account_id=user_2.id
)
)
db_session.add(
ServiceAccountAccessPrivilege(
project_id=project1.id, service_account_id=user_3.id
)
)
access_grp = GoogleBucketAccessGroup(
bucket_id=bucket.id, email="*****@*****.**"
)
access_grp2 = GoogleBucketAccessGroup(
bucket_id=bucket2.id, email="*****@*****.**"
)
access_grp3 = GoogleBucketAccessGroup(
bucket_id=bucket3.id, email="*****@*****.**"
)
db_session.add(access_grp)
db_session.add(access_grp2)
db_session.add(access_grp3)
db_session.commit()
service_account_grp1 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=user.id, access_group_id=access_grp.id
)
service_account_grp2 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=user.id, access_group_id=access_grp2.id
)
db_session.add(service_account_grp1)
db_session.add(service_account_grp2)
db_session.commit()
def _setup_google_access(db_session,
access_1_expires=None,
access_2_expires=None):
"""
Setup some testing data.
Args:
access_1_expires (str, optional): expiration for the Proxy Group ->
Google Bucket Access Group for user 1, defaults to None
access_2_expires (str, optional): expiration for the Proxy Group ->
Google Bucket Access Group for user 2, defaults to None
"""
cloud_provider = CloudProvider(
name="test_provider",
endpoint="https://test.com",
backend="test_backend",
description="description",
service="service",
)
db_session.add(cloud_provider)
db_session.add(
UserServiceAccount(
google_unique_id="test_id1",
email="*****@*****.**",
google_project_id="efewf444",
))
db_session.add(
UserServiceAccount(
google_unique_id="test_id2",
email="*****@*****.**",
google_project_id="edfwf444",
))
db_session.commit()
bucket1 = Bucket(name="test_bucket1", provider_id=cloud_provider.id)
db_session.add(bucket1)
db_session.commit()
gpg1 = GoogleProxyGroup(id=1, email="*****@*****.**")
gpg2 = GoogleProxyGroup(id=2, email="*****@*****.**")
db_session.add(gpg1)
db_session.add(gpg2)
db_session.commit()
gbag1 = GoogleBucketAccessGroup(
bucket_id=bucket1.id,
email="*****@*****.**",
privileges=["read-storage", "write-storage"],
)
gbag2 = GoogleBucketAccessGroup(
bucket_id=bucket1.id,
email="*****@*****.**",
privileges=["read-storage"],
)
db_session.add(gbag1)
db_session.add(gbag2)
db_session.commit()
db_session.add(
GoogleProxyGroupToGoogleBucketAccessGroup(proxy_group_id=gpg1.id,
access_group_id=gbag1.id,
expires=access_1_expires))
db_session.add(
GoogleProxyGroupToGoogleBucketAccessGroup(proxy_group_id=gpg2.id,
access_group_id=gbag2.id,
expires=access_2_expires))
db_session.commit()
return {"google_proxy_group_ids": {"1": gpg1.id, "2": gpg2.id}}
