def create_project(s, project_data):
auth_id = project_data["auth_id"]
name = project_data.get("name", auth_id)
project = s.query(Project).filter_by(name=name).first()
if project is None:
project = Project(name=name, auth_id=auth_id)
s.add(project)
if "storage_accesses" in project_data:
sa_list = project_data["storage_accesses"]
for storage_access in sa_list:
provider = storage_access["name"]
buckets = storage_access.get("buckets", [])
sa = (s.query(StorageAccess).join(
StorageAccess.provider, StorageAccess.project).filter(
Project.name == project.name).filter(
CloudProvider.name == provider).first())
if not sa:
c_provider = s.query(CloudProvider).filter_by(
name=provider).first()
sa = StorageAccess(provider=c_provider, project=project)
s.add(sa)
print("created storage access for {} to {}".format(
project.name, c_provider.name))
for bucket in buckets:
b = (s.query(Bucket).filter_by(name=bucket).join(
Bucket.provider).filter(
CloudProvider.name == provider).first())
print(b)
if not b:
b = Bucket(name=bucket)
b.provider = c_provider
s.add(b)
print("created bucket {} in db".format(bucket))
return project
def link_bucket_to_project(db, bucket_id, bucket_provider, project_auth_id):
"""
Associate a bucket to a specific project (with provided auth_id).
Args:
db (TYPE): database
bucket_id (str): bucket db id or unique name
WARNING: name uniqueness is only required for Google so it's not
a requirement of the db table. You will get an error if
there are multiple buckets with the given name. In that
case, you'll have to use the bucket's id.
bucket_provider (str): CloudProvider.name for the bucket
project_auth_id (str): Project.auth_id to link to bucket
"""
driver = SQLAlchemyDriver(db)
with driver.session as current_session:
cloud_provider = (
current_session.query(CloudProvider).filter_by(name=bucket_provider).first()
)
if not cloud_provider:
raise NameError(
'No bucket with provider "{}" exists.'.format(bucket_provider)
)
# first try by searching using id
try:
bucket_id = int(bucket_id)
bucket_db_entry = (
current_session.query(Bucket).filter_by(
id=bucket_id, provider_id=cloud_provider.id
)
).first()
except ValueError:
# invalid id, must be int
bucket_db_entry = None
# nothing found? try searching for single bucket with name bucket_id
if not bucket_db_entry:
buckets_by_name = current_session.query(Bucket).filter_by(
name=bucket_id, provider_id=cloud_provider.id
)
# don't get a bucket if the name isn't unique. NOTE: for Google,
# these have to be globally unique so they'll be unique here.
buckets_with_name = buckets_by_name.count()
if buckets_with_name == 1:
bucket_db_entry = buckets_by_name[0]
elif buckets_with_name > 1:
raise NameError(
'No bucket with id "{bucket_id}" exists. Tried buckets '
'with name "{bucket_id}", but this returned multiple '
"buckets. Please specify the id from the db and not just "
"the name.".format(bucket_id=bucket_id)
)
else:
# keep bucket_db_entry as None
pass
if not bucket_db_entry:
raise NameError('No bucket with id or name "{}" exists.'.format(bucket_id))
project_db_entry = (
current_session.query(Project).filter_by(auth_id=project_auth_id).first()
)
if not project_db_entry:
logger.warning(
'WARNING: No project with auth_id "{}" exists. Creating...'.format(
project_auth_id
)
)
project_db_entry = Project(name=project_auth_id, auth_id=project_auth_id)
current_session.add(project_db_entry)
current_session.commit()
# Add StorageAccess if it doesn't exist for the project
storage_access = (
current_session.query(StorageAccess)
.filter_by(project_id=project_db_entry.id, provider_id=cloud_provider.id)
.first()
)
if not storage_access:
storage_access = StorageAccess(
project_id=project_db_entry.id, provider_id=cloud_provider.id
)
current_session.add(storage_access)
current_session.commit()
project_linkage = (
current_session.query(ProjectToBucket)
.filter_by(project_id=project_db_entry.id, bucket_id=bucket_db_entry.id)
.first()
)
if not project_linkage:
project_linkage = ProjectToBucket(
project_id=project_db_entry.id,
bucket_id=bucket_db_entry.id,
privilege=["owner"], # TODO What should this be???
)
current_session.add(project_linkage)
current_session.commit()
def _create_or_update_google_bucket_and_db(
db_session,
name,
storage_class,
public,
requester_pays,
storage_creds_project_id,
project_auth_id,
access_logs_bucket,
):
"""
Handles creates the Google bucket and adding necessary db entry
"""
manager = GoogleCloudManager(
storage_creds_project_id, creds=cirrus_config.configs["GOOGLE_STORAGE_CREDS"]
)
with manager as g_mgr:
g_mgr.create_or_update_bucket(
name,
storage_class=storage_class,
public=public,
requester_pays=requester_pays,
access_logs_bucket=access_logs_bucket,
)
# add bucket to db
google_cloud_provider = _get_or_create_google_provider(db_session)
bucket_db_entry = (
db_session.query(Bucket)
.filter_by(name=name, provider_id=google_cloud_provider.id)
.first()
)
if not bucket_db_entry:
bucket_db_entry = Bucket(name=name, provider_id=google_cloud_provider.id)
db_session.add(bucket_db_entry)
db_session.commit()
logger.info("Successfully updated Google Bucket {}.".format(name))
# optionally link this new bucket to an existing project
if project_auth_id:
project_db_entry = (
db_session.query(Project).filter_by(auth_id=project_auth_id).first()
)
if project_db_entry:
project_linkage = (
db_session.query(ProjectToBucket)
.filter_by(
project_id=project_db_entry.id, bucket_id=bucket_db_entry.id
)
.first()
)
if not project_linkage:
project_linkage = ProjectToBucket(
project_id=project_db_entry.id,
bucket_id=bucket_db_entry.id,
privilege=["owner"], # TODO What should this be???
)
db_session.add(project_linkage)
db_session.commit()
logger.info(
"Successfully linked project with auth_id {} "
"to the bucket.".format(project_auth_id)
)
else:
logger.info(
"No project with auth_id {} found. No linking "
"occured.".format(project_auth_id)
)
# Add StorageAccess if it doesn't exist for the project
storage_access = (
db_session.query(StorageAccess)
.filter_by(
project_id=project_db_entry.id, provider_id=google_cloud_provider.id
)
.first()
)
if not storage_access:
storage_access = StorageAccess(
project_id=project_db_entry.id, provider_id=google_cloud_provider.id
)
db_session.add(storage_access)
db_session.commit()
return bucket_db_entry