def cleanup_unassociated_projects():
"""Clean up unassociated projects.
This looks for projects that have no children entries on the dependent
tables and removes them.
"""
LOG.debug("Cleaning up unassociated projects")
session = repo.get_session()
project_children_tables = [models.Order,
models.KEKDatum,
models.Secret,
models.ContainerConsumerMetadatum,
models.Container,
models.PreferredCertificateAuthority,
models.CertificateAuthority,
models.ProjectCertificateAuthority,
models.ProjectQuotas]
children_names = map(lambda child: child.__name__, project_children_tables)
LOG.debug("Children tables for Project table being checked: %s",
str(children_names))
sub_query = session.query(models.Project.id)
for model in project_children_tables:
sub_query = sub_query.outerjoin(model,
models.Project.id == model.project_id)
sub_query = sub_query.filter(model.id == None) # nopep8
sub_query = sub_query.subquery()
sub_query = sa_sql.select([sub_query])
query = session.query(models.Project)
query = query.filter(models.Project.id.in_(sub_query))
delete_count = query.delete(synchronize_session='fetch')
LOG.info("Cleaned up %(delete_count)s entries for "
"%(project_name)s",
{'delete_count': str(delete_count),
'project_name': models.Project.__name__})
return delete_count
def _entry_is_soft_deleted(entry):
model = entry.__class__
entry_id = entry.id
session = repos.get_session()
query = session.query(model)
result = query.filter(model.id == entry_id).first().deleted
return result
def cleanup_parent_with_no_child(parent_model, child_model,
threshold_date=None):
"""Clean up soft deletions in parent that do not have references in child.
Before running this function, the child table should be cleaned of
soft deletions. This function left outer joins the parent and child
tables and finds the parent entries that do not have a foreign key
reference in the child table. Then the results are filtered by soft
deletions and are cleaned up.
:param parent_model: table class for parent
:param child_model: table class for child which restricts parent deletion
:param threshold_date: soft deletions older than this date will be removed
:returns: total number of entries removed from database
"""
LOG.debug("Cleaning soft deletes for %s without a child in %s",
parent_model.__name__,
child_model.__name__)
session = repo.get_session()
sub_query = session.query(parent_model.id)
sub_query = sub_query.outerjoin(child_model)
sub_query = sub_query.filter(child_model.id == None) # nopep8
sub_query = sub_query.subquery()
sub_query = sa_sql.select([sub_query])
query = session.query(parent_model)
query = query.filter(parent_model.id.in_(sub_query))
query = query.filter(parent_model.deleted)
if threshold_date:
query = query.filter(parent_model.deleted_at <= threshold_date)
delete_count = query.delete(synchronize_session='fetch')
LOG.info("Cleaned up %s entries for %s with no children in %s",
delete_count, parent_model.__name__, child_model.__name__)
return delete_count
def _hard_delete_acls_for_soft_deleted_secrets():
"""Remove acl entries for secrets that have been soft deleted.
Removes entries in SecretACL and SecretACLUser which are for secrets
that have been soft deleted.
"""
session = repo.get_session()
acl_user_sub_query = session.query(models.SecretACLUser.id)
acl_user_sub_query = acl_user_sub_query.join(models.SecretACL)
acl_user_sub_query = acl_user_sub_query.join(models.Secret)
acl_user_sub_query = acl_user_sub_query.filter(models.Secret.deleted)
acl_user_sub_query = acl_user_sub_query.subquery()
acl_user_sub_query = sa_sql.select([acl_user_sub_query])
acl_user_query = session.query(models.SecretACLUser)
acl_user_query = acl_user_query.filter(
models.SecretACLUser.id.in_(acl_user_sub_query))
acl_total = acl_user_query.delete(synchronize_session='fetch')
acl_sub_query = session.query(models.SecretACL.id)
acl_sub_query = acl_sub_query.join(models.Secret)
acl_sub_query = acl_sub_query.filter(models.Secret.deleted)
acl_sub_query = acl_sub_query.subquery()
acl_sub_query = sa_sql.select([acl_sub_query])
acl_query = session.query(models.SecretACL)
acl_query = acl_query.filter(
models.SecretACL.id.in_(acl_sub_query))
acl_total += acl_query.delete(synchronize_session='fetch')
return acl_total
def _entry_exists(entry):
"""Check to see if entry should exist in the database"""
model = entry.__class__
entry_id = entry.id
session = repos.get_session()
query = session.query(model).filter(model.id == entry_id)
count = query.count()
return count >= 1
def _setup_entry(name, *args, **kwargs):
func_name = "create_" + name
if not hasattr(utils, func_name):
raise Exception("Cannot create an entry called %s", name)
func = getattr(utils, func_name)
kwargs['session'] = repos.get_session()
entry = func(*args, **kwargs)
return entry
def cleanup_softdeletes(model, threshold_date=None):
"""Remove soft deletions from a table.
:param model: table class to remove soft deletions
:param threshold_date: soft deletions older than this date will be removed
:returns: total number of entries removed from the database
"""
LOG.debug("Cleaning soft deletes: %s", model.__name__)
session = repo.get_session()
query = session.query(model)
query = query.filter_by(deleted=True)
if threshold_date:
query = query.filter(model.deleted_at <= threshold_date)
delete_count = query.delete()
LOG.info("Cleaned up %s entries for %s", delete_count,
model.__name__)
return delete_count
def _soft_delete_expired_secret_children(threshold_date):
"""Soft delete the children tables of expired secrets.
Soft deletes the children tables and hard deletes the ACL children
tables of the expired secrets.
:param threshold_date: threshold date for secret expiration
:returns: returns a pair for number of soft delete children and deleted
ACLs
"""
current_time = timeutils.utcnow()
secret_children = [models.SecretStoreMetadatum,
models.SecretUserMetadatum,
models.EncryptedDatum,
models.ContainerSecret]
children_names = map(lambda child: child.__name__, secret_children)
LOG.debug("Children tables for Secret table being checked: %s",
str(children_names))
session = repo.get_session()
update_count = 0
for table in secret_children:
# Go through children and soft delete them
sub_query = session.query(table.id)
sub_query = sub_query.join(models.Secret)
sub_query = sub_query.filter(
models.Secret.expiration <= threshold_date
)
sub_query = sub_query.subquery()
sub_query = sa_sql.select([sub_query])
query = session.query(table)
query = query.filter(table.id.in_(sub_query))
current_update_count = query.update(
{
table.deleted: True,
table.deleted_at: current_time
},
synchronize_session='fetch')
update_count += current_update_count
session.flush()
acl_total = _hard_delete_acls_for_soft_deleted_secrets()
return update_count, acl_total
def cleanup_parent_with_no_child(parent_model,
child_model,
threshold_date=None):
"""Clean up soft deletions in parent that do not have references in child.
Before running this function, the child table should be cleaned of
soft deletions. This function left outer joins the parent and child
tables and finds the parent entries that do not have a foreign key
reference in the child table. Then the results are filtered by soft
deletions and are cleaned up.
:param parent_model: table class for parent
:param child_model: table class for child which restricts parent deletion
:param threshold_date: soft deletions older than this date will be removed
:returns: total number of entries removed from database
"""
LOG.debug("Cleaning soft deletes for %(parent_name)s without "
"a child in %(child_name)s" % {
'parent_name': parent_model.__name__,
'child_name': child_model.__name__
})
session = repo.get_session()
sub_query = session.query(parent_model.id)
sub_query = sub_query.outerjoin(child_model)
sub_query = sub_query.filter(child_model.id == None) # nopep8
sub_query = sub_query.subquery()
sub_query = sa_sql.select([sub_query])
query = session.query(parent_model)
query = query.filter(parent_model.id.in_(sub_query))
query = query.filter(parent_model.deleted)
if threshold_date:
query = query.filter(parent_model.deleted_at <= threshold_date)
delete_count = query.delete(synchronize_session='fetch')
LOG.info(
"Cleaned up %(delete_count)s entries for %(parent_name)s "
"with no children in %(child_name)s", {
'delete_count': delete_count,
'parent_name': parent_model.__name__,
'child_name': child_model.__name__
})
return delete_count
def _soft_delete_expired_secrets(threshold_date):
"""Soft delete expired secrets.
:param threshold_date: secrets that have expired past this date
will be soft deleted
:returns: total number of secrets that were soft deleted
"""
current_time = timeutils.utcnow()
session = repo.get_session()
query = session.query(models.Secret)
query = query.filter(~models.Secret.deleted)
query = query.filter(
models.Secret.expiration <= threshold_date
)
update_count = query.update(
{
models.Secret.deleted: True,
models.Secret.deleted_at: current_time
},
synchronize_session='fetch')
return update_count
def _soft_delete_expired_secrets(threshold_date):
"""Soft delete expired secrets.
:param threshold_date: secrets that have expired past this date
will be soft deleted
:returns: total number of secrets that were soft deleted
"""
current_time = timeutils.utcnow()
session = repo.get_session()
query = session.query(models.Secret.id)
query = query.filter(~models.Secret.deleted)
query = query.filter(
models.Secret.expiration <= threshold_date
)
update_count = query.update(
{
models.Secret.deleted: True,
models.Secret.deleted_at: current_time
},
synchronize_session='fetch')
return update_count
def cleanup_unassociated_projects():
"""Clean up unassociated projects.
This looks for projects that have no children entries on the dependent
tables and removes them.
"""
LOG.debug("Cleaning up unassociated projects")
session = repo.get_session()
project_children_tables = [models.Order,
models.KEKDatum,
models.SecretConsumerMetadatum,
models.Secret,
models.ContainerConsumerMetadatum,
models.Container,
models.PreferredCertificateAuthority,
models.CertificateAuthority,
models.ProjectCertificateAuthority,
models.ProjectQuotas]
children_names = map(lambda child: child.__name__, project_children_tables)
LOG.debug("Children tables for Project table being checked: %s",
str(children_names))
sub_query = session.query(models.Project.id)
for model in project_children_tables:
sub_query = sub_query.outerjoin(model,
models.Project.id == model.project_id)
sub_query = sub_query.filter(model.id == None) # noqa
sub_query = sub_query.subquery()
sub_query = sa_sql.select([sub_query])
query = session.query(models.Project)
query = query.filter(models.Project.id.in_(sub_query))
delete_count = query.delete(synchronize_session='fetch')
LOG.info("Cleaned up %(delete_count)s entries for "
"%(project_name)s",
{'delete_count': str(delete_count),
'project_name': models.Project.__name__})
return delete_count
def get_session():
return repositories.get_session()
def get_session(cls, session=None):
return session or repos.get_session()
def get_session():
return repositories.get_session()
def get_session(cls, session=None):
return session or repos.get_session()
def setUp(self):
super(OVOTestCase, self).setUp()
self.session = repos.get_session()
