class Command(BaseCleanupCommand):
"""
Django command to delete orphaned records for `model`.
Orphans are `days_before_orphaning` old records without any objects referencing them.
If the argument `simulate=True` is passed in, the command only simulates the action.
Only one filter is currently supported for each configuration specified in CONFIGS,
and relation filters are also not supported. If these are needed, the tests need to be
updated to account for such scenarios.
"""
CONFIGS = {
'company.Contact':
ModelCleanupConfig((DatetimeLessThanCleanupFilter(
'modified_on', ORPHAN_AGE_THRESHOLD), ), ),
'company.Company':
ModelCleanupConfig(
(DatetimeLessThanCleanupFilter('modified_on',
ORPHAN_AGE_THRESHOLD), ),
# We want to delete the relations below along with any orphaned companies
excluded_relations=(
Company._meta.get_field('dnbmatchingresult'), ),
),
'event.Event':
ModelCleanupConfig((DatetimeLessThanCleanupFilter(
'end_date', relativedelta(months=18)), ), ),
}
class Command(BaseCleanupCommand):
"""
Django command to delete orphaned records for `model`.
Orphans are `days_before_orphaning` old records without any objects referencing them.
If the argument `simulate=True` is passed in, the command only simulates the action.
"""
CONFIGS = {
'company.Contact':
ModelCleanupConfig(ORPHAN_AGE_THRESHOLD),
'company.Company':
ModelCleanupConfig(ORPHAN_AGE_THRESHOLD),
'event.Event':
ModelCleanupConfig(
age_threshold=relativedelta(months=18),
date_field='end_date',
),
}
class Command(BaseCleanupCommand):
"""Command for deleting very old records (as per the data retention policy)."""
help = (
'Irrevocably deletes very old records for a model, using the criteria defined in the '
'DIT Data Hub retention policy. A simulation can be performed using the --simulate '
'argument.')
CONFIGS = {
# TODO: Before adding any more configurations, get_unreferenced_objects_query()
# and BaseCleanupCommand need to be extended to allow filter conditions for related
# models to be given.
#
# (Interactions does not have any dependent models, hence this has
# not been done yet.)
'interaction.Interaction':
ModelCleanupConfig(relativedelta(years=10), 'date'),
}
class Command(BaseCleanupCommand):
"""Deletes all orphaned :class:`datahub.metadata.models.Team` records"""
model_name = 'metadata.Team'
CONFIGS = {'metadata.Team': ModelCleanupConfig([DummyFilter()])}
def add_arguments(self, parser):
"""Override to add the ``list-only`` argument"""
super(Command, self).add_arguments(parser)
parser.add_argument(
'-l',
'--list-only',
action='store_true',
help='List team IDs which will be deleted by the command',
)
def handle(self, *args, list_only, **options):
"""Override to add the ``list-only`` argument"""
if list_only:
for item in self._get_query(Team):
self.stdout.write(str(item.id))
else:
super(Command, self).handle(*args, **options)
class Command(BaseCleanupCommand):
"""Command for deleting very old records (as per the data retention policy)."""
help = (
'Irrevocably deletes very old records for a model, using the criteria defined in the '
'DIT Data Hub retention policy. A simulation can be performed using the --simulate '
'argument.')
# For each configuration, the combination of excluded_relations and the keys of
# relation_filter_mapping should cover all related fields for the model (as
# returned by get_related_fields()). This is to make sure that no relation is
# missed, and there is a test that checks that all relations are covered.
#
# If a field should not be excluded, but should not be filtered, it should be added
# to relation_filter_mapping with an empty list of filters.
CONFIGS = {
# There were multiple large bulk updates of contacts in the legacy system on and just
# before 2013-08-18, and so modified-on dates are not reliable prior to
# COMPANY_MODIFIED_ON_CUT_OFF.
'company.Company':
ModelCleanupConfig(
(
DatetimeLessThanCleanupFilter('created_on',
COMPANY_EXPIRY_PERIOD),
DatetimeLessThanCleanupFilter('modified_on',
COMPANY_MODIFIED_ON_CUT_OFF),
),
relation_filter_mapping={
# Companies are not deleted if they have any related records via these relations.
# Apart from for one_list_core_team_members, we wait for related records to expire
# before we delete the relevant companies
Company._meta.get_field('contacts'): (),
Company._meta.get_field('interactions'): (),
Company._meta.get_field('company_interactions'): (),
Company._meta.get_field('intermediate_investment_projects'):
(),
Company._meta.get_field('investee_projects'): (),
Company._meta.get_field('investor_investment_projects'): (),
Company._meta.get_field('one_list_core_team_members'): (),
Company._meta.get_field('orders'): (),
Company._meta.get_field('subsidiaries'): (),
Company._meta.get_field('transferred_from'): (),
Company._meta.get_field('referrals'): (),
Company._meta.get_field('investor_profiles'):
(DatetimeLessThanCleanupFilter(
'modified_on', INVESTOR_PROFILE_EXPIRY_PERIOD), ),
Company._meta.get_field('opportunities'): (),
},
# We want to delete the relations below along with any expired companies
excluded_relations=(
Company._meta.get_field('company_list_items'),
Company._meta.get_field('export_countries'),
Company._meta.get_field('export_countries_history'),
Company._meta.get_field('pipeline_list_items'),
),
),
# There were multiple large bulk updates of contacts in the legacy system on and just
# before 2014-07-21, and so modified-on dates are not reliable prior to
# CONTACT_MODIFIED_ON_CUT_OFF.
'company.Contact':
ModelCleanupConfig(
(
DatetimeLessThanCleanupFilter('created_on',
CONTACT_EXPIRY_PERIOD),
DatetimeLessThanCleanupFilter('modified_on',
CONTACT_MODIFIED_ON_CUT_OFF),
),
relation_filter_mapping={
# Contacts are not deleted if they have any related interactions, investment
# projects, OMIS orders or OMIS quotes. We wait for those records to expire
# before we delete the related contacts.
Contact._meta.get_field('interactions'): (),
Contact._meta.get_field('investment_projects'): (),
Contact._meta.get_field('orders'): (),
Contact._meta.get_field('referrals'): (),
Contact._meta.get_field('pipeline_items_m2m'): (),
Quote._meta.get_field('accepted_by').remote_field: (),
},
),
'company_referral.CompanyReferral':
ModelCleanupConfig((
DatetimeLessThanCleanupFilter('created_on', COMPANY_EXPIRY_PERIOD),
DatetimeLessThanCleanupFilter('modified_on',
COMPANY_EXPIRY_PERIOD),
), ),
'interaction.Interaction':
ModelCleanupConfig(
(DatetimeLessThanCleanupFilter('date',
INTERACTION_EXPIRY_PERIOD), ),
relation_filter_mapping={
# Interactions are not deleted if they have any related company referrals.
# These must be deleted first (once they've expired).
Interaction._meta.get_field('company_referral'): (),
},
# We want to delete the relations below along with any expired interactions
excluded_relations=(
Interaction._meta.get_field('dit_participants'),
Interaction._meta.get_field('export_countries'),
),
),
# There are no investment projects in the live system with a modified-on date
# before 2013-11-22, because of a bulk event in the legacy system (this was
# probably when data was imported into that system from another legacy system).
#
# Hence, we check various other fields in addition to just modified_on as modified_on is
# not reliable before INVESTMENT_PROJECT_MODIFIED_ON_CUT_OFF.
'investment.InvestmentProject':
ModelCleanupConfig(
(
DatetimeLessThanCleanupFilter(
'modified_on',
INVESTMENT_PROJECT_MODIFIED_ON_CUT_OFF,
),
DatetimeLessThanCleanupFilter(
'created_on', INVESTMENT_PROJECT_EXPIRY_PERIOD),
DatetimeLessThanCleanupFilter(
'actual_land_date',
INVESTMENT_PROJECT_EXPIRY_PERIOD,
include_null=True,
),
DatetimeLessThanCleanupFilter(
'date_abandoned',
INVESTMENT_PROJECT_EXPIRY_PERIOD,
include_null=True,
),
DatetimeLessThanCleanupFilter(
'date_lost',
INVESTMENT_PROJECT_EXPIRY_PERIOD,
include_null=True,
),
),
relation_filter_mapping={
InvestmentProject._meta.get_field('evidence_documents'):
(DatetimeLessThanCleanupFilter(
'modified_on', INVESTMENT_PROJECT_EXPIRY_PERIOD), ),
InvestmentProject._meta.get_field('proposition'):
(DatetimeLessThanCleanupFilter(
'modified_on', INVESTMENT_PROJECT_EXPIRY_PERIOD), ),
# We simply don't delete any records that have any interactions or are
# referred to by another project.
# (Instead, we wait for the referencing objects to expire themselves.)
InvestmentProject._meta.get_field('interactions'): (),
# The related_name for this field is '+', so we reference the field indirectly
InvestmentProject._meta.get_field(
'associated_non_fdi_r_and_d_project', ).remote_field: (),
InvestmentProject._meta.get_field('opportunities'): (),
},
# These relations do not have any datetime fields to check – we just want them to be
# deleted along with expired records.
excluded_relations=(
InvestmentProject._meta.get_field('team_members'),
InvestmentProject._meta.get_field('stage_log'),
InvestmentProject._meta.get_field('investmentprojectcode'),
InvestmentProject._meta.get_field('activities'),
InvestmentProject._meta.get_field('notifications'),
InvestmentProject._meta.get_field(
'no_recent_investment_interaction_reminders'),
InvestmentProject._meta.get_field(
'upcoming_estimated_land_date_reminders'),
),
),
# There are no orders in the live system with a modified-on date before
# 2014-07-11, because of a bulk event in the legacy system (this was when
# data was imported into that system from another legacy system).
#
# Hence, we check various other fields in addition to just modified_on as modified_on is
# not reliable before ORDER_MODIFIED_ON_CUT_OFF.
'order.Order':
ModelCleanupConfig(
(
DatetimeLessThanCleanupFilter('modified_on',
ORDER_MODIFIED_ON_CUT_OFF),
DatetimeLessThanCleanupFilter('created_on',
ORDER_EXPIRY_PERIOD),
DatetimeLessThanCleanupFilter(
'completed_on',
ORDER_EXPIRY_PERIOD,
include_null=True,
),
DatetimeLessThanCleanupFilter(
'cancelled_on',
ORDER_EXPIRY_PERIOD,
include_null=True,
),
),
relation_filter_mapping={
Order._meta.get_field('refunds'): (
DatetimeLessThanCleanupFilter('modified_on',
ORDER_MODIFIED_ON_CUT_OFF),
DatetimeLessThanCleanupFilter('created_on',
ORDER_EXPIRY_PERIOD),
DatetimeLessThanCleanupFilter(
'level2_approved_on',
ORDER_EXPIRY_PERIOD,
include_null=True,
),
),
Order._meta.get_field('payments'): (
DatetimeLessThanCleanupFilter('modified_on',
ORDER_MODIFIED_ON_CUT_OFF),
DatetimeLessThanCleanupFilter('created_on',
ORDER_EXPIRY_PERIOD),
# received_on is non-null
DatetimeLessThanCleanupFilter('received_on',
ORDER_EXPIRY_PERIOD),
),
Order._meta.get_field('payment_gateway_sessions'):
(DatetimeLessThanCleanupFilter('modified_on',
ORDER_EXPIRY_PERIOD), ),
},
# These relations do not have any datetime fields to check – we just want them to be
# deleted along with expired records.
excluded_relations=(
Order._meta.get_field('assignees'),
Order._meta.get_field('subscribers'),
),
),
}