def test_purge_old_jobs_same_dataset():
"""
An old job to be deleted uses the same dataset as one to keep.
So, delete the job keeping at least one in db but the dataset file on disc
"""
app.config['JOB_MAX_PERIOD_TO_KEEP'] = 1
instance_name, backup_dir = init_test()
create_jobs_with_same_datasets(instance_name, backup_dir)
instances_resp = api_get('/v0/instances')
assert len(instances_resp) == 1
jobs_resp = api_get('/v0/jobs/{}'.format(instances_resp[0]['name']))
assert len(jobs_resp['jobs']) == 3
folders = set(glob.glob('{}/*'.format(backup_dir)))
assert len(folders) == 1
tasks.purge_jobs()
jobs_resp = api_get('/v0/jobs/{}'.format(instances_resp[0]['name']))
assert len(jobs_resp['jobs']) == 1
folders = set(glob.glob('{}/*'.format(backup_dir)))
assert len(folders) == 1
def test_purge_old_jobs_of_multi_instance():
"""
We should delete jobs, data_sets as well as metrics of all instances
including discarded = True
"""
app.config['JOB_MAX_PERIOD_TO_KEEP'] = 1
instance_name, backup_dir = init_test()
create_jobs_with_same_datasets(instance_name, backup_dir)
create_jobs_with_same_datasets('fr-discarded', backup_dir)
instances_resp = api_get('/v0/instances')
assert len(instances_resp) == 2
# Delete the second instance (discarded = True)
instances_resp = api_delete('/v0/instances/fr-discarded')
assert instances_resp['discarded'] is True
jobs_resp = api_get('/v0/jobs/fr-discarded')
assert len(jobs_resp['jobs']) == 3
jobs_resp = api_get('/v0/jobs/fr')
assert len(jobs_resp['jobs']) == 3
tasks.purge_jobs()
jobs_resp = api_get('/v0/jobs/fr-discarded')
assert len(jobs_resp['jobs']) == 1
jobs_resp = api_get('/v0/jobs/fr')
assert len(jobs_resp['jobs']) == 1
def test_purge_old_jobs_no_delete():
"""
The jobs related to the instance are the only ones of their type(having one data_set par job).
So, even if they are older than the time limit, they won't be deleted and hence last_datasets
always contains a full set of valid data_sets
"""
app.config['JOB_MAX_PERIOD_TO_KEEP'] = 1
instance_name, backup_dir = init_test()
create_instance_with_one_type_dataset(instance_name, backup_dir)
instances_resp = api_get('/v0/instances')
assert len(instances_resp) == 1
jobs_resp = api_get('/v0/jobs/{}'.format(instances_resp[0]['name']))
assert len(jobs_resp['jobs']) == 3
folders = set(glob.glob('{}/*'.format(backup_dir)))
assert len(folders) == 3
tasks.purge_jobs()
jobs_resp = api_get('/v0/jobs/{}'.format(instances_resp[0]['name']))
assert len(jobs_resp['jobs']) == 3
folders = set(glob.glob('{}/*'.format(backup_dir)))
assert len(folders) == 3
def test_purge_old_jobs():
"""
Delete old jobs created before the time limit
"""
app.config['JOB_MAX_PERIOD_TO_KEEP'] = 1
instance_name, backup_dir = init_test()
create_instance_with_same_type_datasets(instance_name, backup_dir)
instances_resp = api_get('/v0/instances')
assert len(instances_resp) == 1
jobs_resp = api_get('/v0/jobs/{}'.format(instances_resp[0]['name']))
assert len(jobs_resp['jobs']) == 3
folders = set(glob.glob('{}/*'.format(backup_dir)))
assert len(folders) == 3
tasks.purge_jobs()
jobs_resp = api_get('/v0/jobs/{}'.format(instances_resp[0]['name']))
assert len(jobs_resp['jobs']) == 1
folders = set(glob.glob('{}/*'.format(backup_dir)))
assert len(folders) == 1
def test_purge_old_jobs_with_diff_job_states():
"""
we shouldn't delete jobs with status 'running'. This is to avoid
deleting jobs and their files on running state.
"""
app.config['JOB_MAX_PERIOD_TO_KEEP'] = 1
instance_name, backup_dir = init_test()
create_jobs_with_same_datasets(instance_name, backup_dir)
instances_resp = api_get('/v0/instances')
assert len(instances_resp) == 1
jobs_resp = api_get('/v0/jobs/{}'.format(instances_resp[0]['name']))
assert len(jobs_resp['jobs']) == 3
last_datasets = api_get('/v0/instances/{}/last_datasets'.format(instances_resp[0]['name']))
assert len(last_datasets) == 1
# Purge old jobs
tasks.purge_jobs()
jobs_resp = api_get('/v0/jobs/{}'.format(instances_resp[0]['name']))
assert len(jobs_resp['jobs']) == 1
# Add three more jobs with state = 'running'. Purge shouldn't delete these jobs
create_jobs_with_same_datasets('fr-npdc', backup_dir, 'running')
jobs_resp = api_get('/v0/jobs')
assert len(jobs_resp['jobs']) == 4
tasks.purge_jobs()
jobs_resp = api_get('/v0/jobs')
assert len(jobs_resp['jobs']) == 4
def purge_old_jobs(days_to_keep=None, background=False):
"""
Delete old jobs in database and backup folders associated
"""
if background:
tasks.purge_jobs.delay(days_to_keep)
else:
tasks.purge_jobs(days_to_keep)
def test_purge_instance_jobs():
"""
Delete old jobs created before the time limit
Do not delete jobs with state = 'running'
Deletes only directories of delete able jobs but not in the table 'job'
Is used by the job 'purge_datasets' planned daily to purge old jobs
"""
app.config['JOB_MAX_PERIOD_TO_KEEP'] = 1
instance_name, backup_dir = init_test()
create_instance_with_different_dataset_types_and_job_state(
instance_name, backup_dir)
instances_resp = api_get('/v0/instances')
assert len(instances_resp) == 1
jobs_resp = api_get('/v0/jobs/{}'.format(instances_resp[0]['name']))
assert len(jobs_resp['jobs']) == 6
folders = set(glob.glob('{}/*'.format(backup_dir)))
assert len(folders) == 6
last_datasets = api_get('/v0/instances/{}/last_datasets'.format(
instances_resp[0]['name']))
assert len(last_datasets) == 2
tasks.purge_instance(instances_resp[0]['id'], 1)
folders = set(glob.glob('{}/*'.format(backup_dir)))
assert len(folders) == 4
# No job is deleted in the table 'job'
jobs_resp = api_get('/v0/jobs/{}'.format(instances_resp[0]['name']))
assert len(jobs_resp['jobs']) == 6
# if we use the task purge_jobs, data related to old jobs also deleted from the table 'job'
tasks.purge_jobs()
jobs_resp = api_get('/v0/jobs/{}'.format(instances_resp[0]['name']))
assert len(jobs_resp['jobs']) == 4
