def test_purge_instance_jobs():
"""
Delete old jobs created before the time limit
Do not delete jobs with state = 'running'
"""
app.config['JOB_MAX_PERIOD_TO_KEEP'] = 1
instance_name, backup_dir = init_test()
create_instance_with_different_dataset_types_and_job_state(instance_name, backup_dir)
instances_resp = api_get('/v0/instances')
assert len(instances_resp) == 1
jobs_resp = api_get('/v0/jobs/{}'.format(instances_resp[0]['name']))
assert len(jobs_resp['jobs']) == 6
folders = set(glob.glob('{}/*'.format(backup_dir)))
assert len(folders) == 6
last_datasets = api_get('/v0/instances/{}/last_datasets'.format(instances_resp[0]['name']))
assert len(last_datasets) == 2
tasks.purge_instance(instances_resp[0]['id'], 1)
folders = set(glob.glob('{}/*'.format(backup_dir)))
assert len(folders) == 4
def purge_instances(nb_to_keep, background=False):
"""
remove old backup directories of all instances for only keeping the data being actually loaded
"""
for instance in models.Instance.query_existing().all():
if background:
tasks.purge_instance.delay(instance.id, nb_to_keep)
else:
tasks.purge_instance(instance.id, nb_to_keep)
def purge_instance(instance_name, nb_to_keep, background=False):
"""
remove old backup directories for one instance for only keeping the data being actually loaded
"""
instance = models.Instance.query_existing().filter_by(name=instance_name).first()
if not instance:
raise Exception("cannot find instance {}".format(instance_name))
if background:
tasks.purge_instance.delay(instance.id, nb_to_keep)
else:
tasks.purge_instance(instance.id, nb_to_keep)
def test_purge_instance_jobs():
"""
Delete old jobs created before the time limit
Do not delete jobs with state = 'running'
Deletes only directories of delete able jobs but not in the table 'job'
Is used by the job 'purge_datasets' planned daily to purge old jobs
"""
app.config['JOB_MAX_PERIOD_TO_KEEP'] = 1
instance_name, backup_dir = init_test()
create_instance_with_different_dataset_types_and_job_state(
instance_name, backup_dir)
instances_resp = api_get('/v0/instances')
assert len(instances_resp) == 1
jobs_resp = api_get('/v0/jobs/{}'.format(instances_resp[0]['name']))
assert len(jobs_resp['jobs']) == 6
folders = set(glob.glob('{}/*'.format(backup_dir)))
assert len(folders) == 6
last_datasets = api_get('/v0/instances/{}/last_datasets'.format(
instances_resp[0]['name']))
assert len(last_datasets) == 2
tasks.purge_instance(instances_resp[0]['id'], 1)
folders = set(glob.glob('{}/*'.format(backup_dir)))
assert len(folders) == 4
# No job is deleted in the table 'job'
jobs_resp = api_get('/v0/jobs/{}'.format(instances_resp[0]['name']))
assert len(jobs_resp['jobs']) == 6
# if we use the task purge_jobs, data related to old jobs also deleted from the table 'job'
tasks.purge_jobs()
jobs_resp = api_get('/v0/jobs/{}'.format(instances_resp[0]['name']))
assert len(jobs_resp['jobs']) == 4