def test_remove_same_location(self):
# Create two data objects that reference the same data location and check
# that if one object is deleted, the data is not removed upon purge.
# It should be removed only when location is not referenced by any data object.
same_location_data = Data.objects.create(**self.data)
data_location = DataLocation.objects.create(subpath='')
data_location.subpath = str(data_location.id)
data_location.save()
data_location.data.add(same_location_data)
subpath = same_location_data.location.subpath
same_location_data.output = {'sample': {'file': 'test-file'}}
same_location_data.status = Data.STATUS_DONE
self.create_test_file(same_location_data.location, 'test-file')
same_location_data.save()
same_location_data_2 = Data.objects.create(**self.data)
same_location_data.location.data.add(same_location_data_2)
same_location_data_2.output = {'sample': {'file': 'test-file'}}
same_location_data_2.status = Data.STATUS_DONE
same_location_data_2.save()
not_to_be_deleted = Data.objects.create(**self.data)
data_location = DataLocation.objects.create(subpath='')
data_location.subpath = str(data_location.id)
data_location.save()
data_location.data.add(not_to_be_deleted)
not_to_be_deleted.output = {'sample': {'file': 'test-file'}}
not_to_be_deleted.status = Data.STATUS_DONE
self.create_test_file(not_to_be_deleted.location, 'test-file')
not_to_be_deleted.save()
self.assertEqual(Data.objects.count(), 3)
# Delete first object.
same_location_data.delete()
self.assertEqual(Data.objects.count(), 2)
with patch('resolwe.flow.utils.purge.shutil.rmtree',
wraps=shutil.rmtree) as rmtree_mock:
purge.purge_all(delete=True)
rmtree_mock.assert_not_called()
# Delete second object.
same_location_data_2.delete()
self.assertEqual(Data.objects.count(), 1)
self.assertEqual(Data.objects.first().id, not_to_be_deleted.id)
with patch('resolwe.flow.utils.purge.shutil.rmtree',
wraps=shutil.rmtree) as rmtree_mock:
purge.purge_all(delete=True)
rmtree_mock.assert_called_once_with(
os.path.join(settings.FLOW_EXECUTOR['DATA_DIR'], subpath))
def test_remove_same_location(self):
# Create two data objects that reference the same data location and check
# that if one object is deleted, the data is not removed upon purge.
# It should be removed only when location is not referenced by any data object.
same_location_data = Data.objects.create(**self.data)
data_location = DataLocation.objects.create(subpath='')
data_location.subpath = str(data_location.id)
data_location.save()
data_location.data.add(same_location_data)
subpath = same_location_data.location.subpath
same_location_data.output = {'sample': {'file': 'test-file'}}
same_location_data.status = Data.STATUS_DONE
self.create_test_file(same_location_data.location, 'test-file')
same_location_data.save()
same_location_data_2 = Data.objects.create(**self.data)
same_location_data.location.data.add(same_location_data_2)
same_location_data_2.output = {'sample': {'file': 'test-file'}}
same_location_data_2.status = Data.STATUS_DONE
same_location_data_2.save()
not_to_be_deleted = Data.objects.create(**self.data)
data_location = DataLocation.objects.create(subpath='')
data_location.subpath = str(data_location.id)
data_location.save()
data_location.data.add(not_to_be_deleted)
not_to_be_deleted.output = {'sample': {'file': 'test-file'}}
not_to_be_deleted.status = Data.STATUS_DONE
self.create_test_file(not_to_be_deleted.location, 'test-file')
not_to_be_deleted.save()
self.assertEqual(Data.objects.count(), 3)
# Delete first object.
same_location_data.delete()
self.assertEqual(Data.objects.count(), 2)
with patch('resolwe.flow.utils.purge.shutil.rmtree', wraps=shutil.rmtree) as rmtree_mock:
purge.purge_all(delete=True)
rmtree_mock.assert_not_called()
# Delete second object.
same_location_data_2.delete()
self.assertEqual(Data.objects.count(), 1)
self.assertEqual(Data.objects.first().id, not_to_be_deleted.id)
with patch('resolwe.flow.utils.purge.shutil.rmtree', wraps=shutil.rmtree) as rmtree_mock:
purge.purge_all(delete=True)
rmtree_mock.assert_called_once_with(
os.path.join(settings.FLOW_EXECUTOR['DATA_DIR'], subpath))
def test_remove(self):
completed_data = Data.objects.create(**self.data)
data_location = DataLocation.objects.create(subpath='')
data_location.subpath = str(data_location.id)
data_location.save()
data_location.data.add(completed_data)
completed_data.status = Data.STATUS_DONE
completed_data.output = {'sample': {'file': 'test-file'}}
self.create_test_file(completed_data.location, 'test-file')
self.create_test_file(completed_data.location, 'removeme')
completed_data.save()
pending_data = Data.objects.create(**self.data)
data_location = DataLocation.objects.create(subpath='')
data_location.subpath = str(data_location.id)
data_location.save()
data_location.data.add(pending_data)
self.create_test_file(pending_data.location, 'test-file')
self.create_test_file(pending_data.location, 'donotremoveme')
# Check that nothing is removed if delete is False (the default).
with patch('resolwe.flow.utils.purge.os', wraps=os) as os_mock:
os_mock.remove = MagicMock()
purge.purge_all()
os_mock.remove.assert_not_called()
completed_data.location.purged = False
completed_data.location.save()
# Check that only the 'removeme' file from the completed Data objects is removed
# and files from the second (not completed) Data objects are unchanged.
with patch('resolwe.flow.utils.purge.os', wraps=os) as os_mock:
os_mock.remove = MagicMock()
purge.purge_all(delete=True)
os_mock.remove.assert_called_once_with(
completed_data.location.get_path(filename='removeme'))
completed_data.location.purged = False
completed_data.location.save()
# Create dummy data directories for non-existant data objects.
self.create_test_file(DataLocation.objects.create(subpath='990'),
'dummy')
self.create_test_file(DataLocation.objects.create(subpath='991'),
'dummy')
# Check that only the 'removeme' file from the completed Data objects is removed
# together with directories not belonging to any data objects.
with contextlib.ExitStack() as stack:
os_mock = stack.enter_context(
patch('resolwe.flow.utils.purge.os', wraps=os))
shutil_mock = stack.enter_context(
patch('resolwe.flow.utils.purge.shutil', wraps=shutil))
os_mock.remove = MagicMock()
shutil_mock.rmtree = MagicMock()
purge.purge_all(delete=True)
self.assertEqual(os_mock.remove.call_count, 1)
self.assertEqual(shutil_mock.rmtree.call_count, 2)
os_mock.remove.assert_called_once_with(
os.path.join(settings.FLOW_EXECUTOR['DATA_DIR'],
str(completed_data.location.id), 'removeme'))
shutil_mock.rmtree.assert_any_call(
os.path.join(settings.FLOW_EXECUTOR['DATA_DIR'], '990'))
shutil_mock.rmtree.assert_any_call(
os.path.join(settings.FLOW_EXECUTOR['DATA_DIR'], '991'))
completed_data.location.purged = False
completed_data.location.save()
# Create another data object and check that if remove is called on one object,
# only that object's data is removed.
another_data = Data.objects.create(**self.data)
data_location = DataLocation.objects.create(subpath='')
data_location.subpath = str(data_location.id)
data_location.save()
data_location.data.add(another_data)
another_data.status = Data.STATUS_DONE
another_data.output = {'sample': {'file': 'test-file'}}
self.create_test_file(another_data.location, 'test-file')
self.create_test_file(another_data.location, 'removeme')
another_data.save()
with contextlib.ExitStack() as stack:
os_mock = stack.enter_context(
patch('resolwe.flow.utils.purge.os', wraps=os))
shutil_mock = stack.enter_context(
patch('resolwe.flow.utils.purge.shutil', wraps=shutil))
os_mock.remove = MagicMock()
purge.location_purge(location_id=another_data.location.id,
delete=True)
os_mock.remove.assert_called_once_with(
another_data.location.get_path(filename='removeme'))
shutil_mock.rmtree.assert_not_called()
def test_remove(self):
completed_data = Data.objects.create(**self.data)
data_location = DataLocation.objects.create(subpath='')
data_location.subpath = str(data_location.id)
data_location.save()
data_location.data.add(completed_data)
completed_data.status = Data.STATUS_DONE
completed_data.output = {'sample': {'file': 'test-file'}}
self.create_test_file(completed_data.location, 'test-file')
self.create_test_file(completed_data.location, 'removeme')
completed_data.save()
pending_data = Data.objects.create(**self.data)
data_location = DataLocation.objects.create(subpath='')
data_location.subpath = str(data_location.id)
data_location.save()
data_location.data.add(pending_data)
self.create_test_file(pending_data.location, 'test-file')
self.create_test_file(pending_data.location, 'donotremoveme')
# Check that nothing is removed if delete is False (the default).
with patch('resolwe.flow.utils.purge.os', wraps=os) as os_mock:
os_mock.remove = MagicMock()
purge.purge_all()
os_mock.remove.assert_not_called()
completed_data.location.purged = False
completed_data.location.save()
# Check that only the 'removeme' file from the completed Data objects is removed
# and files from the second (not completed) Data objects are unchanged.
with patch('resolwe.flow.utils.purge.os', wraps=os) as os_mock:
os_mock.remove = MagicMock()
purge.purge_all(delete=True)
os_mock.remove.assert_called_once_with(
completed_data.location.get_path(filename='removeme'))
completed_data.location.purged = False
completed_data.location.save()
# Create dummy data directories for non-existant data objects.
self.create_test_file(DataLocation.objects.create(subpath='990'), 'dummy')
self.create_test_file(DataLocation.objects.create(subpath='991'), 'dummy')
# Check that only the 'removeme' file from the completed Data objects is removed
# together with directories not belonging to any data objects.
with contextlib.ExitStack() as stack:
os_mock = stack.enter_context(patch('resolwe.flow.utils.purge.os', wraps=os))
shutil_mock = stack.enter_context(patch('resolwe.flow.utils.purge.shutil', wraps=shutil))
os_mock.remove = MagicMock()
shutil_mock.rmtree = MagicMock()
purge.purge_all(delete=True)
self.assertEqual(os_mock.remove.call_count, 1)
self.assertEqual(shutil_mock.rmtree.call_count, 2)
os_mock.remove.assert_called_once_with(
os.path.join(settings.FLOW_EXECUTOR['DATA_DIR'], str(completed_data.location.id), 'removeme'))
shutil_mock.rmtree.assert_any_call(
os.path.join(settings.FLOW_EXECUTOR['DATA_DIR'], '990'))
shutil_mock.rmtree.assert_any_call(
os.path.join(settings.FLOW_EXECUTOR['DATA_DIR'], '991'))
completed_data.location.purged = False
completed_data.location.save()
# Create another data object and check that if remove is called on one object,
# only that object's data is removed.
another_data = Data.objects.create(**self.data)
data_location = DataLocation.objects.create(subpath='')
data_location.subpath = str(data_location.id)
data_location.save()
data_location.data.add(another_data)
another_data.status = Data.STATUS_DONE
another_data.output = {'sample': {'file': 'test-file'}}
self.create_test_file(another_data.location, 'test-file')
self.create_test_file(another_data.location, 'removeme')
another_data.save()
with contextlib.ExitStack() as stack:
os_mock = stack.enter_context(patch('resolwe.flow.utils.purge.os', wraps=os))
shutil_mock = stack.enter_context(patch('resolwe.flow.utils.purge.shutil', wraps=shutil))
os_mock.remove = MagicMock()
purge.location_purge(location_id=another_data.location.id, delete=True)
os_mock.remove.assert_called_once_with(
another_data.location.get_path(filename='removeme'))
shutil_mock.rmtree.assert_not_called()
def handle(self, *args, **options):
"""Call :func:`~resolwe.flow.utils.purge.purge_all`."""
purge_all(delete=options["force"], verbosity=options["verbosity"])
def handle(self, *args, **options):
"""Call :func:`~resolwe.flow.utils.purge.purge_all`."""
purge_all(delete=options['force'], verbosity=options['verbosity'])