def storagemanager_run(self, event):
"""Start the manager run."""
manager = Manager()
try:
manager.process()
except Exception:
logger.exception("Error while running manager.")
def test_delete(self):
location_local: StorageLocation = StorageLocation.objects.create(
file_storage=self.file_storage1,
url="url",
connector_name="local",
status=StorageLocation.STATUS_DONE,
)
# Do not delete.
process_copy_mock = MagicMock()
process_delete_mock = MagicMock()
copy = MagicMock(return_value=FileStorage.objects.none())
delete = MagicMock(return_value=FileStorage.objects.none())
with patch.multiple(
"resolwe.storage.models.LocationsDoneManager",
to_delete=delete,
to_copy=copy,
):
with patch.multiple(
"resolwe.storage.manager.Manager",
copy_single_location=process_copy_mock,
delete_single_location=process_delete_mock,
):
self.manager = Manager()
self.manager.process()
self.assertEqual(copy.call_count, 3)
self.assertEqual(delete.call_count, 3)
process_copy_mock.assert_not_called()
process_delete_mock.assert_not_called()
# Delete location_local.
delete_data = MagicMock()
location_local.delete_data = delete_data()
copy = MagicMock(return_value=FileStorage.objects.none())
delete = MagicMock(
side_effect=[
FileStorage.objects.filter(pk=location_local.file_storage.pk),
FileStorage.objects.none(),
FileStorage.objects.none(),
]
)
with patch.multiple(
"resolwe.storage.models.LocationsDoneManager",
to_delete=delete,
to_copy=copy,
):
with patch.multiple(
"resolwe.storage.manager.Manager",
copy_single_location=process_copy_mock,
delete_single_location=process_delete_mock,
):
self.manager = Manager()
self.manager.process()
self.assertEqual(copy.call_count, 3)
process_copy_mock.assert_not_called()
delete_data.assert_called_once_with()
self.assertEqual(process_delete_mock.call_count, 1)
def task_b():
rows_locked.wait()
self.manager = Manager()
self.manager.process()
manager_finished.set()
connection.close()
def setUp(self):
self.file_storage1: FileStorage = FileStorage.objects.get(pk=1)
self.file_storage2: FileStorage = FileStorage.objects.get(pk=2)
self.manager = Manager()
super().setUp()
class ManagerTest(TransactionTestCase):
fixtures = [
"storage_processes.yaml",
"storage_data.yaml",
"storage_users.yaml",
]
def setUp(self):
self.file_storage1: FileStorage = FileStorage.objects.get(pk=1)
self.file_storage2: FileStorage = FileStorage.objects.get(pk=2)
self.manager = Manager()
super().setUp()
def test_process(self):
process_copy_mock = MagicMock()
process_delete_mock = MagicMock()
with patch(
"resolwe.storage.manager.Manager.process_copy",
process_copy_mock,
):
with patch(
"resolwe.storage.manager.Manager.process_delete",
process_delete_mock,
):
self.manager.process()
self.assertEqual(process_copy_mock.call_count, 1)
self.assertEqual(process_delete_mock.call_count, 1)
def test_skip_locked(self):
rows_locked = Event()
manager_finished = Event()
StorageLocation.objects.create(
file_storage=self.file_storage1,
url="url1",
connector_name="local",
status=StorageLocation.STATUS_DONE,
)
StorageLocation.objects.create(
file_storage=self.file_storage2,
url="url2",
connector_name="local",
status=StorageLocation.STATUS_DONE,
)
def task_a(lock_ids=[]):
with transaction.atomic():
list(FileStorage.objects.select_for_update().filter(
id__in=lock_ids))
rows_locked.set()
manager_finished.wait()
connection.close()
def task_b():
rows_locked.wait()
self.manager = Manager()
self.manager.process()
manager_finished.set()
connection.close()
process_copy_mock = MagicMock()
process_delete_mock = MagicMock()
copy = MagicMock(return_value=FileStorage.objects.all())
delete = MagicMock(return_value=FileStorage.objects.all())
with patch.multiple(
"resolwe.storage.models.LocationsDoneManager",
to_delete=delete,
to_copy=copy,
):
with patch.multiple(
"resolwe.storage.manager.Manager",
copy_single_location=process_copy_mock,
delete_single_location=process_delete_mock,
):
with ThreadPoolExecutor() as executor:
executor.submit(
task_a, [self.file_storage1.id, self.file_storage2.id])
executor.submit(task_b)
process_copy_mock.assert_not_called()
process_delete_mock.assert_not_called()
rows_locked.clear()
manager_finished.clear()
process_copy_mock = MagicMock()
process_delete_mock = MagicMock()
copy = MagicMock(return_value=FileStorage.objects.filter(
pk=self.file_storage1.pk))
delete = MagicMock(side_effect=[
FileStorage.objects.filter(pk=self.file_storage2.pk),
FileStorage.objects.none(),
FileStorage.objects.none(),
])
with patch.multiple(
"resolwe.storage.models.LocationsDoneManager",
to_delete=delete,
to_copy=copy,
):
with patch.multiple(
"resolwe.storage.manager.Manager",
copy_single_location=process_copy_mock,
delete_single_location=process_delete_mock,
):
with ThreadPoolExecutor() as executor:
executor.submit(task_a, [self.file_storage1.id])
executor.submit(task_b)
process_copy_mock.assert_not_called()
process_delete_mock.assert_called_once_with(self.file_storage2,
"local")
def test_transfer(self):
FileStorage.objects.filter(pk=self.file_storage1.pk).update(
created=timezone.now() - timedelta(days=2))
self.file_storage1.refresh_from_db()
location_local = StorageLocation.objects.create(
file_storage=self.file_storage1,
url="url",
connector_name="local",
status=StorageLocation.STATUS_DONE,
)
path = ReferencedPath.objects.create(path="testme.txt", )
path.storage_locations.add(location_local)
transfer_objects = MagicMock(return_value=[])
transfer_instance = MagicMock(transfer_objects=transfer_objects)
transfer_module = MagicMock(return_value=transfer_instance)
with patch("resolwe.storage.models.Transfer", transfer_module):
self.manager.process_copy()
transfer_objects.assert_called_once()
self.assertEqual(len(transfer_objects.call_args[0]), 2)
arg1, arg2 = transfer_objects.call_args[0]
self.assertEqual(arg1, "url")
self.assertEqual(len(arg2), 1)
self.assertEqual(arg2[0]["path"], "testme.txt")
self.assertEqual(AccessLog.objects.all().count(), 1)
self.assertEqual(StorageLocation.objects.all().count(), 2)
created_location = StorageLocation.objects.exclude(
pk=location_local.pk).get()
self.assertEqual(created_location.connector_name, "S3")
self.assertEqual(created_location.url, "url")
access_log = AccessLog.objects.all().first()
self.assertEqual(access_log.storage_location, location_local)
self.assertIsNotNone(access_log.finished)
def test_transfer_failed(self):
def raise_datatransfererror(*args, **kwargs):
raise DataTransferError()
FileStorage.objects.filter(pk=self.file_storage1.pk).update(
created=timezone.now() - timedelta(days=2))
self.file_storage1.refresh_from_db()
location_local = StorageLocation.objects.create(
file_storage=self.file_storage1,
url="url",
connector_name="local",
status=StorageLocation.STATUS_DONE,
)
path = ReferencedPath.objects.create(path="testme.txt", )
path.storage_locations.add(location_local)
transfer_objects = MagicMock(side_effect=raise_datatransfererror)
transfer_instance = MagicMock(transfer_objects=transfer_objects)
transfer_module = MagicMock(return_value=transfer_instance)
with patch.dict(
CONNECTORS,
{
"GCS": MagicMock(priority=CONNECTORS["GCS"].priority),
"S3": MagicMock(priority=CONNECTORS["S3"].priority),
},
):
with patch("resolwe.storage.models.Transfer", transfer_module):
self.manager.process_copy()
transfer_objects.assert_called_once()
self.assertEqual(len(transfer_objects.call_args[0]), 2)
arg1, arg2 = transfer_objects.call_args[0]
self.assertEqual(arg1, "url")
self.assertEqual(len(arg2), 1)
self.assertEqual(arg2[0]["path"], "testme.txt")
self.assertEqual(AccessLog.objects.all().count(), 1)
self.assertEqual(StorageLocation.objects.all().count(), 1)
self.assertEqual(location_local, StorageLocation.objects.all().first())
access_log = AccessLog.objects.all().first()
self.assertEqual(access_log.storage_location, location_local)
self.assertIsNotNone(access_log.finished)
def test_delete(self):
location_local: StorageLocation = StorageLocation.objects.create(
file_storage=self.file_storage1,
url="url",
connector_name="local",
status=StorageLocation.STATUS_DONE,
)
# Do not delete.
process_copy_mock = MagicMock()
process_delete_mock = MagicMock()
copy = MagicMock(return_value=FileStorage.objects.none())
delete = MagicMock(return_value=FileStorage.objects.none())
with patch.multiple(
"resolwe.storage.models.LocationsDoneManager",
to_delete=delete,
to_copy=copy,
):
with patch.multiple(
"resolwe.storage.manager.Manager",
copy_single_location=process_copy_mock,
delete_single_location=process_delete_mock,
):
self.manager = Manager()
self.manager.process()
self.assertEqual(copy.call_count, 3)
self.assertEqual(delete.call_count, 3)
process_copy_mock.assert_not_called()
process_delete_mock.assert_not_called()
# Delete location_local.
delete_data = MagicMock()
location_local.delete_data = delete_data()
copy = MagicMock(return_value=FileStorage.objects.none())
delete = MagicMock(side_effect=[
FileStorage.objects.filter(pk=location_local.file_storage.pk),
FileStorage.objects.none(),
FileStorage.objects.none(),
])
with patch.multiple(
"resolwe.storage.models.LocationsDoneManager",
to_delete=delete,
to_copy=copy,
):
with patch.multiple(
"resolwe.storage.manager.Manager",
copy_single_location=process_copy_mock,
delete_single_location=process_delete_mock,
):
self.manager = Manager()
self.manager.process()
self.assertEqual(copy.call_count, 3)
process_copy_mock.assert_not_called()
delete_data.assert_called_once_with()
self.assertEqual(process_delete_mock.call_count, 1)