def test_preview_with_migratable_ip(self): old = add_storage_method_rel(DISK, 'old', STORAGE_TARGET_STATUS_MIGRATE) old_medium = add_storage_medium(old.storage_target, 20) add_storage_obj(self.ip, old_medium, DISK, '') new = StorageMethodTargetRelation.objects.create( storage_target=StorageTarget.objects.create(), storage_method=old.storage_method, status=STORAGE_TARGET_STATUS_ENABLED, ) self.policy.storage_methods.add(old.storage_method) data = { 'policy': str(self.policy.pk), } url = reverse('storage-migrations-preview-detail', args=(str(self.ip.pk),)) res = self.client.get(url, data=data) self.assertEqual(res.status_code, status.HTTP_200_OK) self.assertEqual(len(res.data), 1) self.assertEqual(res.data[0]['id'], str(new.storage_target.pk)) # new relation with new method and new target new_method_target = add_storage_method_rel(DISK, 'new', STORAGE_TARGET_STATUS_ENABLED) self.policy.storage_methods.add(new_method_target.storage_method) url = reverse('storage-migrations-preview-detail', args=(str(self.ip.pk),)) res = self.client.get(url, data=data) self.assertEqual(res.status_code, status.HTTP_200_OK) self.assertEqual(len(res.data), 2) self.assertEqual(res.data[0]['id'], str(new.storage_target.pk))
def test_multiple_storage_policies(self): other_policy = StoragePolicy.objects.create( policy_id='other', ingest_path=self.policy.ingest_path, ) ip = InformationPackage.objects.create(archived=True, submission_agreement=self.sa) old = add_storage_method_rel(DISK, 'old', STORAGE_TARGET_STATUS_MIGRATE) old_medium = add_storage_medium(old.storage_target, 20, '1') add_storage_obj(ip, old_medium, DISK, '') self.policy.storage_methods.add(old.storage_method) new = add_storage_method_rel(DISK, 'new', STORAGE_TARGET_STATUS_ENABLED) other_policy.storage_methods.add(new.storage_method) for rstatus in (STORAGE_TARGET_STATUS_MIGRATE, STORAGE_TARGET_STATUS_ENABLED, STORAGE_TARGET_STATUS_READ_ONLY): old.status = rstatus old.save() with self.subTest('old method-target rel status = %s' % old.get_status_display()): response = self.client.get(self.url, data={'migratable': True}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(len(response.data), 0) old.status = STORAGE_TARGET_STATUS_DISABLED old.save() with self.subTest('old method-target rel status = %s' % old.get_status_display()): response = self.client.get(self.url, data={'migratable': True}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(len(response.data), 0)
def test_methods_in_different_policies(self): migrate_rel = add_storage_method_rel(DISK, 'old_migrate', STORAGE_TARGET_STATUS_MIGRATE) migrate_medium = add_storage_medium(migrate_rel.storage_target, 20) add_storage_obj(self.ip, migrate_medium, DISK, '') self.policy.storage_methods.add(migrate_rel.storage_method) enabled_rel = add_storage_method_rel(DISK, 'new_enabled', STORAGE_TARGET_STATUS_ENABLED) other_policy = StoragePolicy.objects.create( policy_id='other', ingest_path=self.policy.ingest_path, ) other_policy.storage_methods.add(enabled_rel.storage_method) self.assertFalse(self.ip.get_migratable_storage_methods().exists())
def test_migrate_and_enabled_method_rels(self): migrate_rel = add_storage_method_rel(DISK, 'old_migrate', STORAGE_TARGET_STATUS_MIGRATE) migrate_medium = add_storage_medium(migrate_rel.storage_target, 20) add_storage_obj(self.ip, migrate_medium, DISK, '') enabled_rel = StorageMethodTargetRelation.objects.create( storage_target=StorageTarget.objects.create(), storage_method=migrate_rel.storage_method, status=STORAGE_TARGET_STATUS_ENABLED, ) self.policy.storage_methods.add(migrate_rel.storage_method) self.assertEqual(1, self.ip.get_migratable_storage_methods().count()) self.assertEqual(enabled_rel.storage_method, self.ip.get_migratable_storage_methods().get())
def test_preview_with_non_migratable_ip(self): old = add_storage_method_rel(DISK, 'old', STORAGE_TARGET_STATUS_ENABLED) old_medium = add_storage_medium(old.storage_target, 20) add_storage_obj(self.ip, old_medium, DISK, '') new = add_storage_method_rel(DISK, 'new', STORAGE_TARGET_STATUS_ENABLED) self.policy.storage_methods.add(old.storage_method, new.storage_method) ip = InformationPackage.objects.create() data = { 'policy': str(self.policy.pk), } url = reverse('storage-migrations-preview-detail', args=(str(ip.pk),)) res = self.client.get(url, data=data) self.assertEqual(res.status_code, status.HTTP_404_NOT_FOUND)
def test_no_change(self): ip = InformationPackage.objects.create(archived=True, submission_agreement=self.sa) old = add_storage_method_rel(DISK, 'old', STORAGE_TARGET_STATUS_ENABLED) old_medium = add_storage_medium(old.storage_target, 20, '1') add_storage_obj(ip, old_medium, DISK, '') self.policy.storage_methods.add(old.storage_method) response = self.client.get(self.url, data={'migratable': True}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data, []) response = self.client.get(self.url, data={'migratable': False}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(len(response.data), 1) self.assertEqual(response.data[0]['id'], str(old_medium.pk))
def test_migrate_container_to_non_container(self): old = add_storage_method_rel(DISK, 'old', STORAGE_TARGET_STATUS_MIGRATE) old.storage_target.target = tempfile.mkdtemp(dir=self.datadir) old.storage_target.save() self.policy.storage_methods.add(old.storage_method) old_medium = add_storage_medium(old.storage_target, 20) old_storage_obj = add_storage_obj(self.ip, old_medium, DISK, '', create_dir=False) old_storage_obj.container = True old_storage_obj.save() f, fpath = tempfile.mkstemp(dir=self.datadir) os.close(f) with tarfile.open(old_storage_obj.get_full_path(), 'w') as tar: tar.add(fpath, os.path.join(self.ip.object_identifier_value, 'dummyfile')) open(os.path.splitext(old_storage_obj.get_full_path())[0] + '.xml', 'a').close() target = old.storage_target.target open(os.path.join(target, str(self.ip.aic.pk)) + '.xml', 'a').close() StorageMethodTargetRelation.objects.create( storage_target=StorageTarget.objects.create( target=tempfile.mkdtemp(dir=self.datadir), ), storage_method=old.storage_method, status=STORAGE_TARGET_STATUS_ENABLED, ) data = { 'information_packages': [str(self.ip.pk)], 'policy': str(self.policy.pk), 'temp_path': self.tempdir, } response = self.client.post(self.url, data=data) self.assertEqual(response.status_code, status.HTTP_201_CREATED)
def test_new_storage_method(self): rel = add_storage_method_rel(DISK, 'old', STORAGE_TARGET_STATUS_MIGRATE) self.policy.storage_methods.add(rel.storage_method) medium = add_storage_medium(rel.storage_target, 20) add_storage_obj(self.ip, medium, DISK, '') new_storage_method = StorageMethod.objects.create() new_storage_target = StorageTarget.objects.create(name='new') StorageMethodTargetRelation.objects.create( storage_method=new_storage_method, storage_target=new_storage_target, status=STORAGE_TARGET_STATUS_ENABLED ) self.policy.storage_methods.add(new_storage_method) method_exists = self.ip.get_migratable_storage_methods().exists() self.assertTrue(method_exists)
def test_method_rel_states(self, mock_task): old = add_storage_method_rel(DISK, 'old', STORAGE_TARGET_STATUS_MIGRATE) old_medium = add_storage_medium(old.storage_target, 20) add_storage_obj(self.ip, old_medium, DISK, '') new = StorageMethodTargetRelation.objects.create( storage_target=StorageTarget.objects.create(), storage_method=old.storage_method, status=STORAGE_TARGET_STATUS_MIGRATE, ) self.policy.storage_methods.add(old.storage_method, new.storage_method) data = { 'information_packages': [str(self.ip.pk)], 'policy': str(self.policy.pk), 'temp_path': 'temp', } with self.subTest('old migrate, new migrate'): response = self.client.post(self.url, data=data) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) mock_task.assert_not_called() with self.subTest('old migrate, new read-only'): new.status = STORAGE_TARGET_STATUS_READ_ONLY new.save() response = self.client.post(self.url, data=data) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) mock_task.assert_not_called() with self.subTest('old migrate, new disabled'): new.status = STORAGE_TARGET_STATUS_DISABLED new.save() response = self.client.post(self.url, data=data) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) mock_task.assert_not_called() with self.subTest('old migrate, new enabled'): new.status = STORAGE_TARGET_STATUS_ENABLED new.save() response = self.client.post(self.url, data=data) self.assertEqual(response.status_code, status.HTTP_201_CREATED) mock_task.assert_called_once()
def test_pagination(self): ip = InformationPackage.objects.create(archived=True, submission_agreement=self.sa) old = add_storage_method_rel(DISK, 'old', STORAGE_TARGET_STATUS_MIGRATE) self.policy.storage_methods.add(old.storage_method) old_medium = add_storage_medium(old.storage_target, 20) add_storage_obj(self.ip, old_medium, DISK, '') add_storage_obj(ip, old_medium, DISK, '') target = StorageTarget.objects.create(name='new target, old method') StorageMethodTargetRelation.objects.create( storage_method=old.storage_method, storage_target=target, status=STORAGE_TARGET_STATUS_ENABLED, ) data = { 'information_packages': [str(self.ip.pk), str(ip.pk)], 'policy': str(self.policy.pk), } res = self.client.get(self.url, data=data) self.assertEqual(res.status_code, status.HTTP_200_OK) self.assertEqual(len(res.data), 2) data['page_size'] = 1 res = self.client.get(self.url, data=data) self.assertEqual(res.status_code, status.HTTP_200_OK) self.assertEqual(len(res.data), 1) first_page_ip = res.data[0] data['page'] = 2 res = self.client.get(self.url, data=data) self.assertEqual(res.status_code, status.HTTP_200_OK) self.assertEqual(len(res.data), 1) second_page_ip = res.data[0] self.assertNotEqual(first_page_ip, second_page_ip) data.pop('page') data.pop('page_size') data['pager'] = 'none' res = self.client.get(self.url, data=data) self.assertEqual(res.status_code, status.HTTP_200_OK) self.assertEqual(len(res.data), 2)
def test_preview_with_migratable_ip(self): old = add_storage_method_rel(DISK, 'old', STORAGE_TARGET_STATUS_MIGRATE) old_medium = add_storage_medium(old.storage_target, 20) add_storage_obj(self.ip, old_medium, DISK, '') StorageMethodTargetRelation.objects.create( storage_method=old.storage_method, storage_target=StorageTarget.objects.create(name='new_target'), status=STORAGE_TARGET_STATUS_ENABLED, ) self.policy.storage_methods.add(old.storage_method) data = { 'information_packages': [str(self.ip.pk)], 'policy': str(self.policy.pk), } res = self.client.get(self.url, data=data) self.assertEqual(res.status_code, status.HTTP_200_OK) self.assertEqual(len(res.data), 1)
def test_migration_task_order(self, mock_task_run, mock_task): old = add_storage_method_rel(TAPE, 'old', STORAGE_TARGET_STATUS_MIGRATE) old_medium = add_storage_medium(old.storage_target, 20) StorageMethodTargetRelation.objects.create( storage_target=StorageTarget.objects.create(), storage_method=old.storage_method, status=STORAGE_TARGET_STATUS_ENABLED, ) self.policy.storage_methods.add(old.storage_method) ips = [ InformationPackage.objects.create(archived=True, submission_agreement=self.sa) for _ in range(6) ] tape_location_values = ['1', '4', '5', '3', '10', '2'] for idx, ip in enumerate(ips): add_storage_obj(ip, old_medium, TAPE, tape_location_values[idx]) data = { 'information_packages': [str(ip.pk) for ip in ips], 'policy': str(self.policy.pk), 'temp_path': 'temp', } response = self.client.post(self.url, data=data) self.assertEqual(response.status_code, status.HTTP_201_CREATED) mock_task.assert_has_calls([ mock.call( name='ESSArch_Core.storage.tasks.StorageMigration', label=mock.ANY, status__in=mock.ANY, information_package=ip, defaults={ 'args': mock.ANY, 'responsible': mock.ANY, 'eager': False, } ) for ip in [ips[0], ips[5], ips[3], ips[1], ips[2], ips[4]] ])
def test_queue_duplicate_migrations(self, mock_task): old = add_storage_method_rel(DISK, 'old', STORAGE_TARGET_STATUS_MIGRATE) old_medium = add_storage_medium(old.storage_target, 20) add_storage_obj(self.ip, old_medium, DISK, '') StorageMethodTargetRelation.objects.create( storage_target=StorageTarget.objects.create(), storage_method=old.storage_method, status=STORAGE_TARGET_STATUS_ENABLED, ) self.policy.storage_methods.add(old.storage_method) data = { 'information_packages': [str(self.ip.pk)], 'policy': str(self.policy.pk), 'temp_path': 'temp', } for _ in range(5): response = self.client.post(self.url, data=data) self.assertEqual(response.status_code, status.HTTP_201_CREATED) data['temp_path'] = 'temp2' response = self.client.post(self.url, data=data) self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertEqual(ProcessTask.objects.count(), 1) mock_task.assert_called_once() with self.subTest('completed task'): ProcessTask.objects.update(status=celery_states.SUCCESS) mock_task.reset_mock() for _ in range(5): response = self.client.post(self.url, data=data) self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertEqual(ProcessTask.objects.count(), 2) mock_task.assert_called_once()
def test_multiple_storage_methods(self): ip = InformationPackage.objects.create(archived=True, submission_agreement=self.sa) old = add_storage_method_rel(DISK, 'old', STORAGE_TARGET_STATUS_MIGRATE) old_medium = add_storage_medium(old.storage_target, 20, '1') add_storage_obj(ip, old_medium, DISK, '') new = add_storage_method_rel(DISK, 'new', STORAGE_TARGET_STATUS_ENABLED) self.policy.storage_methods.add(old.storage_method, new.storage_method) for rstatus in (STORAGE_TARGET_STATUS_MIGRATE, STORAGE_TARGET_STATUS_ENABLED, STORAGE_TARGET_STATUS_READ_ONLY): old.status = rstatus old.save() with self.subTest('old method-target rel status = %s' % old.get_status_display()): response = self.client.get(self.url, data={'migratable': True}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(len(response.data), 1) self.assertEqual(response.data[0]['id'], str(old_medium.pk)) old.status = STORAGE_TARGET_STATUS_DISABLED old.save() with self.subTest('old method-target rel status = %s' % old.get_status_display()): response = self.client.get(self.url, data={'migratable': True}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(len(response.data), 0) # add object to new method old.status = STORAGE_TARGET_STATUS_MIGRATE old.save() new_medium = add_storage_medium(new.storage_target, 20, '2') add_storage_obj(ip, new_medium, DISK, '') response = self.client.get(self.url, data={'migratable': True}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(len(response.data), 0)
def test_migrate(self): old = add_storage_method_rel(DISK, 'old', STORAGE_TARGET_STATUS_MIGRATE) old.storage_target.target = tempfile.mkdtemp(dir=self.datadir) old.storage_target.save() self.policy.storage_methods.add(old.storage_method) old_medium = add_storage_medium(old.storage_target, 20) add_storage_obj(self.ip, old_medium, DISK, '', create_dir=True) StorageMethodTargetRelation.objects.create( storage_target=StorageTarget.objects.create( target=tempfile.mkdtemp(dir=self.datadir), ), storage_method=old.storage_method, status=STORAGE_TARGET_STATUS_ENABLED, ) data = { 'information_packages': [str(self.ip.pk)], 'policy': str(self.policy.pk), 'temp_path': self.tempdir, } response = self.client.post(self.url, data=data) self.assertEqual(response.status_code, status.HTTP_201_CREATED)
def test_migrate_non_container_to_container(self): self.sa.profile_aic_description = Profile.objects.create( profile_type='aic_description', specification=create_mets_spec(True, self.sa), ) self.sa.profile_aip_description = Profile.objects.create( profile_type='aip_description', specification=create_mets_spec(True, self.sa), ) self.sa.profile_preservation_metadata = Profile.objects.create( profile_type='preservation_metadata', specification=create_premis_spec(self.sa), ) self.sa.lock_to_information_package(self.ip, self.user) old = add_storage_method_rel(DISK, 'old', STORAGE_TARGET_STATUS_MIGRATE) old.storage_target.target = tempfile.mkdtemp(dir=self.datadir) old.storage_target.save() self.policy.storage_methods.add(old.storage_method) old_medium = add_storage_medium(old.storage_target, 20) old_storage_obj = add_storage_obj(self.ip, old_medium, DISK, '', create_dir=True) old_storage_obj.save() new_storage_method = StorageMethod.objects.create(containers=True) StorageMethodTargetRelation.objects.create( storage_target=StorageTarget.objects.create( target=tempfile.mkdtemp(dir=self.datadir), ), storage_method=new_storage_method, status=STORAGE_TARGET_STATUS_ENABLED, ) self.policy.storage_methods.add(new_storage_method) data = { 'information_packages': [str(self.ip.pk)], 'policy': str(self.policy.pk), 'temp_path': self.tempdir, } response = self.client.post(self.url, data=data) self.assertEqual(response.status_code, status.HTTP_201_CREATED)
def test_single_storage_method(self): ip = InformationPackage.objects.create(archived=True, submission_agreement=self.sa) old = add_storage_method_rel(DISK, 'old', STORAGE_TARGET_STATUS_MIGRATE) old_medium = add_storage_medium(old.storage_target, 20, '1') add_storage_obj(ip, old_medium, DISK, '') self.policy.storage_methods.add(old.storage_method) new_storage_target = StorageTarget.objects.create(name='new') new_rel = StorageMethodTargetRelation.objects.create( storage_method=old.storage_method, storage_target=new_storage_target, status=STORAGE_TARGET_STATUS_DISABLED ) new_storage_medium = add_storage_medium(new_storage_target, 20, '2') # New target exists but it is disabled response = self.client.get(self.url, data={'migratable': True}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data, []) response = self.client.get(self.url, data={'migratable': False}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(len(response.data), 2) # Enable new target new_rel.status = STORAGE_TARGET_STATUS_ENABLED new_rel.save() # New enabled target exists but no objects are migrated yet response = self.client.get(self.url, data={'migratable': True}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(len(response.data), 1) self.assertEqual(response.data[0]['id'], str(old_medium.pk)) # Add IP to new medium StorageObject.objects.create( ip=ip, storage_medium=new_storage_medium, content_location_type=DISK, ) # All objects migrated and old medium is deactivatable response = self.client.get(self.url, data={'migratable': True}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(len(response.data), 0) # Add new IP to old medium new_ip = InformationPackage.objects.create( archived=True, submission_agreement=self.sa, ) StorageObject.objects.create( ip=new_ip, storage_medium=old_medium, content_location_type=DISK, ) # All objects are not migrated response = self.client.get(self.url, data={'migratable': True}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(len(response.data), 1) self.assertEqual(response.data[0]['id'], str(old_medium.pk)) # Add new IP to new medium StorageObject.objects.create( ip=new_ip, storage_medium=new_storage_medium, content_location_type=DISK, ) # All objects migrated and old medium is deactivatable response = self.client.get(self.url, data={'migratable': True}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data, [])
def test_multiple_ips(self): ip1 = InformationPackage.objects.create( package_type=InformationPackage.AIP, archived=True, submission_agreement=self.sa, ) ip2 = InformationPackage.objects.create( package_type=InformationPackage.AIP, archived=True, submission_agreement=self.sa, ) # default default_rel = add_storage_method_rel(DISK, 'default', STORAGE_TARGET_STATUS_MIGRATE) default_medium = add_storage_medium(default_rel.storage_target, 20, 'default') new_target = StorageTarget.objects.create(name='new_target') new_rel = StorageMethodTargetRelation.objects.create( storage_method=default_rel.storage_method, storage_target=new_target, status=STORAGE_TARGET_STATUS_ENABLED ) # long term long_term_rel = add_storage_method_rel(DISK, 'default_long_term', STORAGE_TARGET_STATUS_ENABLED) long_term_medium = add_storage_medium(long_term_rel.storage_target, 20, 'long_term') self.policy.storage_methods.add( default_rel.storage_method, new_rel.storage_method, long_term_rel.storage_method, ) add_storage_obj(ip1, default_medium, DISK, '') add_storage_obj(ip1, long_term_medium, DISK, '') add_storage_obj(ip2, long_term_medium, DISK, '') response = self.client.get(self.url, data={'migratable': True}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(len(response.data), 2) self.assertCountEqual( [response.data[0]['id'], response.data[1]['id']], [str(default_medium.pk), str(long_term_medium.pk)], ) ip_list_url = reverse('informationpackage-list') response = self.client.get(ip_list_url, data={ 'medium': str(default_medium.pk), 'migratable': True, 'view_type': 'flat', }) self.assertEqual(len(response.data), 1) self.assertEqual(response.data[0]['id'], str(ip1.pk)) response = self.client.get(ip_list_url, data={ 'medium': str(long_term_medium.pk), 'migratable': True, 'view_type': 'flat', }) self.assertEqual(len(response.data), 2) self.assertCountEqual( [response.data[0]['id'], response.data[1]['id']], [str(ip1.pk), str(ip2.pk)], )