Пример #1
0
    def test_import_assigns_correct_user_and_file_size(self):
        container_path = 'my/container/09'  # must have 3 parts
        envelope_path = 'envelope32'
        hierarchy_path = f'{envelope_path}/storage23'
        file_name = 'file1324.test'
        file_content = 'test123test123'
        metadata_file_content = {
            "tum_id": self.user1.username,
            "projects": [],
            "metadata_fields": [],
        }

        # create container hierarchy
        with FakeRequest(), FakeRequestUser(self.user1):
            container = DSSContainer.objects.create(
                name='Test Container',
                path=container_path,
                read_write_setting=DSSContainer.READ_ONLY,
                import_option=DSSContainer.IMPORT_LIST,
            )
            DSSEnvelope.objects.create(
                path='envelope',
                container=container,
                metadata_file_content=metadata_file_content,
            )

        # simulate automated API call from external service
        with FakeRequest(), FakeRequestUser(self.bot_user):
            file_to_import = DSSFilesToImport.objects.create(
                path=f'/dss/{container_path}/{hierarchy_path}/{file_name}')

        # simulate import via celery task
        time_before_import = timezone.now()
        with NoRequest(), \
             TemporaryDSSFile(filename=f'{container_path}/{hierarchy_path}/{file_name}', content=file_content), \
             TemporaryDSSFile(
                 filename=f'{container_path}/{envelope_path}/metadata.json',
                 content=json.dumps(metadata_file_content)
             ):
            import_dss_file(file_to_import.path)

        # check that  the imported file has the correct user set
        self.assertEqual(1, File.objects.all().count())
        file = File.objects.all().first()
        self.assertEqual(self.user1, file.created_by)
        self.assertEqual(self.user1, file.last_modified_by)
        self.assertEqual(len(file_content), file.file_size)
        self.assertEqual(file_name, file.title)
        self.assertEqual(file_name, file.original_filename)

        # check that the imported file is marked as imported
        file_to_import.refresh_from_db()
        self.assertTrue(file_to_import.imported)
        self.assertTrue(
            time_before_import <= file_to_import.imported_at <= timezone.now())
        self.assertFalse(file_to_import.last_import_attempt_failed)
        self.assertIsNone(file_to_import.last_import_attempt_failed_at)
        self.assertIsNone(file_to_import.last_import_fail_reason)
Пример #2
0
    def test_import_fails_for_missing_metadata_file(self):
        container_path = 'my/container/01'  # must have 3 parts
        hierarchy_path = 'envelope_with_missing_metadata/storage'
        file_name = 'file1.test'
        file_content = 'test123test123'

        # create container hierarchy
        with FakeRequest(), FakeRequestUser(self.user1):
            DSSContainer.objects.create(
                name='Test Container',
                path=container_path,
                read_write_setting=DSSContainer.READ_ONLY,
                import_option=DSSContainer.IMPORT_LIST,
            )

        # simulate automated API call from external service
        with FakeRequest(), FakeRequestUser(self.bot_user):
            file_to_import = DSSFilesToImport.objects.create(
                path=f'/dss/{container_path}/{hierarchy_path}/{file_name}')

        # simulate import via celery task
        time_before_import = timezone.now()
        with NoRequest(), TemporaryDSSFile(
                filename=f'{container_path}/{hierarchy_path}/{file_name}',
                content=file_content):
            try:
                import_dss_file(file_to_import.path)
            except Exception as exc:
                self.assertIn('metadata', str(exc))

        # check that no container have been created and the file was not imported
        self.assertEqual(0, File.objects.all().count())
        self.assertEqual(0, Directory.objects.all().count())
        self.assertEqual(0, Drive.objects.all().count())
        self.assertEqual(0, DSSEnvelope.objects.all().count())

        # check DSSFileToImport error stats
        file_to_import.refresh_from_db()
        self.assertTrue(file_to_import.last_import_attempt_failed)
        self.assertIsNotNone(file_to_import.last_import_fail_reason)
        self.assertEqual(1, file_to_import.import_attempts)
        self.assertIsNone(file_to_import.imported_at)
        self.assertFalse(file_to_import.imported)
        self.assertTrue(time_before_import < file_to_import.
                        last_import_attempt_failed_at < timezone.now())
Пример #3
0
    def test_import_creates_storage_hierarchy_and_stores_metadata_file_with_readonly_container(
            self):
        container_path = 'my/container/02'  # must have 3 parts
        envelope_path = 'envelope2'
        hierarchy_path = f'{envelope_path}/storage2/dir2/subdir2/subsubdir2'
        file_name = 'file2.test'
        file_content = 'test TEST 0123456789876543210 TEST test'
        metadata_file_content = json.dumps({
            "tum_id": self.user1.username,
            "projects": [],
            "metadata_fields": []
        })

        # create container
        with FakeRequest(), FakeRequestUser(self.user1):
            container = DSSContainer.objects.create(
                name='Test Container 2',
                path=container_path,
                read_write_setting=DSSContainer.READ_ONLY,
                import_option=DSSContainer.IMPORT_LIST,
            )

        # simulate automated API call from external service
        with FakeRequest(), FakeRequestUser(self.bot_user):
            file_to_import = DSSFilesToImport.objects.create(
                path=f'/dss/{container_path}/{hierarchy_path}/{file_name}')

        # simulate import via celery task
        with NoRequest(), TemporaryDSSFile(
                filename=f'{container_path}/{envelope_path}/metadata.json',
                content=metadata_file_content), TemporaryDSSFile(
                    filename=f'{container_path}/{hierarchy_path}/{file_name}',
                    content=file_content):
            import_dss_file(file_to_import.path)

        # check envelope
        envelope_qs = DSSEnvelope.objects.all()
        self.assertEqual(1, envelope_qs.count())
        envelope = envelope_qs.first()
        self.assertEqual('envelope2', envelope.path)
        self.assertEqual(container, envelope.container)
        self.assertEqual(metadata_file_content,
                         json.dumps(envelope.metadata_file_content))

        # check storage
        storage_qs = Drive.objects.all()
        self.assertEqual(1, storage_qs.count())
        storage = storage_qs.first()
        self.assertEqual('storage2', storage.title)
        self.assertEqual(envelope, storage.envelope)

        # check directories
        directory_qs = Directory.objects.all()
        self.assertEqual(
            4, directory_qs.count())  # 1 virtual root + 3 actual directories
        directories = list(directory_qs)

        # check virtual root directory
        root_dir = directories[0]
        self.assertTrue(root_dir.is_virtual_root)
        self.assertEqual('/', root_dir.name)
        self.assertIsNone(root_dir.directory)
        self.assertEqual(storage, root_dir.drive)

        # check first directory
        first_dir = directories[1]
        self.assertEqual('dir2', first_dir.name)
        self.assertEqual(root_dir, first_dir.directory)

        # check sub directory
        sub_dir = directories[2]
        self.assertEqual('subdir2', sub_dir.name)
        self.assertEqual(first_dir, sub_dir.directory)

        # check sub-sub directory
        sub_sub_dir = directories[3]
        self.assertEqual('subsubdir2', sub_sub_dir.name)
        self.assertEqual(sub_dir, sub_sub_dir.directory)

        # check file
        self.assertEqual(1, File.objects.all().count())
        file = File.objects.all().first()
        self.assertEqual(len(file_content), file.file_size)
        self.assertEqual(file_name, file.title)
        self.assertEqual(sub_sub_dir, file.directory)
        self.assertEqual(f'{container_path}/{hierarchy_path}/{file_name}',
                         str(file.path))
Пример #4
0
    def test_import_adds_projects_to_storage_and_files(self):
        container_path = 'my/container/03'  # must have 3 parts
        hierarchy_path = 'envelope/storage'

        self.memberRole = Role.objects.filter(name="Project Member").first()
        self.observerRole = Role.objects.filter(name="Observer").first()

        self.my_user1 = MyUser.objects.filter(pk=self.user1.pk).first()
        self.user_group = Group.objects.get(name='User')
        self.my_user1.groups.add(self.user_group)

        self.my_bot_user = MyUser.objects.filter(pk=self.bot_user.pk).first()
        self.my_bot_user.groups.add(self.user_group)

        # simulate automated API call from external service
        with FakeRequest(), FakeRequestUser(self.bot_user):
            self.project2 = Project.objects.create(name="Project 2")

            self.project3 = Project.objects.create(name="Project 3")

            # set user1 as observer in project2
            ProjectRoleUserAssignment.objects.create(
                user=self.my_user1,
                project=self.project2,
                role=self.observerRole,
            )

            file_to_import1 = DSSFilesToImport.objects.create(
                path=f'/dss/{container_path}/{hierarchy_path}/file1')
            file_to_import2 = DSSFilesToImport.objects.create(
                path=f'/dss/{container_path}/{hierarchy_path}/file2')

        # create container hierarchy
        with FakeRequest(), FakeRequestUser(self.my_user1):
            self.project1 = Project.objects.create(name="Project 1")

            container = DSSContainer.objects.create(
                name='Test Container 3',
                path=container_path,
                read_write_setting=DSSContainer.READ_ONLY,
                import_option=DSSContainer.IMPORT_LIST,
            )
            DSSEnvelope.objects.create(path='envelope',
                                       container=container,
                                       metadata_file_content={
                                           "tum_id":
                                           self.user1.username,
                                           "projects": [
                                               str(self.project1.pk),
                                               str(self.project2.pk),
                                               str(self.project3.pk)
                                           ],
                                           "metadata_fields": [],
                                       })

        # simulate import via celery task
        time_before_import = timezone.now()
        with NoRequest(), \
             TemporaryDSSFile(f'{container_path}/{hierarchy_path}/file1'), \
             TemporaryDSSFile(f'{container_path}/{hierarchy_path}/file2'):
            import_dss_file(file_to_import1.path)
            import_dss_file(file_to_import2.path)

        # check import stats
        file_to_import1.refresh_from_db()
        file_to_import2.refresh_from_db()
        self.assertTrue(file_to_import1.imported)
        self.assertTrue(file_to_import2.imported)
        self.assertTrue(time_before_import <= file_to_import1.imported_at <=
                        timezone.now())
        self.assertTrue(time_before_import <= file_to_import2.imported_at <=
                        timezone.now())

        # define helper function to check projects values
        def __assert_correct_projects_values(projects_qs):
            self.assertEqual(1, projects_qs.count())
            self.assertEqual([self.project1], list(projects_qs.all()))

        # check created storage
        storage_qs = Drive.objects.all()
        self.assertEqual(1, storage_qs.count())

        # check metadata on storage
        storage = storage_qs.first()
        __assert_correct_projects_values(storage.projects)

        # check imported files
        file_qs = File.objects.all()
        self.assertEqual(2, file_qs.count())
        files = list(file_qs)
        __assert_correct_projects_values(files[0].projects)
        __assert_correct_projects_values(files[1].projects)
Пример #5
0
    def test_import_does_not_add_metadata_redundantly_for_multiple_files_in_storage(
            self):
        container_path = 'my/container/05'  # must have 3 parts
        hierarchy_path = 'envelope/storage_multi_files'

        # create metadata fields
        metadata_field_checkbox = MetadataField.objects.create(
            name='checkbox',
            description='A checkbox field',
            base_type=MetadataField.BASE_TYPE_CHECKBOX,
        )

        # create container hierarchy
        with FakeRequest(), FakeRequestUser(self.user1):
            container = DSSContainer.objects.create(
                name='Test Container 3',
                path=container_path,
                read_write_setting=DSSContainer.READ_ONLY,
                import_option=DSSContainer.IMPORT_LIST,
            )
            DSSEnvelope.objects.create(
                path='envelope',
                container=container,
                metadata_file_content={
                    "tum_id":
                    self.user1.username,
                    "projects": [],
                    "metadata_fields": [
                        {
                            "id": str(metadata_field_checkbox.pk),
                            "values": [
                                {
                                    "value": True
                                },
                            ]
                        },
                    ],
                })

        # simulate automated API call from external service
        with FakeRequest(), FakeRequestUser(self.bot_user):
            file_to_import1 = DSSFilesToImport.objects.create(
                path=f'/dss/{container_path}/{hierarchy_path}/file1')
            file_to_import2 = DSSFilesToImport.objects.create(
                path=f'/dss/{container_path}/{hierarchy_path}/file2')
            file_to_import3 = DSSFilesToImport.objects.create(
                path=f'/dss/{container_path}/{hierarchy_path}/file3')

        # simulate imports via celery task
        with NoRequest(), \
             TemporaryDSSFile(f'{container_path}/{hierarchy_path}/file1'), \
             TemporaryDSSFile(f'{container_path}/{hierarchy_path}/file2'), \
             TemporaryDSSFile(f'{container_path}/{hierarchy_path}/file3'):
            import_dss_file(file_to_import1.path)
            import_dss_file(file_to_import2.path)
            import_dss_file(file_to_import3.path)

        # check created storage
        drive_qs = Drive.objects.all()
        self.assertEqual(1, drive_qs.count())

        # check that the metadata was not added multiple times to the storage
        drive = drive_qs.first()
        self.assertEqual(1, drive.metadata.all().count())
Пример #6
0
    def test_failed_import_marks_filetoimport_as_failed(self):
        container_path = 'my/container/04'  # must have 3 parts
        hierarchy_path = 'envelope/storage/dir1/dir1-1'
        file_name = 'file4.test'
        file_content = 'test 1234'

        # create container hierarchy
        with FakeRequest(), FakeRequestUser(self.user1):
            container = DSSContainer.objects.create(
                name='Test Container 3',
                path=container_path,
                read_write_setting=DSSContainer.READ_ONLY,
                import_option=DSSContainer.IMPORT_LIST,
            )
            DSSEnvelope.objects.create(path='envelope',
                                       container=container,
                                       metadata_file_content={
                                           "tum_id": self.user1.username,
                                           "projects": [],
                                           "metadata_fields": [],
                                       })

        # simulate automated API call from external service
        with FakeRequest(), FakeRequestUser(self.bot_user):
            file_to_import = DSSFilesToImport.objects.create(
                path=f'/dss/{container_path}/{hierarchy_path}/{file_name}')

        # simulate import via celery task
        time_before_import = timezone.now()
        with NoRequest(), TemporaryDSSFile(
                filename=f'{container_path}/{hierarchy_path}/{file_name}',
                content=file_content), patch(
                    'eric.dss.helper_classes.DSSFileImport.create_data',
                    side_effect=OSError('Fake Exception')):
            try:
                import_dss_file(file_to_import.path)
            except Exception as exc:
                self.assertIn('Fake Exception', str(exc))

        # check that the file was not imported and no hierarchy was created
        self.assertEqual(0, File.objects.all().count())
        self.assertEqual(0, Directory.objects.all().count())
        self.assertEqual(0, Drive.objects.all().count())

        # check DSSFileToImport error stats
        file_to_import.refresh_from_db()
        self.assertFalse(file_to_import.imported)
        self.assertIsNone(file_to_import.imported_at)
        self.assertTrue(file_to_import.last_import_attempt_failed)
        self.assertTrue(time_before_import <= file_to_import.
                        last_import_attempt_failed_at <= timezone.now())
        self.assertIsNotNone(file_to_import.last_import_fail_reason)
        self.assertIn('Fake Exception', file_to_import.last_import_fail_reason)
        self.assertEqual(1, file_to_import.import_attempts)

        # retry: simulate import via celery task
        time_before_import = timezone.now()
        with NoRequest(), TemporaryDSSFile(
                filename=f'{container_path}/{hierarchy_path}/{file_name}',
                content=file_content), patch(
                    'eric.dss.helper_classes.DSSFileImport.create_data',
                    side_effect=Exception('Another Fake Exception')):
            try:
                import_dss_file(file_to_import.path)
            except Exception as exc:
                self.assertIn('Another Fake Exception', str(exc))

        # check that the file was not imported and no hierarchy was created
        self.assertEqual(0, File.objects.all().count())
        self.assertEqual(0, Directory.objects.all().count())
        self.assertEqual(0, Drive.objects.all().count())

        # check DSSFileToImport error stats
        file_to_import.refresh_from_db()
        self.assertFalse(file_to_import.imported)
        self.assertIsNone(file_to_import.imported_at)
        self.assertTrue(file_to_import.last_import_attempt_failed)
        self.assertTrue(time_before_import <= file_to_import.
                        last_import_attempt_failed_at <= timezone.now())
        self.assertIsNotNone(file_to_import.last_import_fail_reason)
        self.assertIn('Another Fake Exception',
                      file_to_import.last_import_fail_reason)
        self.assertEqual(2, file_to_import.import_attempts)
Пример #7
0
    def test_import_adds_metadata_to_storage_and_files(self):
        container_path = 'my/container/03'  # must have 3 parts
        hierarchy_path = 'envelope/storage'

        # create metadata fields
        metadata_field_checkbox = MetadataField.objects.create(
            name='checkbox',
            description='desc1',
            base_type=MetadataField.BASE_TYPE_CHECKBOX,
        )
        metadata_field_fraction = MetadataField.objects.create(
            name='fraction',
            description='desc2',
            base_type=MetadataField.BASE_TYPE_FRACTION,
        )

        # create container hierarchy
        with FakeRequest(), FakeRequestUser(self.user1):
            container = DSSContainer.objects.create(
                name='Test Container 3',
                path=container_path,
                read_write_setting=DSSContainer.READ_ONLY,
                import_option=DSSContainer.IMPORT_LIST,
            )
            DSSEnvelope.objects.create(path='envelope',
                                       container=container,
                                       metadata_file_content={
                                           "tum_id":
                                           self.user1.username,
                                           "projects": [],
                                           "metadata_fields": [{
                                               "id":
                                               str(metadata_field_checkbox.pk),
                                               "values": [
                                                   {
                                                       "value": True
                                                   },
                                                   {
                                                       "value": False
                                                   },
                                               ]
                                           }, {
                                               "id":
                                               str(metadata_field_fraction.pk),
                                               "values": [
                                                   {
                                                       "nominator": 5,
                                                       "denominator": 3
                                                   },
                                               ]
                                           }],
                                       })

        # simulate automated API call from external service
        with FakeRequest(), FakeRequestUser(self.bot_user):
            file_to_import1 = DSSFilesToImport.objects.create(
                path=f'/dss/{container_path}/{hierarchy_path}/file1')
            file_to_import2 = DSSFilesToImport.objects.create(
                path=f'/dss/{container_path}/{hierarchy_path}/file2')

        # simulate import via celery task
        time_before_import = timezone.now()
        with NoRequest(), \
             TemporaryDSSFile(f'{container_path}/{hierarchy_path}/file1'), \
             TemporaryDSSFile(f'{container_path}/{hierarchy_path}/file2'):
            import_dss_file(file_to_import1.path)
            import_dss_file(file_to_import2.path)

        # check import stats
        file_to_import1.refresh_from_db()
        file_to_import2.refresh_from_db()
        self.assertTrue(file_to_import1.imported)
        self.assertTrue(file_to_import2.imported)
        self.assertTrue(time_before_import <= file_to_import1.imported_at <=
                        timezone.now())
        self.assertTrue(time_before_import <= file_to_import2.imported_at <=
                        timezone.now())

        # define helper function to check metadata values
        def __assert_correct_metadata_values(metadata_qs):
            md_checkbox_qs = metadata_qs.filter(field=metadata_field_checkbox)
            md_fraction_qs = metadata_qs.filter(field=metadata_field_fraction)
            self.assertEqual(2, md_checkbox_qs.count())
            self.assertEqual(1, md_fraction_qs.count())
            md_fraction = metadata_qs.filter(
                field=metadata_field_fraction).first()
            self.assertEqual(5, md_fraction.values.get('nominator'))
            self.assertEqual(3, md_fraction.values.get('denominator'))

        # check created storage
        storage_qs = Drive.objects.all()
        self.assertEqual(1, storage_qs.count())

        # check metadata on storage
        storage = storage_qs.first()
        __assert_correct_metadata_values(storage.metadata)

        # check imported files
        file_qs = File.objects.all()
        self.assertEqual(2, file_qs.count())
        files = list(file_qs)
        __assert_correct_metadata_values(files[0].metadata)
        __assert_correct_metadata_values(files[1].metadata)