예제 #1
0
    def test_set_block_invalid(self):
        Storage.sync('s-1', storage_id=1)
        versions = []
        good_uid = BlockUid(1, 2)
        bad_uid = BlockUid(3, 4)
        for i in range(6):
            version = Version.create(version_uid=VersionUid(f'v{i + 1}'),
                                     volume='backup-name',
                                     snapshot='snapshot-name.{}'.format(i),
                                     size=16 * 1024 * 4096,
                                     storage_id=1,
                                     block_size=4 * 1024 * 4096)
            blocks = [{
                'idx': 0,
                'uid_left': bad_uid.left if i < 3 else good_uid.left,
                'uid_right': bad_uid.right if i < 3 else good_uid.right,
                'checksum': 'aabbcc',
                'size': 4 * 1024 * 4096,
                'valid': True,
            }]
            version.create_blocks(blocks=blocks)
            version.set(status=VersionStatus.valid)

            versions.append(version)

        Version.set_block_invalid(bad_uid)

        for i in range(3):
            self.assertEqual(VersionStatus.invalid, versions[i].status)
            self.assertFalse(list(versions[i].blocks)[0].valid)

        for i in range(3, 6):
            self.assertEqual(VersionStatus.valid, versions[i].status)
            self.assertTrue(list(versions[i].blocks)[0].valid)
예제 #2
0
 def test_version_blocks_count(self):
     Storage.sync('s-1', storage_id=1)
     for i in range(256):
         version = Version.create(version_uid=f'v{i + 1}',
                                  volume='backup-name',
                                  snapshot='snapshot-name.{}'.format(i),
                                  size=i * 1111 * 4096,
                                  storage_id=1,
                                  block_size=4 * 1024 * 4096)
         self.assertEqual(math.ceil(version.size / version.block_size),
                          version.blocks_count)
예제 #3
0
    def test_version_filter_dateparse(self):
        Storage.sync('s-1', storage_id=1)
        version_uids = set()
        for i in range(3):
            version = Version.create(version_uid=VersionUid(f'v{i + 1}'),
                                     volume='backup-name',
                                     snapshot='snapshot-name.{}'.format(i),
                                     size=16 * 1024 * 4096,
                                     storage_id=1,
                                     block_size=4 * 1024 * 4096)
            self.assertNotIn(version.uid, version_uids)
            version_uids.add(version.uid)

        # Wait at least one seconds
        time.sleep(1)

        versions = Version.find_with_filter('date <= "now"')
        self.assertEqual(3, len(versions))

        versions = Version.find_with_filter('date > "1 month ago"')
        self.assertEqual(3, len(versions))

        versions = Version.find_with_filter('date > "now"')
        self.assertEqual(0, len(versions))

        versions = Version.find_with_filter('date < "1 month ago"')
        self.assertEqual(0, len(versions))

        versions = Version.find_with_filter('"now" >= date')
        self.assertEqual(3, len(versions))

        versions = Version.find_with_filter('"1 month ago" < date')
        self.assertEqual(3, len(versions))

        versions = Version.find_with_filter('"now" < date')
        self.assertEqual(0, len(versions))

        versions = Version.find_with_filter('"1 month ago" > date')
        self.assertEqual(0, len(versions))

        versions = Version.find_with_filter('date <= "{}"'.format(
            datetime.datetime.now(
                tz=tz.tzlocal()).strftime("%Y-%m-%dT%H:%M:%S")))
        self.assertEqual(3, len(versions))
예제 #4
0
    def test_version_filter_issue_9_slowness(self):
        Storage.sync('s-1', storage_id=1)
        version_uids = set()
        for i in range(3):
            version = Version.create(version_uid=VersionUid(f'v{i + 1}'),
                                     volume='backup-name',
                                     snapshot='snapshot-name.{}'.format(i),
                                     size=16 * 1024 * 4096,
                                     storage_id=1,
                                     block_size=4 * 1024 * 4096)
            self.assertNotIn(version.uid, version_uids)
            version_uids.add(version.uid)

        t1 = timeit.timeit(lambda: Version.find_with_filter(
            'snapshot == "snapshot-name.2" and volume == "backup-name"'),
                           number=1)
        t2 = timeit.timeit(lambda: Version.find_with_filter(
            '(snapshot == "snapshot-name.2" and volume == "backup-name")'),
                           number=1)
        logger.debug(
            'test_version_filter_issue_9_slowness: t1 {}, t2 {}'.format(
                t1, t2))
        self.assertLess(t1 - t2, 5)
예제 #5
0
    def test_version_filter_issue_9(self):
        Storage.sync('s-1', storage_id=1)
        version_uids = set()
        for i in range(3):
            version = Version.create(version_uid=VersionUid(f'v{i + 1}'),
                                     volume='backup-name',
                                     snapshot='snapshot-name.{}'.format(i),
                                     size=16 * 1024 * 4096,
                                     storage_id=1,
                                     block_size=4 * 1024 * 4096,
                                     status=VersionStatus.valid)
            self.assertNotIn(version.uid, version_uids)
            version_uids.add(version.uid)

        versions = Version.find_with_filter(
            'snapshot == "snapshot-name.2" and volume == "backup-name" and status == "valid"'
        )
        self.assertEqual(1, len(versions))

        versions = Version.find_with_filter(
            'snapshot == "snapshot-name.2" or volume == "backup-name" or status == "valid"'
        )
        self.assertEqual(3, len(versions))
예제 #6
0
    def test_block(self):
        Storage.sync('s-1', storage_id=1)
        version = Version.create(version_uid=VersionUid('v1'),
                                 volume='name-' + self.random_string(12),
                                 snapshot='snapshot-name-' +
                                 self.random_string(12),
                                 size=256 * 1024 * 4096,
                                 block_size=1024 * 4096,
                                 storage_id=1)

        checksums = []
        uids = []
        num_blocks = 256
        blocks: List[Dict[str, Any]] = []
        for idx in range(num_blocks):
            checksums.append(self.random_hex(64))
            uids.append(BlockUid(1, idx))
            blocks.append({
                'idx': idx,
                'uid_left': uids[idx].left,
                'uid_right': uids[idx].right,
                'checksum': checksums[idx],
                'size': 1024 * 4096,
                'valid': True
            })
        version.create_blocks(blocks=blocks)

        for idx, checksum in enumerate(checksums):
            block = version.get_block_by_checksum(checksum)
            self.assertEqual(idx, block.idx)
            self.assertEqual(version.id, block.version_id)
            self.assertEqual(uids[idx], block.uid)
            self.assertEqual(checksum, block.checksum)
            self.assertEqual(1024 * 4096, block.size)
            self.assertTrue(block.valid)

        for idx, uid in enumerate(uids):
            block = version.get_block_by_idx(idx)
            self.assertEqual(idx, block.idx)
            self.assertEqual(version.id, block.version_id)
            self.assertEqual(uid, block.uid)
            self.assertEqual(checksums[idx], block.checksum)
            self.assertEqual(1024 * 4096, block.size)
            self.assertTrue(block.valid)

        self.assertEqual(num_blocks, len(list(version.blocks)))
        self.assertEqual(num_blocks, version.blocks_count)
        self.assertEqual(0, version.sparse_blocks_count)

        for idx, block in enumerate(version.blocks):
            self.assertEqual(idx, block.idx)
            self.assertEqual(version.id, block.version_id)
            self.assertEqual(uids[idx], block.uid)
            self.assertEqual(checksums[idx], block.checksum)
            self.assertEqual(1024 * 4096, block.size)
            self.assertTrue(block.valid)

        for idx, block in enumerate(version.blocks):
            dereferenced_block = block.deref()
            self.assertEqual(idx, dereferenced_block.idx)
            self.assertEqual(version.id, dereferenced_block.version_id)
            self.assertEqual(uids[idx].left, dereferenced_block.uid.left)
            self.assertEqual(uids[idx].right, dereferenced_block.uid.right)
            self.assertEqual(checksums[idx], dereferenced_block.checksum)
            self.assertEqual(1024 * 4096, dereferenced_block.size)
            self.assertTrue(dereferenced_block.valid)

        version.remove()
        deleted_count = 0
        for uids_deleted in DeletedBlock.get_unused_block_uids(-1):
            for storage in uids_deleted.values():
                for uid in storage:
                    self.assertIn(uid, uids)
                    deleted_count += 1
        self.assertEqual(num_blocks, deleted_count)
예제 #7
0
    def test_storage_usage(self):
        Storage.sync('s-1', storage_id=1)
        for i in range(2):
            version = Version.create(version_uid=VersionUid(f'v{i + 1}'),
                                     volume='backup-name',
                                     snapshot='snapshot-name.{}'.format(i),
                                     size=32 * 1024 * 4096,
                                     storage_id=1,
                                     block_size=1024 * 4096)

            blocks: List[Dict[str, Any]] = []
            # shared
            for idx in range(0, 6):
                uid = BlockUid(1, idx)
                blocks.append({
                    'idx': idx,
                    'uid_left': uid.left,
                    'uid_right': uid.right,
                    'checksum': None,
                    'size': 1024 * 4096,
                    'valid': True
                })
            # sparse
            for idx in range(6, 13):
                uid = BlockUid(i + 1, idx)
                blocks.append({
                    'idx': idx,
                    'uid_left': None,
                    'uid_right': None,
                    'checksum': None,
                    'size': 1024 * 4096,
                    'valid': True
                })
            # exclusive
            for idx in range(13, 25):
                uid = BlockUid(i + 1, idx)
                blocks.append({
                    'idx': idx,
                    'uid_left': uid.left,
                    'uid_right': uid.right,
                    'checksum': None,
                    'size': 1024 * 4096,
                    'valid': True
                })
            # exclusive deduplicated
            for idx in range(25, 32):
                uid = BlockUid(i + 1, idx - 7)
                blocks.append({
                    'idx': idx,
                    'uid_left': uid.left,
                    'uid_right': uid.right,
                    'checksum': None,
                    'size': 1024 * 4096,
                    'valid': True
                })
            version.create_blocks(blocks=blocks)

        for uid in ('v1', 'v2'):
            usage = Version.storage_usage(f'uid == "{uid}"')
            self.assertIsInstance(usage.get('s-1', None), dict)
            usage_s_1 = usage['s-1']
            check_value_matrix = (
                ('virtual', 32 * 1024 * 4096),
                ('shared', 6 * 1024 * 4096),
                ('sparse', 7 * 1024 * 4096),
                ('exclusive', 19 * 1024 * 4096),
                ('deduplicated_exclusive', (19 - 7) * 1024 * 4096),
            )
            for field, amount in check_value_matrix:
                value = usage_s_1.get(field, None)
                self.assertIsInstance(value, int)
                self.assertEqual(amount, value)
예제 #8
0
    def test_version_filter(self):
        Storage.sync('s-1', storage_id=1)
        for i in range(256):
            version = Version.create(version_uid=VersionUid(f'v{i + 1}'),
                                     volume='backup-name',
                                     snapshot='snapshot-name.{}'.format(i),
                                     size=16 * 1024 * 4096,
                                     storage_id=1,
                                     block_size=4 * 1024 * 4096,
                                     status=VersionStatus.valid)
            version = Version.get_by_uid(version.uid)
            self.assertEqual(0, len(version.labels))
            version.add_label('label-key', 'label-value')
            version.add_label('label-key-2', str(i))
            version.add_label('label-key-3', '')
            if i > 127:
                version.add_label('label-key-4', '')
                self.assertEqual(4, len(version.labels))
            else:
                self.assertEqual(3, len(version.labels))

        versions = Version.find_with_filter()
        self.assertEqual(256, len(versions))

        versions = Version.find_with_filter(
            'labels["label-key"] == "label-value"')
        self.assertEqual(256, len(versions))

        versions = Version.find_with_filter(
            '"label-value" == labels["label-key"]')
        self.assertEqual(256, len(versions))

        self.assertRaises(
            UsageError, lambda: Version.find_with_filter(
                'labels["label-key"] and "label-value"'))
        self.assertRaises(UsageError, lambda: Version.find_with_filter('True'))
        self.assertRaises(UsageError, lambda: Version.find_with_filter('10'))
        self.assertRaises(
            UsageError, lambda: Version.find_with_filter(
                'labels["label-key"] == "label-value" and True'))
        self.assertRaises(
            UsageError, lambda: Version.find_with_filter(
                'labels["label-key"] == "label-value" and False'))
        self.assertRaises(UsageError,
                          lambda: Version.find_with_filter('"hallo" == "hey"'))

        # volume is always true because it is never empty
        versions = Version.find_with_filter('volume')
        self.assertEqual(256, len(versions))

        versions = Version.find_with_filter('status == "valid"')
        self.assertEqual(256, len(versions))

        self.assertRaises(UsageError,
                          lambda: Version.find_with_filter('status == wrong'))

        versions = Version.find_with_filter('labels["label-key-3"] == ""')
        self.assertEqual(256, len(versions))

        versions = Version.find_with_filter(
            'labels["label-key"] != "label-value"')
        self.assertEqual(0, len(versions))

        versions = Version.find_with_filter('labels["label-key-2"] == 9')
        self.assertEqual(1, len(versions))

        versions = Version.find_with_filter('snapshot == "snapshot-name.1"')
        self.assertEqual(1, len(versions))
        self.assertEqual(VersionUid('v2'), versions[0].uid)

        versions = Version.find_with_filter(
            'snapshot == "snapshot-name.1" and labels["label-key-2"] == 1')
        self.assertEqual(1, len(versions))
        self.assertEqual(VersionUid('v2'), versions[0].uid)

        versions = Version.find_with_filter(
            'snapshot == "snapshot-name.1" and labels["label-key-2"] == "2"')
        self.assertEqual(0, len(versions))

        versions = Version.find_with_filter(
            'snapshot == "snapshot-name.1" or labels["label-key-2"] == 2')
        self.assertEqual(2, len(versions))
        self.assertSetEqual(
            {VersionUid('v2'), VersionUid('v3')},
            {version.uid
             for version in versions})

        versions = Version.find_with_filter(
            'volume == "backup-name" and snapshot == "snapshot-name.1"')
        self.assertEqual(1, len(versions))
        self.assertEqual(VersionUid('v2'), versions[0].uid)

        versions = Version.find_with_filter(
            'volume == "backup-name" and (snapshot == "snapshot-name.1" or snapshot == "snapshot-name.2")'
        )
        self.assertEqual(2, len(versions))
        self.assertSetEqual(
            {VersionUid('v2'), VersionUid('v3')},
            {version.uid
             for version in versions})

        versions = Version.find_with_filter('uid == "v1" or uid == "v12"')
        self.assertEqual(2, len(versions))
        self.assertSetEqual(
            {VersionUid('v1'), VersionUid('v12')},
            {version.uid
             for version in versions})

        versions = Version.find_with_filter('uid == "v1" and uid == "v12"')
        self.assertEqual(0, len(versions))

        versions = Version.find_with_filter('not labels["not-exists"]')
        self.assertEqual(256, len(versions))

        versions = Version.find_with_filter('labels["label-key-4"]')
        self.assertEqual(128, len(versions))

        versions = Version.find_with_filter('labels["label-key-4"] and volume')
        self.assertEqual(128, len(versions))
예제 #9
0
    def test_version(self):
        Storage.sync('s-1', storage_id=1)
        version = Version.create(version_uid=VersionUid('v1'),
                                 volume='backup-name',
                                 snapshot='snapshot-name',
                                 size=16 * 1024 * 4096,
                                 storage_id=1,
                                 block_size=4 * 1024 * 4096)

        version = Version.get_by_uid(version.uid)
        self.assertEqual('backup-name', version.volume)
        self.assertEqual('snapshot-name', version.snapshot)
        self.assertEqual(16 * 1024 * 4096, version.size)
        self.assertEqual(4 * 1024 * 4096, version.block_size)
        self.assertEqual(version.status, VersionStatus.incomplete)
        self.assertFalse(version.protected)

        version.set(status=VersionStatus.valid)
        version = Version.get_by_uid(version.uid)
        self.assertEqual(version.status, VersionStatus.valid)

        version.set(status=VersionStatus.invalid)
        version = Version.get_by_uid(version.uid)
        self.assertEqual(version.status, VersionStatus.invalid)

        version.set(protected=True)
        version = Version.get_by_uid(version.uid)
        self.assertTrue(version.protected)

        version.set(protected=False)
        version = Version.get_by_uid(version.uid)
        self.assertFalse(version.protected)

        version.add_label('label-1', 'bla')
        version.add_label('label-2', '')
        version = Version.get_by_uid(version.uid)
        self.assertEqual(2, len(version.labels))
        self.assertEqual(version.id, version.labels['label-1'].version_id)
        self.assertEqual('label-1', version.labels['label-1'].name)
        self.assertEqual('bla', version.labels['label-1'].value)
        self.assertEqual(version.id, version.labels['label-2'].version_id)
        self.assertEqual('label-2', version.labels['label-2'].name)
        self.assertEqual('', version.labels['label-2'].value)

        version.add_label('label-2', 'test123')
        version = Version.get_by_uid(version.uid)
        self.assertEqual(version.id, version.labels['label-2'].version_id)
        self.assertEqual('label-2', version.labels['label-2'].name)
        self.assertEqual('test123', version.labels['label-2'].value)

        version.rm_label('label-1')
        version = Version.get_by_uid(version.uid)
        self.assertEqual(1, len(version.labels))

        version.rm_label('label-2')
        version = Version.get_by_uid(version.uid)
        self.assertEqual(0, len(version.labels))

        version.rm_label('label-3')
        version = Version.get_by_uid(version.uid)
        self.assertEqual(0, len(version.labels))