Пример #1
0
    def _exp_obj_md5(self, read_size):
        logging.getLogger("pyocf").warning(
            "Reading whole exported object! This disturbs statistics values")

        read_buffer_all = Data(self.parent.device.size)

        read_buffer = Data(read_size)

        position = 0
        while position < read_buffer_all.size:
            io = self.new_io(self.parent.get_default_queue(), position,
                             read_size, IoDir.READ, 0, 0)
            io.set_data(read_buffer)

            cmpl = OcfCompletion([("err", c_int)])
            io.callback = cmpl.callback
            io.submit()
            cmpl.wait()

            if cmpl.results["err"]:
                raise Exception("Error reading whole exported object")

            read_buffer_all.copy(read_buffer, position, 0, read_size)
            position += read_size

        return read_buffer_all.md5()
Пример #2
0
def test_secure_erase_simple_io_cleaning():
    """
        Perform simple IO which will trigger WB cleaning. Track all the data from
        cleaner (locked) and make sure they are erased and unlocked after use.

        1. Start cache in WB mode
        2. Write single sector at LBA 0
        3. Read whole cache line at LBA 0
        4. Assert that 3. triggered cleaning
        5. Check if all locked Data copies were erased and unlocked
    """
    ctx = OcfCtx(
        OcfLib.getInstance(),
        b"Security tests ctx",
        DefaultLogger(LogLevel.WARN),
        DataCopyTracer,
        Cleaner,
    )

    ctx.register_volume_type(RamVolume)

    cache_device = RamVolume(S.from_MiB(50))
    cache = Cache.start_on_device(cache_device, cache_mode=CacheMode.WB)

    core_device = RamVolume(S.from_MiB(100))
    core = Core.using_device(core_device)
    cache.add_core(core)
    vol = CoreVolume(core, open=True)
    queue = cache.get_default_queue()

    read_data = Data(S.from_sector(1).B)
    io = vol.new_io(queue,
                    S.from_sector(1).B, read_data.size, IoDir.WRITE, 0, 0)
    io.set_data(read_data)

    cmpl = OcfCompletion([("err", c_int)])
    io.callback = cmpl.callback
    io.submit()
    cmpl.wait()

    read_data = Data(S.from_sector(8).B)
    io = vol.new_io(queue,
                    S.from_sector(1).B, read_data.size, IoDir.READ, 0, 0)
    io.set_data(read_data)

    cmpl = OcfCompletion([("err", c_int)])
    io.callback = cmpl.callback
    io.submit()
    cmpl.wait()

    stats = cache.get_stats()

    ctx.exit()

    assert (len(DataCopyTracer.needs_erase) == 0
            ), "Not all locked Data instances were secure erased!"
    assert (len(DataCopyTracer.locked_instances) == 0
            ), "Not all locked Data instances were unlocked!"
    assert (stats["usage"]["clean"]["value"]) > 0, "Cleaner didn't run!"
Пример #3
0
def test_partial_hit_promotion(pyocf_ctx):
    """
    Check if NHIT promotion policy doesn't prevent partial hits from getting
    promoted to cache

    1. Create core/cache pair with promotion policy ALWAYS
    2. Issue one-sector IO to cache to insert partially valid cache line
    3. Set NHIT promotion policy with trigger=0 (always triggered) and high
    insertion threshold
    4. Issue a request containing partially valid cache line and next cache line
        * occupancy should rise - partially hit request should bypass nhit criteria
    """

    # Step 1
    cache_device = Volume(Size.from_MiB(30))
    core_device = Volume(Size.from_MiB(30))

    cache = Cache.start_on_device(cache_device)
    core = Core.using_device(core_device)
    cache.add_core(core)

    # Step 2
    comp = OcfCompletion([("error", c_int)])
    write_data = Data(Size.from_sector(1))
    io = core.new_io(cache.get_default_queue(), 0, write_data.size, IoDir.READ, 0, 0)
    io.set_data(write_data)
    io.callback = comp.callback
    io.submit()

    comp.wait()

    stats = cache.get_stats()
    cache_lines = stats["conf"]["size"]
    assert stats["usage"]["occupancy"]["value"] == 1

    # Step 3
    cache.set_promotion_policy(PromotionPolicy.NHIT)
    cache.set_promotion_policy_param(
        PromotionPolicy.NHIT, NhitParams.TRIGGER_THRESHOLD, 0
    )
    cache.set_promotion_policy_param(
        PromotionPolicy.NHIT, NhitParams.INSERTION_THRESHOLD, 100
    )

    # Step 4
    comp = OcfCompletion([("error", c_int)])
    write_data = Data(2 * cache_lines.line_size)
    io = core.new_io(cache.get_default_queue(), 0, write_data.size, IoDir.WRITE, 0, 0)
    io.set_data(write_data)
    io.callback = comp.callback
    io.submit()
    comp.wait()

    stats = cache.get_stats()
    assert (
        stats["usage"]["occupancy"]["value"] == 2
    ), "Second cache line should be mapped"
Пример #4
0
def test_write_size_greater_than_cache(pyocf_ctx, mode: CacheMode,
                                       cls: CacheLineSize):
    """Test if eviction does not occur when IO greater than cache size is submitted.
    """
    cache_device = Volume(Size.from_MiB(20))

    core_device = Volume(Size.from_MiB(5))
    cache = Cache.start_on_device(cache_device,
                                  cache_mode=mode,
                                  cache_line_size=cls)
    cache_size = cache.get_stats()['conf']['size']
    core_exported = Core.using_device(core_device)
    cache.add_core(core_exported)
    cache.set_seq_cut_off_policy(SeqCutOffPolicy.NEVER)

    valid_io_size = Size.from_B(cache_size.B // 2)
    test_data = Data(valid_io_size)
    send_io(core_exported, test_data)

    stats = core_exported.cache.get_stats()
    first_block_sts = stats['block']
    first_usage_sts = stats['usage']
    pt_writes_first = stats['req']['wr_pt']
    assert stats["usage"]["occupancy"]["value"] == (valid_io_size.B / Size.from_KiB(4).B),\
        "Occupancy after first IO"
    prev_writes_to_core = stats["block"]["core_volume_wr"]["value"]

    # Anything below 5 MiB is a valid size (less than core device size)
    # Writing over cache size (to the offset above first io) in this case should go
    # directly to core and shouldn't trigger eviction
    io_size_bigger_than_cache = Size.from_MiB(2)
    io_offset = valid_io_size
    test_data = Data(io_size_bigger_than_cache)
    send_io(core_exported, test_data, io_offset)

    if mode is not CacheMode.WT:
        # Flush first write
        cache.flush()
    stats = core_exported.cache.get_stats()
    second_block_sts = stats['block']
    second_usage_sts = stats['usage']
    pt_writes_second = stats['req']['wr_pt']

    # Second write shouldn't affect cache and should go directly to core.
    # Cache occupancy shouldn't change
    # Second IO should go in PT
    assert first_usage_sts['occupancy'] == \
        second_usage_sts['occupancy']
    assert pt_writes_first['value'] == 0
    assert pt_writes_second['value'] == 1
    assert second_block_sts['cache_volume_wr'][
        'value'] == valid_io_size.blocks_4k
    assert second_block_sts['core_volume_wr']['value'] == valid_io_size.blocks_4k + \
        io_size_bigger_than_cache.blocks_4k
Пример #5
0
def fill_cache(cache, fill_ratio):
    """
    Helper to fill cache from LBA 0.
    TODO:
        * make it generic and share across all tests
        * reasonable error handling
    """

    cache_lines = cache.get_stats()["conf"]["size"]

    bytes_to_fill = cache_lines.bytes * fill_ratio
    max_io_size = cache.device.get_max_io_size().bytes

    ios_to_issue = math.floor(bytes_to_fill / max_io_size)

    core = cache.cores[0]
    completions = []
    for i in range(ios_to_issue):
        comp = OcfCompletion([("error", c_int)])
        write_data = Data(max_io_size)
        io = core.new_io(
            cache.get_default_queue(),
            i * max_io_size,
            write_data.size,
            IoDir.WRITE,
            0,
            0,
        )
        io.set_data(write_data)
        io.callback = comp.callback
        completions += [comp]
        io.submit()

    if bytes_to_fill % max_io_size:
        comp = OcfCompletion([("error", c_int)])
        write_data = Data(
            Size.from_B(bytes_to_fill % max_io_size, sector_aligned=True))
        io = core.new_io(
            cache.get_default_queue(),
            ios_to_issue * max_io_size,
            write_data.size,
            IoDir.WRITE,
            0,
            0,
        )
        io.set_data(write_data)
        io.callback = comp.callback
        completions += [comp]
        io.submit()

    for c in completions:
        c.wait()
Пример #6
0
def test_eviction_two_cores(pyocf_ctx, mode: CacheMode, cls: CacheLineSize):
    """Test if eviction works correctly when remapping cachelines between distinct cores."""
    cache_device = Volume(Size.from_MiB(20))

    core_device1 = Volume(Size.from_MiB(40))
    core_device2 = Volume(Size.from_MiB(40))
    cache = Cache.start_on_device(cache_device,
                                  cache_mode=mode,
                                  cache_line_size=cls)
    cache.set_seq_cut_off_policy(SeqCutOffPolicy.NEVER)
    cache_size = cache.get_stats()["conf"]["size"]
    core_exported1 = Core.using_device(core_device1, name="core1")
    core_exported2 = Core.using_device(core_device2, name="core2")
    cache.add_core(core_exported1)
    cache.add_core(core_exported2)

    valid_io_size = Size.from_B(cache_size.B)
    test_data = Data(valid_io_size)
    send_io(core_exported1, test_data)
    send_io(core_exported2, test_data)

    stats1 = core_exported1.get_stats()
    stats2 = core_exported2.get_stats()
    # IO to the second core should evict all the data from the first core
    assert stats1["usage"]["occupancy"]["value"] == 0
    assert stats2["usage"]["occupancy"]["value"] == valid_io_size.blocks_4k
def ocf_read(vol, queue, offset):
    data = Data(byte_count=512)
    comp = OcfCompletion([("error", c_int)])
    io = vol.new_io(queue, offset, 512, IoDir.READ, 0, 0)
    io.set_data(data)
    io.callback = comp.callback
    io.submit()
    comp.wait()
    return data.get_bytes()[0]
Пример #8
0
def test_write_size_greater_than_cache(pyocf_ctx, mode: CacheMode,
                                       cls: CacheLineSize):
    """Test if eviction does not occur when IO greater than cache size is submitted.
    """
    cache_device = Volume(
        Size.from_MiB(20))  # this gives about 1.375 MiB actual caching space

    core_device = Volume(Size.from_MiB(5))
    cache = Cache.start_on_device(cache_device,
                                  cache_mode=mode,
                                  cache_line_size=cls)
    core_exported = Core.using_device(core_device)
    cache.add_core(core_exported)
    cache.set_seq_cut_off_policy(SeqCutOffPolicy.NEVER)

    valid_io_size = Size.from_KiB(512)
    test_data = Data(valid_io_size)
    send_io(core_exported, test_data)

    stats = core_exported.cache.get_stats()
    assert stats["usage"]["occupancy"]["value"] == (valid_io_size.B / Size.from_KiB(4).B),\
        "Occupancy after first IO"
    prev_writes_to_core = stats["block"]["core_volume_wr"]["value"]

    # Anything below 5 MiB is a valid size (less than core device size)
    # Writing over 1.375 MiB in this case should go directly to core and shouldn't trigger eviction
    io_size_bigger_than_cache = Size.from_MiB(2)
    test_data = Data(io_size_bigger_than_cache)
    send_io(core_exported, test_data)

    stats = core_exported.cache.get_stats()

    # Writes from IO greater than cache size should go directly to core
    # Writes to core should equal the following:
    # Previous writes to core + size written + size cleaned (reads from cache)
    assert stats["block"]["core_volume_wr"]["value"] == \
        stats["block"]["cache_volume_rd"]["value"] + \
        prev_writes_to_core + io_size_bigger_than_cache.B / Size.from_KiB(4).B, \
        "Writes to core after second IO"

    # Occupancy shouldn't change (no eviction)
    assert stats["usage"]["occupancy"]["value"] == (valid_io_size.B / Size.from_KiB(4).B),\
        "Occupancy after second IO"
Пример #9
0
def _io(vol, queue, addr, size, direction, context):
    comp = OcfCompletion([("error", c_int)], context=context)
    data = Data(size)

    io = vol.new_io(queue, addr, size, direction, 0, 0)
    io.set_data(data)
    io.callback = comp.callback
    io.submit()

    return comp
Пример #10
0
def _io(core, addr, size, direction, context):
    comp = OcfCompletion([("error", c_int)], context=context)
    data = Data(size)

    io = core.new_io(core.cache.get_default_queue(), addr, size, direction, 0,
                     0)
    io.set_data(data)
    io.callback = comp.callback
    io.submit()

    return comp
Пример #11
0
def test_neg_size_unaligned(pyocf_ctx, c_uint16_randomize):
    """
        Check that write operations are blocked when
        IO size is not aligned
    """

    vol, queue = prepare_cache_and_core(Size.from_MiB(2))
    data = Data(int(Size.from_B(c_uint16_randomize)))
    if c_uint16_randomize % 512 != 0:
        with pytest.raises(Exception):
            vol.new_io(queue, 0, data.size, IoDir.WRITE, 0, 0)
Пример #12
0
def test_neg_size_unaligned(pyocf_ctx, c_uint16_randomize):
    """
        Check that write operations are blocked when
        IO size is not aligned
    """

    core = prepare_cache_and_core(Size.from_MiB(2))
    data = Data(int(Size.from_B(c_uint16_randomize)))
    if c_uint16_randomize % 512 != 0:
        with pytest.raises(Exception, match="Failed to create io!"):
            core.new_io(core.cache.get_default_queue(), 0, data.size,
                        IoDir.WRITE, 0, 0)
Пример #13
0
def test_neg_offset_unaligned(pyocf_ctx, c_int_randomize):
    """
        Check that write operations are blocked when
        IO offset is not aligned
    """

    vol, queue = prepare_cache_and_core(Size.from_MiB(2))
    vol = vol.get_front_volume()
    data = Data(int(Size.from_KiB(1)))
    if c_int_randomize % 512 != 0:
        with pytest.raises(Exception):
            vol.new_io(queue, c_int_randomize, data.size, IoDir.WRITE, 0, 0)
Пример #14
0
def test_secure_erase_simple_io_cleaning():
    """
        Perform simple IO which will trigger WB cleaning. Track all the data from
        cleaner (locked) and make sure they are erased and unlocked after use.
    """
    ctx = OcfCtx(
        OcfLib.getInstance(),
        b"Security tests ctx",
        DefaultLogger(LogLevel.WARN),
        DataCopyTracer,
        MetadataUpdater,
        Cleaner,
    )

    ctx.register_volume_type(Volume)

    cache_device = Volume(S.from_MiB(30))
    cache = Cache.start_on_device(cache_device, cache_mode=CacheMode.WB)

    core_device = Volume(S.from_MiB(100))
    core = Core.using_device(core_device)
    cache.add_core(core)

    cmpls = []
    for i in range(10000):
        read_data = Data(S.from_KiB(120))
        io = core.new_io()
        io.set_data(read_data)
        io.configure(
            (i * 1259) % int(core_device.size), read_data.size, IoDir.WRITE, 0, 0
        )
        io.set_queue(cache.get_default_queue())

        cmpl = OcfCompletion([("err", c_int)])
        io.callback = cmpl.callback
        cmpls.append(cmpl)
        io.submit()

    for c in cmpls:
        c.wait()

    stats = cache.get_stats()

    ctx.exit()

    assert (
        len(DataCopyTracer.needs_erase) == 0
    ), "Not all locked Data instances were secure erased!"
    assert (
        len(DataCopyTracer.locked_instances) == 0
    ), "Not all locked Data instances were unlocked!"
    assert (stats["usage"]["clean"]["value"]) > 0, "Cleaner didn't run!"
Пример #15
0
def io_from_exported_object(vol: Volume, queue: Queue, buffer_size: int,
                            offset: int):
    read_buffer = Data(buffer_size)
    io = vol.new_io(queue, offset, read_buffer.size, IoDir.READ, 0, 0)
    io.set_data(read_buffer)

    completion = OcfCompletion([("err", c_int)])
    io.callback = completion.callback
    io.submit()
    completion.wait()

    assert completion.results["err"] == 0, "IO from exported object completion"
    return read_buffer
Пример #16
0
def io_from_exported_object(exported_obj: Core, buffer_size: int, offset: int):
    read_buffer = Data(buffer_size)
    io = exported_obj.new_io(exported_obj.cache.get_default_queue(), offset,
                             read_buffer.size, IoDir.READ, 0, 0)
    io.set_data(read_buffer)

    completion = OcfCompletion([("err", c_int)])
    io.callback = completion.callback
    io.submit()
    completion.wait()

    assert completion.results["err"] == 0, "IO from exported object completion"
    return read_buffer
Пример #17
0
def test_neg_read_too_long_data(pyocf_ctx, c_uint16_randomize):
    """
        Check if reading data larger than exported object size is properly blocked
    """

    core = prepare_cache_and_core(Size.from_MiB(1))
    data = Data(int(Size.from_KiB(c_uint16_randomize)))
    completion = io_operation(core, data, IoDir.READ)

    if c_uint16_randomize > 1024:
        assert completion.results["err"] != 0
    else:
        assert completion.results["err"] == 0
Пример #18
0
def test_neg_write_offset_outside_of_device(pyocf_ctx, c_int_randomize):
    """
        Check that write operations are blocked when
        IO offset is located outside of device range
    """

    core = prepare_cache_and_core(Size.from_MiB(2))
    data = Data(int(Size.from_KiB(1)))
    completion = io_operation(core, data, IoDir.WRITE, offset=c_int_randomize)

    if 0 <= c_int_randomize <= int(Size.from_MiB(2)) - int(Size.from_KiB(1)):
        assert completion.results["err"] == 0
    else:
        assert completion.results["err"] != 0
Пример #19
0
def test_neg_io_direction(pyocf_ctx, c_int_randomize):
    """
        Check that IO operations are not executed for unknown IO direction,
        that is when IO direction value is not in allowed values {0, 1}
    """

    core = prepare_cache_and_core(Size.from_MiB(2))
    data = Data(int(Size.from_MiB(1)))
    completion = io_operation(core, data, c_int_randomize)

    if c_int_randomize in [0, 1]:
        assert completion.results["err"] == 0
    else:
        assert completion.results["err"] != 0
Пример #20
0
def test_neg_io_class(pyocf_ctx, c_int_randomize):
    """
        Check that IO operations are blocked when IO class
        number is not in allowed values {0, ..., 32}
    """

    core = prepare_cache_and_core(Size.from_MiB(2))
    data = Data(int(Size.from_MiB(1)))
    completion = io_operation(core, data, randrange(0, 2), io_class=c_int_randomize)

    if 0 <= c_int_randomize <= 32:
        assert completion.results["err"] == 0
    else:
        assert completion.results["err"] != 0
Пример #21
0
def run_io_and_cache_data_if_possible(exported_obj, mode, cls, cls_no):
    test_data = Data(cls_no * cls)

    if mode in {CacheMode.WI, CacheMode.WA}:
        logger.info("[STAGE] Write to core device")
        io_to_core(exported_obj, test_data, 0, True)
        logger.info("[STAGE] Read from exported object")
        io_from_exported_object(exported_obj, test_data.size, 0)
    else:
        logger.info("[STAGE] Write to exported object")
        io_to_core(exported_obj, test_data, 0)

    stats = exported_obj.cache.get_stats()
    assert stats["usage"]["occupancy"]["value"] == \
        ((cls_no * cls / CacheLineSize.LINE_4KiB) if mode != CacheMode.PT else 0), "Occupancy"
Пример #22
0
def test_neg_read_too_far(pyocf_ctx, c_uint16_randomize):
    """
        Check if reading data which would normally fit on exported object is
        blocked when offset is set so that data is read beyond exported device end
    """

    limited_size = c_uint16_randomize % (int(Size.from_KiB(4)) + 1)
    core = prepare_cache_and_core(Size.from_MiB(4))
    data = Data(int(Size.from_KiB(limited_size)))
    completion = io_operation(core, data, IoDir.READ, offset=(Size.from_MiB(3)))

    if limited_size > 1024:
        assert completion.results["err"] != 0
    else:
        assert completion.results["err"] == 0
Пример #23
0
def test_neg_write_too_far(pyocf_ctx, c_uint16_randomize):
    """
        Check if writing data which would normally fit on exported object is
        blocked when offset is set so that data goes over exported device end
    """

    limited_size = c_uint16_randomize % (int(Size.from_KiB(4)) + 1)
    vol, queue = prepare_cache_and_core(Size.from_MiB(4))
    data = Data(int(Size.from_KiB(limited_size)))
    completion = io_operation(vol, queue, data, IoDir.WRITE,
                              int(Size.from_MiB(3)))

    if limited_size > 1024:
        assert completion.results["err"] != 0
    else:
        assert completion.results["err"] == 0
Пример #24
0
        def run(self):
            iogen = IoGen(
                (self.jobspec.offset, self.jobspec.size),
                self.jobspec.bs,
                self.jobspec.randseed,
                self.jobspec.readwrite.is_random(),
                self.jobspec.randommap,
            )

            if self.jobspec.time_based:
                self.finish_time = datetime.now() + self.jobspec.time
            else:
                if int(self.jobspec.io_size) != 0:
                    self.io_target = min(
                        self.jobspec.io_size,
                        self.jobspec.size - self.jobspec.offset)
                else:
                    self.io_target = self.jobspec.size - self.jobspec.offset

            # TODO randrw
            iodir = (IoDir.WRITE if self.jobspec.readwrite
                     in [ReadWrite.WRITE, ReadWrite.RANDWRITE] else IoDir.READ)

            while not self.should_finish():
                with self.qd_condition:
                    self.qd_condition.wait_for(
                        lambda: self.qd <= self.jobspec.qd)

                data = Data(self.jobspec.bs)  # TODO pattern and verify
                io = self.jobspec.target.new_io(
                    self.queue,
                    next(iogen),
                    self.jobspec.bs,
                    iodir,
                    0,
                    0,
                )
                io.set_data(data)
                io.callback = self.get_io_cb()
                self.ios += self.jobspec.bs
                io.submit()
                with self.qd_condition:
                    self.qd += 1

            with self.qd_condition:
                self.qd_condition.wait_for(lambda: self.qd == 0)
Пример #25
0
def run_io_and_cache_data_if_possible(core, mode, cls, cls_no):
    front_vol = core.get_front_volume()
    bottom_vol = core.get_volume()
    queue = core.cache.get_default_queue()

    test_data = Data(cls_no * cls)

    if mode in {CacheMode.WI, CacheMode.WA}:
        logger.info("[STAGE] Write to core device")
        io_to_core(bottom_vol, queue, test_data, 0)
        logger.info("[STAGE] Read from exported object")
        io_from_exported_object(front_vol, queue, test_data.size, 0)
    else:
        logger.info("[STAGE] Write to exported object")
        io_to_core(front_vol, queue, test_data, 0)

    stats = core.cache.get_stats()
    assert stats["usage"]["occupancy"]["value"] == \
        ((cls_no * cls / CacheLineSize.LINE_4KiB) if mode != CacheMode.PT else 0), "Occupancy"
Пример #26
0
def test_load_cache_with_cores(pyocf_ctx, open_cores):
    cache_device = RamVolume(S.from_MiB(40))
    core_device = RamVolume(S.from_MiB(40))

    cache = Cache.start_on_device(cache_device)
    core = Core.using_device(core_device, name="test_core")

    cache.add_core(core)
    vol = CoreVolume(core, open=True)

    write_data = Data.from_string("This is test data")
    io = vol.new_io(cache.get_default_queue(),
                    S.from_sector(3).B, write_data.size, IoDir.WRITE, 0, 0)
    io.set_data(write_data)

    cmpl = OcfCompletion([("err", c_int)])
    io.callback = cmpl.callback
    io.submit()
    cmpl.wait()

    cache.stop()

    cache = Cache.load_from_device(cache_device, open_cores=open_cores)
    if not open_cores:
        cache.add_core(core, try_add=True)
    else:
        core = cache.get_core_by_name("test_core")

    vol = CoreVolume(core, open=True)

    read_data = Data(write_data.size)
    io = vol.new_io(cache.get_default_queue(),
                    S.from_sector(3).B, read_data.size, IoDir.READ, 0, 0)
    io.set_data(read_data)

    cmpl = OcfCompletion([("err", c_int)])
    io.callback = cmpl.callback
    io.submit()
    cmpl.wait()

    assert read_data.md5() == write_data.md5()
    assert vol.md5() == core_device.md5()
Пример #27
0
def test_remove_dirty_no_flush(pyocf_ctx, cache_mode, cls):
    # Start cache device
    cache_device = Volume(S.from_MiB(30))
    cache = Cache.start_on_device(cache_device,
                                  cache_mode=cache_mode,
                                  cache_line_size=cls)

    # Create core device
    core_device = Volume(S.from_MiB(10))
    core = Core.using_device(core_device)
    cache.add_core(core)

    # Prepare data
    core_size = core.get_stats()["size"]
    data = Data(core_size.B)

    _io_to_core(core, data)

    # Remove core from cache
    cache.remove_core(core)
Пример #28
0
def test_change_to_nhit_and_back_io_in_flight(pyocf_ctx):
    """
    Try switching promotion policy during io, no io's should return with error

    1. Create core/cache pair with promotion policy ALWAYS
    2. Issue IOs without waiting for completion
    3. Change promotion policy to NHIT
    4. Wait for IO completions
        * no IOs should fail
    5. Issue IOs without waiting for completion
    6. Change promotion policy to ALWAYS
    7. Wait for IO completions
        * no IOs should fail
    """

    # Step 1
    cache_device = Volume(Size.from_MiB(30))
    core_device = Volume(Size.from_MiB(30))

    cache = Cache.start_on_device(cache_device)
    core = Core.using_device(core_device)

    cache.add_core(core)

    # Step 2
    completions = []
    for i in range(2000):
        comp = OcfCompletion([("error", c_int)])
        write_data = Data(4096)
        io = core.new_io(cache.get_default_queue(), i * 4096, write_data.size,
                         IoDir.WRITE, 0, 0)
        completions += [comp]
        io.set_data(write_data)
        io.callback = comp.callback
        io.submit()

    # Step 3
    cache.set_promotion_policy(PromotionPolicy.NHIT)

    # Step 4
    for c in completions:
        c.wait()
        assert not c.results[
            "error"], "No IO's should fail when turning NHIT policy on"

    # Step 5
    completions = []
    for i in range(2000):
        comp = OcfCompletion([("error", c_int)])
        write_data = Data(4096)
        io = core.new_io(cache.get_default_queue(), i * 4096, write_data.size,
                         IoDir.WRITE, 0, 0)
        completions += [comp]
        io.set_data(write_data)
        io.callback = comp.callback
        io.submit()

    # Step 6
    cache.set_promotion_policy(PromotionPolicy.ALWAYS)

    # Step 7
    for c in completions:
        c.wait()
        assert not c.results[
            "error"], "No IO's should fail when turning NHIT policy off"
Пример #29
0
def test_promoted_after_hits_various_thresholds(pyocf_ctx, insertion_threshold,
                                                fill_percentage):
    """
    Check promotion policy behavior with various set thresholds

    1. Create core/cache pair with promotion policy NHIT
    2. Set TRIGGER_THRESHOLD/INSERTION_THRESHOLD to predefined values
    3. Fill cache from the beggining until occupancy reaches TRIGGER_THRESHOLD%
    4. Issue INSERTION_THRESHOLD - 1 requests to core line not inserted to cache
        * occupancy should not change
    5. Issue one request to LBA from step 4
        * occupancy should rise by one cache line
    """

    # Step 1
    cache_device = Volume(Size.from_MiB(30))
    core_device = Volume(Size.from_MiB(30))

    cache = Cache.start_on_device(cache_device,
                                  promotion_policy=PromotionPolicy.NHIT)
    core = Core.using_device(core_device)
    cache.add_core(core)

    # Step 2
    cache.set_promotion_policy_param(PromotionPolicy.NHIT,
                                     NhitParams.TRIGGER_THRESHOLD,
                                     fill_percentage)
    cache.set_promotion_policy_param(PromotionPolicy.NHIT,
                                     NhitParams.INSERTION_THRESHOLD,
                                     insertion_threshold)
    # Step 3
    fill_cache(cache, fill_percentage / 100)

    stats = cache.get_stats()
    cache_lines = stats["conf"]["size"]
    assert stats["usage"]["occupancy"]["fraction"] // 10 == fill_percentage * 10
    filled_occupancy = stats["usage"]["occupancy"]["value"]

    # Step 4
    last_core_line = int(core_device.size) - cache_lines.line_size
    completions = []
    for i in range(insertion_threshold - 1):
        comp = OcfCompletion([("error", c_int)])
        write_data = Data(cache_lines.line_size)
        io = core.new_io(
            cache.get_default_queue(),
            last_core_line,
            write_data.size,
            IoDir.WRITE,
            0,
            0,
        )
        completions += [comp]
        io.set_data(write_data)
        io.callback = comp.callback
        io.submit()

    for c in completions:
        c.wait()

    stats = cache.get_stats()
    threshold_reached_occupancy = stats["usage"]["occupancy"]["value"]
    assert threshold_reached_occupancy == filled_occupancy, (
        "No insertion should occur while NHIT is triggered and core line ",
        "didn't reach INSERTION_THRESHOLD",
    )

    # Step 5
    comp = OcfCompletion([("error", c_int)])
    write_data = Data(cache_lines.line_size)
    io = core.new_io(cache.get_default_queue(), last_core_line,
                     write_data.size, IoDir.WRITE, 0, 0)
    io.set_data(write_data)
    io.callback = comp.callback
    io.submit()

    comp.wait()

    assert (threshold_reached_occupancy ==
            cache.get_stats()["usage"]["occupancy"]["value"] -
            1), "Previous request should be promoted and occupancy should rise"
Пример #30
0
def test_evict_overflown_pinned(pyocf_ctx, cls: CacheLineSize):
    """ Verify if overflown pinned ioclass is evicted """
    cache_device = Volume(Size.from_MiB(35))
    core_device = Volume(Size.from_MiB(100))
    cache = Cache.start_on_device(cache_device,
                                  cache_mode=CacheMode.WT,
                                  cache_line_size=cls)
    core = Core.using_device(core_device)
    cache.add_core(core)

    test_ioclass_id = 1
    pinned_ioclass_id = 2
    pinned_ioclass_max_occupancy = 10

    cache.configure_partition(
        part_id=test_ioclass_id,
        name="default_ioclass",
        max_size=100,
        priority=1,
    )
    cache.configure_partition(
        part_id=pinned_ioclass_id,
        name="pinned_ioclass",
        max_size=pinned_ioclass_max_occupancy,
        priority=-1,
    )

    cache.set_seq_cut_off_policy(SeqCutOffPolicy.NEVER)

    cache_size = cache.get_stats()["conf"]["size"]

    data = Data(4096)

    # Populate cache with data
    for i in range(cache_size.blocks_4k):
        send_io(core, data, i * 4096, test_ioclass_id)

    part_current_size = CacheLines(
        cache.get_partition_info(part_id=test_ioclass_id)["_curr_size"], cls)
    assert isclose(
        part_current_size.blocks_4k,
        cache_size.blocks_4k,
        abs_tol=Size(
            cls).blocks_4k), "Failed to populate the default partition"

    # Repart - force overflow of second partition occupancy limit
    pinned_double_size = ceil(
        (cache_size.blocks_4k * pinned_ioclass_max_occupancy * 2) / 100)
    for i in range(pinned_double_size):
        send_io(core, data, i * 4096, pinned_ioclass_id)

    part_current_size = CacheLines(
        cache.get_partition_info(part_id=pinned_ioclass_id)["_curr_size"], cls)
    assert isclose(
        part_current_size.blocks_4k,
        pinned_double_size,
        abs_tol=Size(cls).blocks_4k
    ), "Occupancy of pinned ioclass doesn't match expected value"

    # Trigger IO to the default ioclass - force eviction from overlown ioclass
    for i in range(cache_size.blocks_4k):
        send_io(core, data, (cache_size.blocks_4k + i) * 4096, test_ioclass_id)

    part_current_size = CacheLines(
        cache.get_partition_info(part_id=pinned_ioclass_id)["_curr_size"], cls)
    assert isclose(
        part_current_size.blocks_4k,
        ceil(cache_size.blocks_4k * 0.1),
        abs_tol=Size(cls).blocks_4k,
    ), "Overflown part has not been evicted"