def test_region(self):
fc = FileCreator(self.k_window_test_size, "window_test")
half_size = fc.size // 2
rofs = align_to_mmap(4200, False)
rfull = MapRegion(fc.path, 0, fc.size)
rhalfofs = MapRegion(fc.path, rofs, fc.size)
rhalfsize = MapRegion(fc.path, 0, half_size)
# offsets
assert rfull.ofs_begin() == 0 and rfull.size() == fc.size
assert rfull.ofs_end() == fc.size # if this method works, it works always
assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs
assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size
assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size - 1) and rfull.includes_ofs(half_size)
assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize)
# auto-refcount
assert rfull.client_count() == 1
rfull2 = rfull
assert rfull.client_count() == 1, "no auto-counting"
# window constructor
w = MapWindow.from_region(rfull)
assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end()
def test_region(self):
fc = FileCreator(self.k_window_test_size, "window_test")
half_size = fc.size // 2
rofs = align_to_mmap(4200, False)
rfull = MapRegion(fc.path, 0, fc.size)
rhalfofs = MapRegion(fc.path, rofs, fc.size)
rhalfsize = MapRegion(fc.path, 0, half_size)
# offsets
assert rfull.ofs_begin() == 0 and rfull.size() == fc.size
assert rfull.ofs_end() == fc.size # if this method works, it works always
assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs
assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size
assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size - 1) and rfull.includes_ofs(half_size)
assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize)
# with the values we have, this test only works on windows where an alignment
# size of 4096 is assumed.
# We only test on linux as it is inconsitent between the python versions
# as they use different mapping techniques to circumvent the missing offset
# argument of mmap.
if sys.platform != 'win32':
assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0)
# END handle platforms
# auto-refcount
assert rfull.client_count() == 1
rfull2 = rfull
assert rfull.client_count() == 1, "no auto-counting"
# window constructor
w = MapWindow.from_region(rfull)
assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end()
def test_window(self):
wl = MapWindow(0, 1) # left
wc = MapWindow(1, 1) # center
wc2 = MapWindow(10, 5) # another center
wr = MapWindow(8000, 50) # right
assert wl.ofs_end() == 1
assert wc.ofs_end() == 2
assert wr.ofs_end() == 8050
# extension does nothing if already in place
maxsize = 100
wc.extend_left_to(wl, maxsize)
assert wc.ofs == 1 and wc.size == 1
wl.extend_right_to(wc, maxsize)
wl.extend_right_to(wc, maxsize)
assert wl.ofs == 0 and wl.size == 1
# an actual left extension
pofs_end = wc2.ofs_end()
wc2.extend_left_to(wc, maxsize)
assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end()
# respects maxsize
wc.extend_right_to(wr, maxsize)
assert wc.ofs == 1 and wc.size == maxsize
wc.extend_right_to(wr, maxsize)
assert wc.ofs == 1 and wc.size == maxsize
# without maxsize
wc.extend_right_to(wr, sys.maxsize)
assert wc.ofs_end() == wr.ofs and wc.ofs == 1
# extend left
wr.extend_left_to(wc2, maxsize)
wr.extend_left_to(wc2, maxsize)
assert wr.size == maxsize
wr.extend_left_to(wc2, sys.maxsize)
assert wr.ofs == wc2.ofs_end()
wc.align()
assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True)
def test_window(self):
wl = MapWindow(0, 1) # left
wc = MapWindow(1, 1) # center
wc2 = MapWindow(10, 5) # another center
wr = MapWindow(8000, 50) # right
assert wl.ofs_end() == 1
assert wc.ofs_end() == 2
assert wr.ofs_end() == 8050
# extension does nothing if already in place
maxsize = 100
wc.extend_left_to(wl, maxsize)
assert wc.ofs == 1 and wc.size == 1
wl.extend_right_to(wc, maxsize)
wl.extend_right_to(wc, maxsize)
assert wl.ofs == 0 and wl.size == 1
# an actual left extension
pofs_end = wc2.ofs_end()
wc2.extend_left_to(wc, maxsize)
assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end()
# respects maxsize
wc.extend_right_to(wr, maxsize)
assert wc.ofs == 1 and wc.size == maxsize
wc.extend_right_to(wr, maxsize)
assert wc.ofs == 1 and wc.size == maxsize
# without maxsize
wc.extend_right_to(wr, sys.maxsize)
assert wc.ofs_end() == wr.ofs and wc.ofs == 1
# extend left
wr.extend_left_to(wc2, maxsize)
wr.extend_left_to(wc2, maxsize)
assert wr.size == maxsize
wr.extend_left_to(wc2, sys.maxsize)
assert wr.ofs == wc2.ofs_end()
wc.align()
assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True)
def test_region(self):
fc = FileCreator(self.k_window_test_size, "window_test")
half_size = fc.size // 2
rofs = align_to_mmap(4200, False)
rfull = MapRegion(fc.path, 0, fc.size)
rhalfofs = MapRegion(fc.path, rofs, fc.size)
rhalfsize = MapRegion(fc.path, 0, half_size)
# offsets
assert rfull.ofs_begin() == 0 and rfull.size() == fc.size
assert rfull.ofs_end(
) == fc.size # if this method works, it works always
assert rhalfofs.ofs_begin() == rofs and rhalfofs.size(
) == fc.size - rofs
assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size
assert rfull.includes_ofs(0) and rfull.includes_ofs(
fc.size - 1) and rfull.includes_ofs(half_size)
assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(
sys.maxsize)
# with the values we have, this test only works on windows where an alignment
# size of 4096 is assumed.
# We only test on linux as it is inconsitent between the python versions
# as they use different mapping techniques to circumvent the missing offset
# argument of mmap.
if sys.platform != 'win32':
assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0)
# END handle platforms
# auto-refcount
assert rfull.client_count() == 1
rfull2 = rfull
assert rfull.client_count() == 1, "no auto-counting"
# window constructor
w = MapWindow.from_region(rfull)
assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end()
def test_util(self):
assert isinstance(is_64_bit(), bool) # just call it
assert align_to_mmap(1, False) == 0
assert align_to_mmap(1, True) == ALLOCATIONGRANULARITY
def test_memman_operation(self):
# test more access, force it to actually unmap regions
fc = FileCreator(self.k_window_test_size, "manager_operation_test")
with open(fc.path, 'rb') as fp:
data = fp.read()
fd = os.open(fc.path, os.O_RDONLY)
max_num_handles = 15
#small_size =
for mtype, args in ( (StaticWindowMapManager, (0, fc.size // 3, max_num_handles)),
(SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),):
for item in (fc.path, fd):
assert len(data) == fc.size
# small windows, a reasonable max memory. Not too many regions at once
man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2])
c = man.make_cursor(item)
# still empty (more about that is tested in test_memory_manager()
assert man.num_open_files() == 0
assert man.mapped_memory_size() == 0
base_offset = 5000
# window size is 0 for static managers, hence size will be 0. We take that into consideration
size = man.window_size() // 2
assert c.use_region(base_offset, size).is_valid()
rr = c.region_ref()
assert rr().client_count() == 2 # the manager and the cursor and us
assert man.num_open_files() == 1
assert man.num_file_handles() == 1
assert man.mapped_memory_size() == rr().size()
#assert c.size() == size # the cursor may overallocate in its static version
assert c.ofs_begin() == base_offset
assert rr().ofs_begin() == 0 # it was aligned and expanded
if man.window_size():
assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned)
else:
assert rr().size() == fc.size
#END ignore static managers which dont use windows and are aligned to file boundaries
assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())]
# obtain second window, which spans the first part of the file - it is a still the same window
nsize = (size or fc.size) - 10
assert c.use_region(0, nsize).is_valid()
assert c.region_ref()() == rr()
assert man.num_file_handles() == 1
assert c.size() == nsize
assert c.ofs_begin() == 0
assert c.buffer()[:] == data[:nsize]
# map some part at the end, our requested size cannot be kept
overshoot = 4000
base_offset = fc.size - (size or c.size()) + overshoot
assert c.use_region(base_offset, size).is_valid()
if man.window_size():
assert man.num_file_handles() == 2
assert c.size() < size
assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore
assert rr().client_count() == 1 # only held by manager
else:
assert c.size() < fc.size
#END ignore static managers which only have one handle per file
rr = c.region_ref()
assert rr().client_count() == 2 # manager + cursor
assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left
assert rr().ofs_end() <= fc.size # it cannot be larger than the file
assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())]
# unising a region makes the cursor invalid
c.unuse_region()
assert not c.is_valid()
if man.window_size():
# but doesn't change anything regarding the handle count - we cache it and only
# remove mapped regions if we have to
assert man.num_file_handles() == 2
#END ignore this for static managers
# iterate through the windows, verify data contents
# this will trigger map collection after a while
max_random_accesses = 5000
num_random_accesses = max_random_accesses
memory_read = 0
st = time()
# cache everything to get some more performance
includes_ofs = c.includes_ofs
max_mapped_memory_size = man.max_mapped_memory_size()
max_file_handles = man.max_file_handles()
mapped_memory_size = man.mapped_memory_size
num_file_handles = man.num_file_handles
while num_random_accesses:
num_random_accesses -= 1
base_offset = randint(0, fc.size - 1)
# precondition
if man.window_size():
assert max_mapped_memory_size >= mapped_memory_size()
#END statics will overshoot, which is fine
assert max_file_handles >= num_file_handles()
assert c.use_region(base_offset, (size or c.size())).is_valid()
csize = c.size()
assert c.buffer()[:] == data[base_offset:base_offset+csize]
memory_read += csize
assert includes_ofs(base_offset)
assert includes_ofs(base_offset+csize-1)
assert not includes_ofs(base_offset+csize)
# END while we should do an access
elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows
mb = float(1000 * 1000)
print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n"
% (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed),
file=sys.stderr)
# an offset as large as the size doesn't work !
assert not c.use_region(fc.size, size).is_valid()
# collection - it should be able to collect all
assert man.num_file_handles()
assert man.collect()
assert man.num_file_handles() == 0
#END for each item
# END for each manager type
os.close(fd)
def test_memman_operation(self):
# test more access, force it to actually unmap regions
fc = FileCreator(self.k_window_test_size, "manager_operation_test")
with open(fc.path, 'rb') as fp:
data = fp.read()
fd = os.open(fc.path, os.O_RDONLY)
max_num_handles = 15
# small_size =
for mtype, args in ((StaticWindowMapManager, (0, fc.size // 3, max_num_handles)),
(SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),):
for item in (fc.path, fd):
assert len(data) == fc.size
# small windows, a reasonable max memory. Not too many regions at once
man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2])
c = man.make_cursor(item)
# still empty (more about that is tested in test_memory_manager()
assert man.num_open_files() == 0
assert man.mapped_memory_size() == 0
base_offset = 5000
# window size is 0 for static managers, hence size will be 0. We take that into consideration
size = man.window_size() // 2
assert c.use_region(base_offset, size).is_valid()
rr = c.region()
assert rr.client_count() == 2 # the manager and the cursor and us
assert man.num_open_files() == 1
assert man.num_file_handles() == 1
assert man.mapped_memory_size() == rr.size()
# assert c.size() == size # the cursor may overallocate in its static version
assert c.ofs_begin() == base_offset
assert rr.ofs_begin() == 0 # it was aligned and expanded
if man.window_size():
# but isn't larger than the max window (aligned)
assert rr.size() == align_to_mmap(man.window_size(), True)
else:
assert rr.size() == fc.size
# END ignore static managers which dont use windows and are aligned to file boundaries
assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())]
# obtain second window, which spans the first part of the file - it is a still the same window
nsize = (size or fc.size) - 10
assert c.use_region(0, nsize).is_valid()
assert c.region() == rr
assert man.num_file_handles() == 1
assert c.size() == nsize
assert c.ofs_begin() == 0
assert c.buffer()[:] == data[:nsize]
# map some part at the end, our requested size cannot be kept
overshoot = 4000
base_offset = fc.size - (size or c.size()) + overshoot
assert c.use_region(base_offset, size).is_valid()
if man.window_size():
assert man.num_file_handles() == 2
assert c.size() < size
assert c.region() is not rr # old region is still available, but has not curser ref anymore
assert rr.client_count() == 1 # only held by manager
else:
assert c.size() < fc.size
# END ignore static managers which only have one handle per file
rr = c.region()
assert rr.client_count() == 2 # manager + cursor
assert rr.ofs_begin() < c.ofs_begin() # it should have extended itself to the left
assert rr.ofs_end() <= fc.size # it cannot be larger than the file
assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())]
# unising a region makes the cursor invalid
c.unuse_region()
assert not c.is_valid()
if man.window_size():
# but doesn't change anything regarding the handle count - we cache it and only
# remove mapped regions if we have to
assert man.num_file_handles() == 2
# END ignore this for static managers
# iterate through the windows, verify data contents
# this will trigger map collection after a while
max_random_accesses = 5000
num_random_accesses = max_random_accesses
memory_read = 0
st = time()
# cache everything to get some more performance
includes_ofs = c.includes_ofs
max_mapped_memory_size = man.max_mapped_memory_size()
max_file_handles = man.max_file_handles()
mapped_memory_size = man.mapped_memory_size
num_file_handles = man.num_file_handles
while num_random_accesses:
num_random_accesses -= 1
base_offset = randint(0, fc.size - 1)
# precondition
if man.window_size():
assert max_mapped_memory_size >= mapped_memory_size()
# END statics will overshoot, which is fine
assert max_file_handles >= num_file_handles()
assert c.use_region(base_offset, (size or c.size())).is_valid()
csize = c.size()
assert c.buffer()[:] == data[base_offset:base_offset + csize]
memory_read += csize
assert includes_ofs(base_offset)
assert includes_ofs(base_offset + csize - 1)
assert not includes_ofs(base_offset + csize)
# END while we should do an access
elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows
mb = float(1000 * 1000)
print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n"
% (mtype, memory_read / mb, max_random_accesses, type(item), elapsed, (memory_read / mb) / elapsed),
file=sys.stderr)
# an offset as large as the size doesn't work !
assert not c.use_region(fc.size, size).is_valid()
# collection - it should be able to collect all
assert man.num_file_handles()
assert man.collect()
assert man.num_file_handles() == 0
# END for each item
# END for each manager type
os.close(fd)