def test_mlocking(self):
self.thisHostMust(hasmemlocking=True)
with self.getTestDir() as dirn:
fsize = 8 * 1024
fn = pathlib.Path(dirn) / 'mapfile'
with open(fn, 'wb') as f:
f.write(b'x' * fsize)
f.flush()
with open(fn, 'r+b') as f:
with s_thisplat.mmap(0, fsize, 0x1, 0x8001, f.fileno(), 0):
addr, size = s_thisplat.getFileMappedRegion(fn)
self.ne(addr, 0)
self.eq(size, fsize)
beforelock = s_thisplat.getCurrentLockedMemory()
maxlocked = s_thisplat.getMaxLockedMemory()
self.ge(maxlocked, beforelock)
self.ge(s_thisplat.getTotalMemory(), beforelock)
self.ge(s_thisplat.getAvailableMemory(), beforelock)
s_thisplat.mlock(addr, size)
locktotal = s_thisplat.getCurrentLockedMemory()
self.ge(locktotal, size)
self.ge(locktotal, beforelock)
maxlocked = s_thisplat.getMaxLockedMemory()
self.ge(maxlocked, locktotal)
s_thisplat.munlock(addr, size)
locktotal = s_thisplat.getCurrentLockedMemory()
self.eq(locktotal, beforelock)
# Make sure we get the largest mapped region
with s_thisplat.mmap(0, int(fsize / 2), 0x1, 0x8001, f.fileno(), 0):
addr, size = s_thisplat.getFileMappedRegion(fn)
self.eq(size, fsize)
# Sad tests
bfn = pathlib.Path(dirn) / 'mapfile.newp'
self.raises(s_exc.NoSuchFile, s_thisplat.getFileMappedRegion, bfn)
# Sad tests
with self.raises(OSError) as cm:
s_thisplat.mlock(0x01, 16)
# Cannot allocate memory to lock
self.eq(cm.exception.errno, 12)
with self.raises(OSError) as cm:
s_thisplat.munlock(0xFF, 16)
# Cannot allocate memory to unlock
self.eq(cm.exception.errno, 12)
def _memorylockloop(self):
'''
Separate thread loop that manages the prefaulting and locking of the memory backing the data file
'''
if not s_thishost.get('hasmemlocking'):
return
MAX_TOTAL_PERCENT = .90 # how much of all the RAM to take
MAX_LOCK_AT_ONCE = s_const.gibibyte
# Calculate a reasonable maximum amount of memory to lock
s_thisplat.maximizeMaxLockedMemory()
locked_ulimit = s_thisplat.getMaxLockedMemory()
if locked_ulimit < s_const.gibibyte // 2:
logger.warning(
'Operating system limit of maximum amount of locked memory (currently %d) is \n'
'too low for optimal performance.', locked_ulimit)
logger.debug('memory locking thread started')
# Note: available might be larger than max_total in a container
max_total = s_thisplat.getTotalMemory()
available = s_thisplat.getAvailableMemory()
PAGESIZE = 4096
max_to_lock = (min(locked_ulimit, int(max_total * MAX_TOTAL_PERCENT),
int(available * MAX_TOTAL_PERCENT)) //
PAGESIZE) * PAGESIZE
self.max_could_lock = max_to_lock
path = self.path.absolute(
) / 'data.mdb' # Path to the file that gets mapped
fh = open(path, 'r+b')
fileno = fh.fileno()
prev_memend = 0 # The last end of the file mapping, so we can start from there
# Avoid spamming messages
first_end = True
limit_warned = False
self.locking_memory = True
self.resizeevent.set()
while not self.isfini:
self.resizeevent.wait()
if self.isfini:
break
self.schedCallSafe(self.lockdoneevent.clear)
self.resizeevent.clear()
try:
memstart, memlen = s_thisplat.getFileMappedRegion(path)
except s_exc.NoSuchFile:
logger.warning('map not found for %s', path)
if not self.resizeevent.is_set():
self.schedCallSafe(self.lockdoneevent.set)
continue
if memlen > max_to_lock:
memlen = max_to_lock
if not limit_warned:
logger.warning('memory locking limit reached')
limit_warned = True
# Even in the event that we've hit our limit we still have to loop because further mmaps may cause
# the base address to change, necessitating relocking what we can
# The file might be a little bit smaller than the map because rounding (and mmap fails if you give it a
# too-long length)
filesize = os.fstat(fileno).st_size
goal_end = memstart + min(memlen, filesize)
self.lock_goal = goal_end - memstart
self.lock_progress = 0
prev_memend = memstart
# Actually do the prefaulting and locking. Only do it a chunk at a time to maintain responsiveness.
while prev_memend < goal_end:
new_memend = min(prev_memend + MAX_LOCK_AT_ONCE, goal_end)
memlen = new_memend - prev_memend
PROT = 1 # PROT_READ
FLAGS = 0x8001 # MAP_POPULATE | MAP_SHARED (Linux only) (for fast prefaulting)
try:
self.prefaulting = True
with s_thisplat.mmap(0,
length=new_memend - prev_memend,
prot=PROT,
flags=FLAGS,
fd=fileno,
offset=prev_memend - memstart):
s_thisplat.mlock(prev_memend, memlen)
except OSError as e:
logger.warning(
'error while attempting to lock memory of %s: %s',
path, e)
break
finally:
self.prefaulting = False
prev_memend = new_memend
self.lock_progress = prev_memend - memstart
if first_end:
first_end = False
logger.info('completed prefaulting and locking slab')
if not self.resizeevent.is_set():
self.schedCallSafe(self.lockdoneevent.set)
self.locking_memory = False
logger.debug('memory locking thread ended')