def lock_files(handles_dir):
with pl.InterProcessLock(lock_path):
# Open some files we can use for locking
handles = []
for n in range(50):
path = os.path.join(handles_dir, ('file-%s' % n))
handles.append(open(path, 'w'))
# Loop over all the handles and try locking the file
# without blocking, keep a count of how many files we
# were able to lock and then unlock. If the lock fails
# we get an IOError and bail out with bad exit code
count = 0
for handle in handles:
try:
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
count += 1
fcntl.flock(handle, fcntl.LOCK_UN)
except IOError:
os._exit(2)
finally:
handle.close()
# Check if we were able to open all files
self.assertEqual(50, count)
def test_non_destructive(lock_dir):
lock_file = os.path.join(lock_dir, 'not-destroyed')
with open(lock_file, 'w') as f:
f.write('test')
with pl.InterProcessLock(lock_file):
with open(lock_file) as f:
assert f.read() == 'test'
def test_interprocess_lock(lock_dir):
lock_file = os.path.join(lock_dir, 'lock')
lock_name = 'foo'
child_pipe, them = multiprocessing.Pipe()
child = multiprocessing.Process(target=inter_processlock_helper,
args=(lock_name, lock_file, them))
with scoped_child_processes((child, )):
# Make sure the child grabs the lock first
if not child_pipe.poll(5):
pytest.fail('Timed out waiting for child to grab lock')
start = time.time()
lock1 = pl.InterProcessLock(lock_name)
lock1.lockfile = open(lock_file, 'w')
# NOTE(bnemec): There is a brief window between when the lock file
# is created and when it actually becomes locked. If we happen to
# context switch in that window we may succeed in locking the
# file. Keep retrying until we either get the expected exception
# or timeout waiting.
while time.time() - start < 5:
try:
lock1.trylock()
lock1.unlock()
time.sleep(0)
except IOError:
# This is what we expect to happen
break
else:
pytest.fail('Never caught expected lock exception')
child_pipe.send(None)
def test_non_destructive(self):
lock_file = os.path.join(self.lock_dir, 'not-destroyed')
with open(lock_file, 'w') as f:
f.write('test')
with pl.InterProcessLock(lock_file):
with open(lock_file) as f:
self.assertEqual(f.read(), 'test')
def test_lock_acquire_release_file_lock(self):
lock_file = os.path.join(self.lock_dir, 'lock')
lock = pl.InterProcessLock(lock_file)
def try_lock():
try:
my_lock = pl.InterProcessLock(lock_file)
my_lock.lockfile = open(lock_file, 'w')
my_lock.trylock()
my_lock.unlock()
os._exit(1)
except IOError:
os._exit(0)
def attempt_acquire(count):
children = []
for i in range(count):
child = multiprocessing.Process(target=try_lock)
child.start()
children.append(child)
exit_codes = []
for child in children:
child.join()
exit_codes.append(child.exitcode)
return sum(exit_codes)
self.assertTrue(lock.acquire())
try:
acquired_children = attempt_acquire(10)
self.assertEqual(0, acquired_children)
finally:
lock.release()
acquired_children = attempt_acquire(5)
self.assertNotEqual(0, acquired_children)
def _lock_files(lock_path, handles_dir, num_handles=50):
with pl.InterProcessLock(lock_path):
# Open some files we can use for locking
handles = []
for n in range(num_handles):
path = os.path.join(handles_dir, ('file-%s' % n))
handles.append(open(path, 'w'))
# Loop over all the handles and try locking the file
# without blocking, keep a count of how many files we
# were able to lock and then unlock. If the lock fails
# we get an IOError and bail out with bad exit code
count = 0
for handle in handles:
try:
pl.InterProcessLock._trylock(handle)
count += 1
pl.InterProcessLock._unlock(handle)
except IOError:
sys.exit(2)
finally:
handle.close()
# Check if we were able to open all files
if count != num_handles:
raise AssertionError("Unable to open all handles")
def acquire(self, pid=None):
self.pidfile_lock = process_lock.InterProcessLock(self.pidfile_lock_path)
if not self.pidfile_lock.acquire():
LOG.error("Unable to acquire pifile_lock %s", self.pidfile_lock_path)
return False
if pid is None:
pid = os.getpid()
if os.path.exists(self.pidfile_path):
try:
with open(self.pidfile_path) as f:
stalepid = int(f.read())
if stalepid in psutil.pids():
raise Error(
"Pidfile %s exists and is not locked but pid %s is still alive"
% (self.pidfile_path, stalepid)
)
except Exception as exc:
LOG.exception("Exception %r checking the stale pidfile, ignoring", exc)
LOG.warning(
"Pidfile exists but is not locked, removing %s", self.pidfile_path
)
os.unlink(self.pidfile_path)
self.pidfile = open(
self.pidfile_path, "x", opener=functools.partial(os.open, mode=0o600)
)
self.pidfile.write(str(pid))
self.pidfile.flush()
return True
def test_interprocess_lock(self):
lock_file = os.path.join(self.lock_dir, 'lock')
pid = os.fork()
if pid:
# Make sure the child grabs the lock first
start = time.time()
while not os.path.exists(lock_file):
if time.time() - start > 5:
self.fail('Timed out waiting for child to grab lock')
time.sleep(0)
lock1 = pl.InterProcessLock('foo')
lock1.lockfile = open(lock_file, 'w')
# NOTE(bnemec): There is a brief window between when the lock file
# is created and when it actually becomes locked. If we happen to
# context switch in that window we may succeed in locking the
# file. Keep retrying until we either get the expected exception
# or timeout waiting.
while time.time() - start < 5:
try:
lock1.trylock()
lock1.unlock()
time.sleep(0)
except IOError:
# This is what we expect to happen
break
else:
self.fail('Never caught expected lock exception')
# We don't need to wait for the full sleep in the child here
os.kill(pid, signal.SIGKILL)
else:
try:
lock2 = pl.InterProcessLock('foo')
lock2.lockfile = open(lock_file, 'w')
have_lock = False
while not have_lock:
try:
lock2.trylock()
have_lock = True
except IOError:
pass
finally:
# NOTE(bnemec): This is racy, but I don't want to add any
# synchronization primitives that might mask a problem
# with the one we're trying to test here.
time.sleep(.5)
os._exit(0)
def try_lock(lock_file):
try:
my_lock = pl.InterProcessLock(lock_file)
my_lock.lockfile = open(lock_file, 'w')
my_lock.trylock()
my_lock.unlock()
sys.exit(1)
except IOError:
sys.exit(0)
def test_lock_twice(lock_dir):
lock_file = os.path.join(lock_dir, 'lock')
lock = pl.InterProcessLock(lock_file)
ok = lock.acquire(blocking=False)
assert ok
# ok on Unix, not ok on Windows
ok = lock.acquire(blocking=False)
assert ok or not ok
# should release without crashing
lock.release()
def inter_processlock_helper(lockname, lock_filename, pipe):
lock2 = pl.InterProcessLock(lockname)
lock2.lockfile = open(lock_filename, 'w')
have_lock = False
while not have_lock:
try:
lock2.trylock()
have_lock = True
except IOError:
pass
# Hold the lock and wait for the parent
pipe.send(None)
pipe.recv()
def test_lock_acquire_release_file_lock(lock_dir):
lock_file = os.path.join(lock_dir, 'lock')
lock = pl.InterProcessLock(lock_file)
def attempt_acquire(count):
children = [
multiprocessing.Process(target=try_lock, args=(lock_file, ))
for i in range(count)
]
with scoped_child_processes(children, timeout=10, exitcode=None):
pass
return sum(c.exitcode for c in children)
assert lock.acquire()
try:
acquired_children = attempt_acquire(10)
assert acquired_children == 0
finally:
lock.release()
acquired_children = attempt_acquire(5)
assert acquired_children != 0
def test_close_fd_on_failed_acquire(self):
'''Tests that the lockfile fd is closed after failing to acquire the
lock (e.g., non-blocking timeout).'''
lock_file = os.path.join(self.lock_dir, 'lock')
lock_name = 'foo'
child_pipe, them = multiprocessing.Pipe()
child = multiprocessing.Process(target=inter_processlock_helper,
args=(lock_name, lock_file, them))
with scoped_child_processes((child, )):
# Make sure the child grabs the lock first
if not child_pipe.poll(5):
self.fail('Timed out waiting for child to grab lock')
# Attempt to lock the file without blocking. This will fail, since
# the child has it.
lock1 = pl.InterProcessLock(lock_name)
lock1.lockfile = open(lock_file, 'w')
self.assertFalse(lock1.acquire(blocking=False))
# Since locking failed, the fd was closed.
self.assertIsNone(lock1.lockfile)
# Ask the child to release the lock.
child_pipe.send(None)
# Acquire the lock, to verify it remains open while locked.
self.assertTrue(lock1.acquire(blocking=False, timeout=0.1))
# The lockfile remains open; necessary while the lock is held.
self.assertIsNotNone(lock1.lockfile)
self.assertFalse(lock1.lockfile.closed)
# Releasing the lock should close the lockfile.
lock1.release()
self.assertTrue(lock1.lockfile is None or lock1.lockfile.closed)
def _get_process_lock(self):
lock = process_lock.InterProcessLock("server.lock")
if not lock.acquire(blocking=False):
raise ChildProcessError("Server process is already running")
def test_bad_release(self):
lock_file = os.path.join(self.lock_dir, 'lock')
lock = pl.InterProcessLock(lock_file)
self.assertRaises(threading.ThreadError, lock.release)
def __init__(self, path=".", *args, **kwargs):
from fasteners import process_lock
self._file = process_lock.InterProcessLock(
os.path.join(path, ".ninja.lock"))
def test_bad_release(lock_dir):
lock_file = os.path.join(lock_dir, 'lock')
lock = pl.InterProcessLock(lock_file)
with pytest.raises(threading.ThreadError):
lock.release()
def __init__(self, path, logfunc=None, *args, **kwargs):
self._file = process_lock.InterProcessLock(os.path.join(path, "lock"))
if not self._file.acquire(blocking=False):
if logfunc is not None:
logfunc(*args, **kwargs)
self._file.acquire()
