def lock_files(handles_dir): with pl.InterProcessLock(lock_path): # Open some files we can use for locking handles = [] for n in range(50): path = os.path.join(handles_dir, ('file-%s' % n)) handles.append(open(path, 'w')) # Loop over all the handles and try locking the file # without blocking, keep a count of how many files we # were able to lock and then unlock. If the lock fails # we get an IOError and bail out with bad exit code count = 0 for handle in handles: try: fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) count += 1 fcntl.flock(handle, fcntl.LOCK_UN) except IOError: os._exit(2) finally: handle.close() # Check if we were able to open all files self.assertEqual(50, count)
def test_non_destructive(lock_dir): lock_file = os.path.join(lock_dir, 'not-destroyed') with open(lock_file, 'w') as f: f.write('test') with pl.InterProcessLock(lock_file): with open(lock_file) as f: assert f.read() == 'test'
def test_interprocess_lock(lock_dir): lock_file = os.path.join(lock_dir, 'lock') lock_name = 'foo' child_pipe, them = multiprocessing.Pipe() child = multiprocessing.Process(target=inter_processlock_helper, args=(lock_name, lock_file, them)) with scoped_child_processes((child, )): # Make sure the child grabs the lock first if not child_pipe.poll(5): pytest.fail('Timed out waiting for child to grab lock') start = time.time() lock1 = pl.InterProcessLock(lock_name) lock1.lockfile = open(lock_file, 'w') # NOTE(bnemec): There is a brief window between when the lock file # is created and when it actually becomes locked. If we happen to # context switch in that window we may succeed in locking the # file. Keep retrying until we either get the expected exception # or timeout waiting. while time.time() - start < 5: try: lock1.trylock() lock1.unlock() time.sleep(0) except IOError: # This is what we expect to happen break else: pytest.fail('Never caught expected lock exception') child_pipe.send(None)
def test_non_destructive(self): lock_file = os.path.join(self.lock_dir, 'not-destroyed') with open(lock_file, 'w') as f: f.write('test') with pl.InterProcessLock(lock_file): with open(lock_file) as f: self.assertEqual(f.read(), 'test')
def test_lock_acquire_release_file_lock(self): lock_file = os.path.join(self.lock_dir, 'lock') lock = pl.InterProcessLock(lock_file) def try_lock(): try: my_lock = pl.InterProcessLock(lock_file) my_lock.lockfile = open(lock_file, 'w') my_lock.trylock() my_lock.unlock() os._exit(1) except IOError: os._exit(0) def attempt_acquire(count): children = [] for i in range(count): child = multiprocessing.Process(target=try_lock) child.start() children.append(child) exit_codes = [] for child in children: child.join() exit_codes.append(child.exitcode) return sum(exit_codes) self.assertTrue(lock.acquire()) try: acquired_children = attempt_acquire(10) self.assertEqual(0, acquired_children) finally: lock.release() acquired_children = attempt_acquire(5) self.assertNotEqual(0, acquired_children)
def _lock_files(lock_path, handles_dir, num_handles=50): with pl.InterProcessLock(lock_path): # Open some files we can use for locking handles = [] for n in range(num_handles): path = os.path.join(handles_dir, ('file-%s' % n)) handles.append(open(path, 'w')) # Loop over all the handles and try locking the file # without blocking, keep a count of how many files we # were able to lock and then unlock. If the lock fails # we get an IOError and bail out with bad exit code count = 0 for handle in handles: try: pl.InterProcessLock._trylock(handle) count += 1 pl.InterProcessLock._unlock(handle) except IOError: sys.exit(2) finally: handle.close() # Check if we were able to open all files if count != num_handles: raise AssertionError("Unable to open all handles")
def acquire(self, pid=None): self.pidfile_lock = process_lock.InterProcessLock(self.pidfile_lock_path) if not self.pidfile_lock.acquire(): LOG.error("Unable to acquire pifile_lock %s", self.pidfile_lock_path) return False if pid is None: pid = os.getpid() if os.path.exists(self.pidfile_path): try: with open(self.pidfile_path) as f: stalepid = int(f.read()) if stalepid in psutil.pids(): raise Error( "Pidfile %s exists and is not locked but pid %s is still alive" % (self.pidfile_path, stalepid) ) except Exception as exc: LOG.exception("Exception %r checking the stale pidfile, ignoring", exc) LOG.warning( "Pidfile exists but is not locked, removing %s", self.pidfile_path ) os.unlink(self.pidfile_path) self.pidfile = open( self.pidfile_path, "x", opener=functools.partial(os.open, mode=0o600) ) self.pidfile.write(str(pid)) self.pidfile.flush() return True
def test_interprocess_lock(self): lock_file = os.path.join(self.lock_dir, 'lock') pid = os.fork() if pid: # Make sure the child grabs the lock first start = time.time() while not os.path.exists(lock_file): if time.time() - start > 5: self.fail('Timed out waiting for child to grab lock') time.sleep(0) lock1 = pl.InterProcessLock('foo') lock1.lockfile = open(lock_file, 'w') # NOTE(bnemec): There is a brief window between when the lock file # is created and when it actually becomes locked. If we happen to # context switch in that window we may succeed in locking the # file. Keep retrying until we either get the expected exception # or timeout waiting. while time.time() - start < 5: try: lock1.trylock() lock1.unlock() time.sleep(0) except IOError: # This is what we expect to happen break else: self.fail('Never caught expected lock exception') # We don't need to wait for the full sleep in the child here os.kill(pid, signal.SIGKILL) else: try: lock2 = pl.InterProcessLock('foo') lock2.lockfile = open(lock_file, 'w') have_lock = False while not have_lock: try: lock2.trylock() have_lock = True except IOError: pass finally: # NOTE(bnemec): This is racy, but I don't want to add any # synchronization primitives that might mask a problem # with the one we're trying to test here. time.sleep(.5) os._exit(0)
def try_lock(lock_file): try: my_lock = pl.InterProcessLock(lock_file) my_lock.lockfile = open(lock_file, 'w') my_lock.trylock() my_lock.unlock() sys.exit(1) except IOError: sys.exit(0)
def test_lock_twice(lock_dir): lock_file = os.path.join(lock_dir, 'lock') lock = pl.InterProcessLock(lock_file) ok = lock.acquire(blocking=False) assert ok # ok on Unix, not ok on Windows ok = lock.acquire(blocking=False) assert ok or not ok # should release without crashing lock.release()
def inter_processlock_helper(lockname, lock_filename, pipe): lock2 = pl.InterProcessLock(lockname) lock2.lockfile = open(lock_filename, 'w') have_lock = False while not have_lock: try: lock2.trylock() have_lock = True except IOError: pass # Hold the lock and wait for the parent pipe.send(None) pipe.recv()
def test_lock_acquire_release_file_lock(lock_dir): lock_file = os.path.join(lock_dir, 'lock') lock = pl.InterProcessLock(lock_file) def attempt_acquire(count): children = [ multiprocessing.Process(target=try_lock, args=(lock_file, )) for i in range(count) ] with scoped_child_processes(children, timeout=10, exitcode=None): pass return sum(c.exitcode for c in children) assert lock.acquire() try: acquired_children = attempt_acquire(10) assert acquired_children == 0 finally: lock.release() acquired_children = attempt_acquire(5) assert acquired_children != 0
def test_close_fd_on_failed_acquire(self): '''Tests that the lockfile fd is closed after failing to acquire the lock (e.g., non-blocking timeout).''' lock_file = os.path.join(self.lock_dir, 'lock') lock_name = 'foo' child_pipe, them = multiprocessing.Pipe() child = multiprocessing.Process(target=inter_processlock_helper, args=(lock_name, lock_file, them)) with scoped_child_processes((child, )): # Make sure the child grabs the lock first if not child_pipe.poll(5): self.fail('Timed out waiting for child to grab lock') # Attempt to lock the file without blocking. This will fail, since # the child has it. lock1 = pl.InterProcessLock(lock_name) lock1.lockfile = open(lock_file, 'w') self.assertFalse(lock1.acquire(blocking=False)) # Since locking failed, the fd was closed. self.assertIsNone(lock1.lockfile) # Ask the child to release the lock. child_pipe.send(None) # Acquire the lock, to verify it remains open while locked. self.assertTrue(lock1.acquire(blocking=False, timeout=0.1)) # The lockfile remains open; necessary while the lock is held. self.assertIsNotNone(lock1.lockfile) self.assertFalse(lock1.lockfile.closed) # Releasing the lock should close the lockfile. lock1.release() self.assertTrue(lock1.lockfile is None or lock1.lockfile.closed)
def _get_process_lock(self): lock = process_lock.InterProcessLock("server.lock") if not lock.acquire(blocking=False): raise ChildProcessError("Server process is already running")
def test_bad_release(self): lock_file = os.path.join(self.lock_dir, 'lock') lock = pl.InterProcessLock(lock_file) self.assertRaises(threading.ThreadError, lock.release)
def __init__(self, path=".", *args, **kwargs): from fasteners import process_lock self._file = process_lock.InterProcessLock( os.path.join(path, ".ninja.lock"))
def test_bad_release(lock_dir): lock_file = os.path.join(lock_dir, 'lock') lock = pl.InterProcessLock(lock_file) with pytest.raises(threading.ThreadError): lock.release()
def __init__(self, path, logfunc=None, *args, **kwargs): self._file = process_lock.InterProcessLock(os.path.join(path, "lock")) if not self._file.acquire(blocking=False): if logfunc is not None: logfunc(*args, **kwargs) self._file.acquire()