def test_upgrade_read_to_write(private_lock_path):
"""Test that a read lock can be upgraded to a write lock.
Note that to upgrade a read lock to a write lock, you have the be the
only holder of a read lock. Client code needs to coordinate that for
shared locks. For this test, we use a private lock just to test that an
upgrade is possible.
"""
# ensure lock file exists the first time, so we open it read-only
# to begin wtih.
touch(private_lock_path)
lock = Lock(private_lock_path)
assert lock._reads == 0
assert lock._writes == 0
lock.acquire_read()
assert lock._reads == 1
assert lock._writes == 0
assert lock._file.mode == 'r+'
lock.acquire_write()
assert lock._reads == 1
assert lock._writes == 1
assert lock._file.mode == 'r+'
lock.release_write()
assert lock._reads == 1
assert lock._writes == 0
assert lock._file.mode == 'r+'
lock.release_read()
assert lock._reads == 0
assert lock._writes == 0
assert lock._file is None
def test_transaction_with_exception(lock_path):
def enter_fn():
vals['entered'] = True
def exit_fn(t, v, tb):
vals['exited'] = True
vals['exception'] = (t or v or tb)
lock = Lock(lock_path)
def do_read_with_exception():
with ReadTransaction(lock, enter_fn, exit_fn):
raise Exception()
def do_write_with_exception():
with WriteTransaction(lock, enter_fn, exit_fn):
raise Exception()
vals = {'entered': False, 'exited': False, 'exception': False}
with pytest.raises(Exception):
do_read_with_exception()
assert vals['entered']
assert vals['exited']
assert vals['exception']
vals = {'entered': False, 'exited': False, 'exception': False}
with pytest.raises(Exception):
do_write_with_exception()
assert vals['entered']
assert vals['exited']
assert vals['exception']
def __init__(self, root, db_dir=None):
"""Create a Database for Spack installations under ``root``.
A Database is a cache of Specs data from ``$prefix/spec.yaml``
files in Spack installation directories.
By default, Database files (data and lock files) are stored
under ``root/.spack-db``, which is created if it does not
exist. This is the ``db_dir``.
The Database will attempt to read an ``index.json`` file in
``db_dir``. If it does not find one, it will fall back to read
an ``index.yaml`` if one is present. If that does not exist, it
will create a database when needed by scanning the entire
Database root for ``spec.yaml`` files according to Spack's
``DirectoryLayout``.
Caller may optionally provide a custom ``db_dir`` parameter
where data will be stored. This is intended to be used for
testing the Database class.
"""
self.root = root
if db_dir is None:
# If the db_dir is not provided, default to within the db root.
self._db_dir = join_path(self.root, _db_dirname)
else:
# Allow customizing the database directory location for testing.
self._db_dir = db_dir
# Set up layout of database files within the db dir
self._old_yaml_index_path = join_path(self._db_dir, 'index.yaml')
self._index_path = join_path(self._db_dir, 'index.json')
self._lock_path = join_path(self._db_dir, 'lock')
# This is for other classes to use to lock prefix directories.
self.prefix_lock_path = join_path(self._db_dir, 'prefix_lock')
# Create needed directories and files
if not os.path.exists(self._db_dir):
mkdirp(self._db_dir)
# initialize rest of state.
self.lock = Lock(self._lock_path)
self._data = {}
# whether there was an error at the start of a read transaction
self._error = None
def prefix_lock(self, spec):
"""Get a lock on a particular spec's installation directory.
NOTE: The installation directory **does not** need to exist.
Prefix lock is a byte range lock on the nth byte of a file.
The lock file is ``spack.store.db.prefix_lock`` -- the DB
tells us what to call it and it lives alongside the install DB.
n is the sys.maxsize-bit prefix of the DAG hash. This makes
likelihood of collision is very low AND it gives us
readers-writer lock semantics with just a single lockfile, so no
cleanup required.
"""
prefix = spec.prefix
if prefix not in self._prefix_locks:
self._prefix_locks[prefix] = Lock(
self.prefix_lock_path,
spec.dag_hash_bit_prefix(bit_length(sys.maxsize)), 1)
return self._prefix_locks[prefix]
def test_upgrade_read_to_write_fails_with_readonly_file(private_lock_path):
# ensure lock file exists the first time, so we open it read-only
# to begin wtih.
touch(private_lock_path)
with read_only(private_lock_path):
lock = Lock(private_lock_path)
assert lock._reads == 0
assert lock._writes == 0
lock.acquire_read()
assert lock._reads == 1
assert lock._writes == 0
assert lock._file.mode == 'r'
with pytest.raises(LockError):
lock.acquire_write()
def test_transaction(lock_path):
def enter_fn():
vals['entered'] = True
def exit_fn(t, v, tb):
vals['exited'] = True
vals['exception'] = (t or v or tb)
lock = Lock(lock_path)
vals = {'entered': False, 'exited': False, 'exception': False}
with ReadTransaction(lock, enter_fn, exit_fn):
pass
assert vals['entered']
assert vals['exited']
assert not vals['exception']
vals = {'entered': False, 'exited': False, 'exception': False}
with WriteTransaction(lock, enter_fn, exit_fn):
pass
assert vals['entered']
assert vals['exited']
assert not vals['exception']
def test_transaction_with_context_manager_and_exception(lock_path):
class TestContextManager(object):
def __enter__(self):
vals['entered'] = True
def __exit__(self, t, v, tb):
vals['exited'] = True
vals['exception'] = (t or v or tb)
def exit_fn(t, v, tb):
vals['exited_fn'] = True
vals['exception_fn'] = (t or v or tb)
lock = Lock(lock_path)
def do_read_with_exception(exit_fn):
with ReadTransaction(lock, TestContextManager, exit_fn):
raise Exception()
def do_write_with_exception(exit_fn):
with WriteTransaction(lock, TestContextManager, exit_fn):
raise Exception()
vals = {
'entered': False,
'exited': False,
'exited_fn': False,
'exception': False,
'exception_fn': False
}
with pytest.raises(Exception):
do_read_with_exception(exit_fn)
assert vals['entered']
assert vals['exited']
assert vals['exception']
assert vals['exited_fn']
assert vals['exception_fn']
vals = {
'entered': False,
'exited': False,
'exited_fn': False,
'exception': False,
'exception_fn': False
}
with pytest.raises(Exception):
do_read_with_exception(None)
assert vals['entered']
assert vals['exited']
assert vals['exception']
assert not vals['exited_fn']
assert not vals['exception_fn']
vals = {
'entered': False,
'exited': False,
'exited_fn': False,
'exception': False,
'exception_fn': False
}
with pytest.raises(Exception):
do_write_with_exception(exit_fn)
assert vals['entered']
assert vals['exited']
assert vals['exception']
assert vals['exited_fn']
assert vals['exception_fn']
vals = {
'entered': False,
'exited': False,
'exited_fn': False,
'exception': False,
'exception_fn': False
}
with pytest.raises(Exception):
do_write_with_exception(None)
assert vals['entered']
assert vals['exited']
assert vals['exception']
assert not vals['exited_fn']
assert not vals['exception_fn']
def test_transaction_with_context_manager(lock_path):
class TestContextManager(object):
def __enter__(self):
vals['entered'] = True
def __exit__(self, t, v, tb):
vals['exited'] = True
vals['exception'] = (t or v or tb)
def exit_fn(t, v, tb):
vals['exited_fn'] = True
vals['exception_fn'] = (t or v or tb)
lock = Lock(lock_path)
vals = {
'entered': False,
'exited': False,
'exited_fn': False,
'exception': False,
'exception_fn': False
}
with ReadTransaction(lock, TestContextManager, exit_fn):
pass
assert vals['entered']
assert vals['exited']
assert not vals['exception']
assert vals['exited_fn']
assert not vals['exception_fn']
vals = {
'entered': False,
'exited': False,
'exited_fn': False,
'exception': False,
'exception_fn': False
}
with ReadTransaction(lock, TestContextManager):
pass
assert vals['entered']
assert vals['exited']
assert not vals['exception']
assert not vals['exited_fn']
assert not vals['exception_fn']
vals = {
'entered': False,
'exited': False,
'exited_fn': False,
'exception': False,
'exception_fn': False
}
with WriteTransaction(lock, TestContextManager, exit_fn):
pass
assert vals['entered']
assert vals['exited']
assert not vals['exception']
assert vals['exited_fn']
assert not vals['exception_fn']
vals = {
'entered': False,
'exited': False,
'exited_fn': False,
'exception': False,
'exception_fn': False
}
with WriteTransaction(lock, TestContextManager):
pass
assert vals['entered']
assert vals['exited']
assert not vals['exception']
assert not vals['exited_fn']
assert not vals['exception_fn']
def p3(barrier):
lock = Lock(lock_path)
# p1 acquires write
barrier.wait() # ---------------------------------------- 1
with pytest.raises(LockError):
lock.acquire_write(lock_fail_timeout)
with pytest.raises(LockError):
lock.acquire_read(lock_fail_timeout)
barrier.wait() # ---------------------------------------- 2
lock.acquire_read()
barrier.wait() # ---------------------------------------- 3
# p1 tests shared read
barrier.wait() # ---------------------------------------- 4
lock.release_read()
barrier.wait() # ---------------------------------------- 5
# p2 upgrades read to write
barrier.wait() # ---------------------------------------- 6
with pytest.raises(LockError):
lock.acquire_write(lock_fail_timeout)
with pytest.raises(LockError):
lock.acquire_read(lock_fail_timeout)
barrier.wait() # ---------------------------------------- 7
# p2 releases write & read
barrier.wait() # ---------------------------------------- 8
lock.acquire_read()
barrier.wait() # ---------------------------------------- 9
lock.acquire_write()
barrier.wait() # ---------------------------------------- 10
# others test timeout
barrier.wait() # ---------------------------------------- 11
lock.release_read() # release read AND write in opposite
lock.release_write() # order from before on p2
barrier.wait() # ---------------------------------------- 12
lock.acquire_read()
barrier.wait() # ---------------------------------------- 13
lock.release_read()
def p2(barrier):
lock = Lock(lock_path)
# p1 acquires write
barrier.wait() # ---------------------------------------- 1
with pytest.raises(LockError):
lock.acquire_write(lock_fail_timeout)
with pytest.raises(LockError):
lock.acquire_read(lock_fail_timeout)
barrier.wait() # ---------------------------------------- 2
lock.acquire_read()
barrier.wait() # ---------------------------------------- 3
# p1 tests shared read
barrier.wait() # ---------------------------------------- 4
# others release reads
barrier.wait() # ---------------------------------------- 5
lock.acquire_write() # upgrade read to write
barrier.wait() # ---------------------------------------- 6
# others test timeout
barrier.wait() # ---------------------------------------- 7
lock.release_write() # release read AND write (need both)
lock.release_read()
barrier.wait() # ---------------------------------------- 8
# p3 acquires read
barrier.wait() # ---------------------------------------- 9
# p3 upgrades read to write
barrier.wait() # ---------------------------------------- 10
with pytest.raises(LockError):
lock.acquire_write(lock_fail_timeout)
with pytest.raises(LockError):
lock.acquire_read(lock_fail_timeout)
barrier.wait() # ---------------------------------------- 11
# p3 releases locks
barrier.wait() # ---------------------------------------- 12
lock.acquire_read()
barrier.wait() # ---------------------------------------- 13
lock.release_read()
def p1(barrier):
lock = Lock(lock_path)
lock.acquire_write()
barrier.wait() # ---------------------------------------- 1
# others test timeout
barrier.wait() # ---------------------------------------- 2
lock.release_write() # release and others acquire read
barrier.wait() # ---------------------------------------- 3
with pytest.raises(LockError):
lock.acquire_write(lock_fail_timeout)
lock.acquire_read()
barrier.wait() # ---------------------------------------- 4
lock.release_read()
barrier.wait() # ---------------------------------------- 5
# p2 upgrades read to write
barrier.wait() # ---------------------------------------- 6
with pytest.raises(LockError):
lock.acquire_write(lock_fail_timeout)
with pytest.raises(LockError):
lock.acquire_read(lock_fail_timeout)
barrier.wait() # ---------------------------------------- 7
# p2 releases write and read
barrier.wait() # ---------------------------------------- 8
# p3 acquires read
barrier.wait() # ---------------------------------------- 9
# p3 upgrades read to write
barrier.wait() # ---------------------------------------- 10
with pytest.raises(LockError):
lock.acquire_write(lock_fail_timeout)
with pytest.raises(LockError):
lock.acquire_read(lock_fail_timeout)
barrier.wait() # ---------------------------------------- 11
# p3 releases locks
barrier.wait() # ---------------------------------------- 12
lock.acquire_read()
barrier.wait() # ---------------------------------------- 13
lock.release_read()
def fn(barrier):
lock = Lock(lock_path, start, length)
barrier.wait() # wait for lock acquire in first process
with pytest.raises(LockError):
lock.acquire_read(lock_fail_timeout)
barrier.wait()
def fn(barrier):
lock = Lock(lock_path, start, length)
lock.acquire_read() # grab shared lock
barrier.wait()
barrier.wait() # hold the lock until timeout in other procs.
def fn(barrier):
lock = Lock(lock_path, start, length)
lock.acquire_write() # grab exclusive lock
barrier.wait()
barrier.wait() # hold the lock until timeout in other procs.
def _get_lock(self, key):
"""Create a lock for a key, if necessary, and return a lock object."""
if key not in self._locks:
self._locks[key] = Lock(self._lock_path(key))
return self._locks[key]