async def f():
async with Client(processes=False, asynchronous=True) as c:
assert c.asynchronous == True
async with Lock('x'):
lock2 = Lock('x')
with pytest.raises(gen.TimeoutError):
await lock2.acquire(timeout=0.1)
async def test_locks():
async with Client(processes=False, asynchronous=True) as c:
assert c.asynchronous
async with Lock("x"):
lock2 = Lock("x")
result = await lock2.acquire(timeout=0.1)
assert result is False
async def f():
async with Client(processes=False, asynchronous=True) as c:
assert c.asynchronous == True
async with Lock('x'):
lock2 = Lock('x')
result = await lock2.acquire(timeout=0.1)
assert result is False
def test_lock_types(c, s, a, b):
for name in [1, ('a', 1), ['a', 1], b'123', '123']:
lock = Lock(name)
assert lock.name == name
yield lock.acquire()
yield lock.release()
assert not s.extensions['locks'].events
def test_lock_types(c, s, a, b):
for name in [1, ('a', 1), ['a', 1], b'123', '123']:
lock = Lock(name)
assert lock.name == name
yield lock.acquire()
yield lock.release()
assert not s.extensions['locks'].events
def test_lock_types(c, s, a, b):
for name in [1, ("a", 1), ["a", 1], b"123", "123"]:
lock = Lock(name)
assert lock.name == name
yield lock.acquire()
yield lock.release()
assert not s.extensions["locks"].events
async def test_acquires_with_zero_timeout(c, s, a, b):
lock = Lock("x")
await lock.acquire(timeout=0)
assert lock.locked()
await lock.release()
await lock.acquire(timeout="1s")
await lock.release()
await lock.acquire(timeout=timedelta(seconds=1))
await lock.release()
def test_acquires_with_zero_timeout(c, s, a, b):
lock = Lock('x')
yield lock.acquire(timeout=0)
assert lock.locked()
yield lock.release()
yield lock.acquire(timeout=1)
yield lock.release()
yield lock.acquire(timeout=1)
yield lock.release()
async def test_steal_stimulus_id_unique(c, s, *workers):
steal = s.extensions["stealing"]
num_futs = 1_000
async with Lock() as lock:
def blocked(x, lock):
lock.acquire()
# Setup all tasks on worker 0 such that victim/thief relation is the
# same for all tasks.
futures = c.map(blocked,
range(num_futs),
lock=lock,
workers=[workers[0].address])
# Ensure all tasks are assigned to the worker since otherwise the
# move_task_request fails.
while len(workers[0].tasks) != num_futs:
await asyncio.sleep(0.1)
tasks = [s.tasks[f.key] for f in futures]
w0 = s.workers[workers[0].address]
w1 = s.workers[workers[1].address]
# Generating the move task requests as fast as possible increases the
# chance of duplicates if the uniqueness is not guaranteed.
for ts in tasks:
steal.move_task_request(ts, w0, w1)
# Values stored in in_flight are used for response verification.
# Therefore all stimulus IDs are stored here and must be unique
stimulus_ids = {dct["stimulus_id"] for dct in steal.in_flight.values()}
assert len(stimulus_ids) == num_futs
await c.cancel(futures)
def test_acquires_blocking(c, s, a, b):
lock = Lock("x")
yield lock.acquire(blocking=False)
assert lock.locked()
yield lock.release()
assert not lock.locked()
with pytest.raises(ValueError):
lock.acquire(blocking=False, timeout=1)
def f(x):
with Lock('x') as lock:
client = get_client()
assert client.get_metadata('locked') == False
client.set_metadata('locked', True)
sleep(0.05)
assert client.get_metadata('locked') == True
client.set_metadata('locked', False)
def f(x):
with Lock("x") as lock:
client = get_client()
assert client.get_metadata("locked") is False
client.set_metadata("locked", True)
sleep(0.05)
assert client.get_metadata("locked") is True
client.set_metadata("locked", False)
def test_acquires_with_zero_timeout(c, s, a, b):
lock = Lock('x')
yield lock.acquire(timeout=0)
assert lock.locked()
yield lock.release()
yield lock.acquire(timeout=1)
yield lock.release()
yield lock.acquire(timeout=1)
yield lock.release()
async def test_timeout(c, s, a, b):
locks = s.extensions["locks"]
lock = Lock("x")
result = await lock.acquire()
assert result is True
assert locks.ids["x"] == lock.id
lock2 = Lock("x")
assert lock.id != lock2.id
start = time()
result = await lock2.acquire(timeout=0.1)
stop = time()
assert stop - start < 0.3
assert result is False
assert locks.ids["x"] == lock.id
assert not locks.events["x"]
await lock.release()
def test_timeout(c, s, a, b):
locks = s.extensions['locks']
lock = Lock('x')
result = yield lock.acquire()
assert result is True
assert locks.ids['x'] == lock.id
lock2 = Lock('x')
assert lock.id != lock2.id
start = time()
result = yield lock2.acquire(timeout=0.1)
stop = time()
assert stop - start < 0.3
assert result is False
assert locks.ids['x'] == lock.id
assert not locks.events['x']
yield lock.release()
def test_acquires_blocking(c, s, a, b):
lock = Lock('x')
yield lock.acquire(blocking=False)
assert lock.locked()
yield lock.release()
assert not lock.locked()
with pytest.raises(ValueError):
lock.acquire(blocking=False, timeout=1)
def __init__(self, array: zarr.core.Array, client: Client):
self._locks = get_chunklock(array, client)
# from the perspective of a zarr array, metadata has this key regardless of the
# location on storage. unfortunately, the synchronizer does not get access to the
# indirection provided by the the store class.
array_attrs_key = f"{array.path}/.zarray"
if is_n5(array):
attrs_path = f"{array.path}/attributes.json"
else:
attrs_path = array_attrs_key
self._locks[array_attrs_key] = Lock(attrs_path, client=client)
def test_serializable(c, s, a, b):
def f(x, lock=None):
with lock:
assert lock.name == 'x'
return x + 1
lock = Lock('x')
futures = c.map(f, range(10), lock=lock)
yield c.gather(futures)
lock2 = pickle.loads(pickle.dumps(lock))
assert lock2.name == lock.name
assert lock2.client is lock.client
def process(self, events):
from distributed import worker_client, Variable, Lock
assert isinstance(self.proc, BaseProcessor)
assert not isinstance(self.proc, _Preheater)
s = self.proc.get_dataset(events).data_source
d = self.prefix + s
with worker_client(separate_thread=False) as c:
v = Variable(d, c)
l = Lock(d, c)
if l.acquire(blocking=False):
self.proc.process(events)
cols = set()
for col in events.materialized:
col = col.replace("_", ".", 1)
try:
attrgetter(col)(events)
except AttributeError:
pass
else:
cols.add(col)
cols = sorted(cols)
v.set(cols)
return dict_accumulator({s: set_accumulator(cols)})
else:
cols = v.get()
for ag in map(attrgetter, cols):
data = ag(events)
data = getattr(data, "content", data)
if callable(getattr(data, "materialize")):
data.materialize()
return dict_accumulator({})
def test_timeout(c, s, a, b):
locks = s.extensions['locks']
lock = Lock('x')
yield lock.acquire()
lock2 = Lock('x')
assert lock.id != lock2.id
start = time()
with pytest.raises(gen.TimeoutError):
yield lock2.acquire(timeout=0.1)
stop = time()
assert stop - start < 0.3
yield lock.release()
def _mosaic(im, ch, conf, pos, abs_err, imgtype, out):
assert imgtype in ["raw", "pos_err", "overlap"]
assert out is not None
y0, x0 = pos
yslice = slice(y0, y0 + im.shape[0])
xslice = slice(x0, x0 + im.shape[1])
lock_name = f"{imgtype}-{ch}"
with Lock(lock_name):
if imgtype == "raw":
out[yslice, xslice] = out[yslice, xslice] + im * conf
elif imgtype == "overlap":
out[yslice, xslice] = out[yslice, xslice] + conf
elif imgtype == "pos_err":
out[yslice,
xslice] = out[yslice,
xslice] + conf * np.sum(np.array(abs_err)**2)
def fit(self, X, y=None):
w = get_worker()
dsk_lock = Lock(self.lock_name, client=w.client)
dsk_counter = Variable(self.counter_name, client=w.client)
dsk_killed_workers = Variable(self.killed_workers_name, client=w.client)
for e in list(w.executing):
should_die = False
t = literal_eval(e)
with dsk_lock:
c = dsk_counter.get()
dsk_counter.set(c + 1)
killed_workers = dsk_killed_workers.get()
if c > self.min_complete and t not in killed_workers:
killed_workers[t] = True
should_die = True
dsk_killed_workers.set(killed_workers)
if should_die:
os.kill(os.getpid(), 9)
return self
def test_timeout(c, s, a, b):
locks = s.extensions['locks']
lock = Lock('x')
result = yield lock.acquire()
assert result is True
assert locks.ids['x'] == lock.id
lock2 = Lock('x')
assert lock.id != lock2.id
start = time()
result = yield lock2.acquire(timeout=0.1)
stop = time()
assert stop - start < 0.3
assert result is False
assert locks.ids['x'] == lock.id
assert not locks.events['x']
yield lock.release()
async def test_locks(c, s):
async with Lock("x") as l1:
l2 = Lock("x")
assert l1.client is c
assert l2.client is c
assert await l2.acquire(timeout=0.01) is False
def test_errors(c, s, a, b):
lock = Lock('x')
with pytest.raises(ValueError):
yield lock.release()
def test_timeout_sync(loop):
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
with Lock('x') as lock:
assert Lock('x').acquire(timeout=0.1) is False
def test_timeout_sync(client):
with Lock('x') as lock:
assert Lock('x').acquire(timeout=0.1) is False
def test_timeout_sync(loop):
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
with Lock('x') as lock:
with pytest.raises(gen.TimeoutError):
Lock('x').acquire(timeout=0.1)
def get_chunklock(array: zarr.core.Array, client: Client) -> Dict[str, Lock]:
result = {key: Lock(key, client=client) for key in get_chunk_keys(array)}
return result
def test_errors(c, s, a, b):
lock = Lock('x')
with pytest.raises(ValueError):
yield lock.release()
async def test_errors(c, s, a, b):
lock = Lock("x")
with pytest.raises(ValueError):
await lock.release()
