async def f(): async with Client(processes=False, asynchronous=True) as c: assert c.asynchronous == True async with Lock('x'): lock2 = Lock('x') with pytest.raises(gen.TimeoutError): await lock2.acquire(timeout=0.1)
async def test_locks(): async with Client(processes=False, asynchronous=True) as c: assert c.asynchronous async with Lock("x"): lock2 = Lock("x") result = await lock2.acquire(timeout=0.1) assert result is False
async def f(): async with Client(processes=False, asynchronous=True) as c: assert c.asynchronous == True async with Lock('x'): lock2 = Lock('x') result = await lock2.acquire(timeout=0.1) assert result is False
def test_lock_types(c, s, a, b): for name in [1, ('a', 1), ['a', 1], b'123', '123']: lock = Lock(name) assert lock.name == name yield lock.acquire() yield lock.release() assert not s.extensions['locks'].events
def test_lock_types(c, s, a, b): for name in [1, ("a", 1), ["a", 1], b"123", "123"]: lock = Lock(name) assert lock.name == name yield lock.acquire() yield lock.release() assert not s.extensions["locks"].events
async def test_acquires_with_zero_timeout(c, s, a, b): lock = Lock("x") await lock.acquire(timeout=0) assert lock.locked() await lock.release() await lock.acquire(timeout="1s") await lock.release() await lock.acquire(timeout=timedelta(seconds=1)) await lock.release()
def test_acquires_with_zero_timeout(c, s, a, b): lock = Lock('x') yield lock.acquire(timeout=0) assert lock.locked() yield lock.release() yield lock.acquire(timeout=1) yield lock.release() yield lock.acquire(timeout=1) yield lock.release()
async def test_steal_stimulus_id_unique(c, s, *workers): steal = s.extensions["stealing"] num_futs = 1_000 async with Lock() as lock: def blocked(x, lock): lock.acquire() # Setup all tasks on worker 0 such that victim/thief relation is the # same for all tasks. futures = c.map(blocked, range(num_futs), lock=lock, workers=[workers[0].address]) # Ensure all tasks are assigned to the worker since otherwise the # move_task_request fails. while len(workers[0].tasks) != num_futs: await asyncio.sleep(0.1) tasks = [s.tasks[f.key] for f in futures] w0 = s.workers[workers[0].address] w1 = s.workers[workers[1].address] # Generating the move task requests as fast as possible increases the # chance of duplicates if the uniqueness is not guaranteed. for ts in tasks: steal.move_task_request(ts, w0, w1) # Values stored in in_flight are used for response verification. # Therefore all stimulus IDs are stored here and must be unique stimulus_ids = {dct["stimulus_id"] for dct in steal.in_flight.values()} assert len(stimulus_ids) == num_futs await c.cancel(futures)
def test_acquires_blocking(c, s, a, b): lock = Lock("x") yield lock.acquire(blocking=False) assert lock.locked() yield lock.release() assert not lock.locked() with pytest.raises(ValueError): lock.acquire(blocking=False, timeout=1)
def f(x): with Lock('x') as lock: client = get_client() assert client.get_metadata('locked') == False client.set_metadata('locked', True) sleep(0.05) assert client.get_metadata('locked') == True client.set_metadata('locked', False)
def f(x): with Lock("x") as lock: client = get_client() assert client.get_metadata("locked") is False client.set_metadata("locked", True) sleep(0.05) assert client.get_metadata("locked") is True client.set_metadata("locked", False)
async def test_timeout(c, s, a, b): locks = s.extensions["locks"] lock = Lock("x") result = await lock.acquire() assert result is True assert locks.ids["x"] == lock.id lock2 = Lock("x") assert lock.id != lock2.id start = time() result = await lock2.acquire(timeout=0.1) stop = time() assert stop - start < 0.3 assert result is False assert locks.ids["x"] == lock.id assert not locks.events["x"] await lock.release()
def test_timeout(c, s, a, b): locks = s.extensions['locks'] lock = Lock('x') result = yield lock.acquire() assert result is True assert locks.ids['x'] == lock.id lock2 = Lock('x') assert lock.id != lock2.id start = time() result = yield lock2.acquire(timeout=0.1) stop = time() assert stop - start < 0.3 assert result is False assert locks.ids['x'] == lock.id assert not locks.events['x'] yield lock.release()
def test_acquires_blocking(c, s, a, b): lock = Lock('x') yield lock.acquire(blocking=False) assert lock.locked() yield lock.release() assert not lock.locked() with pytest.raises(ValueError): lock.acquire(blocking=False, timeout=1)
def __init__(self, array: zarr.core.Array, client: Client): self._locks = get_chunklock(array, client) # from the perspective of a zarr array, metadata has this key regardless of the # location on storage. unfortunately, the synchronizer does not get access to the # indirection provided by the the store class. array_attrs_key = f"{array.path}/.zarray" if is_n5(array): attrs_path = f"{array.path}/attributes.json" else: attrs_path = array_attrs_key self._locks[array_attrs_key] = Lock(attrs_path, client=client)
def test_serializable(c, s, a, b): def f(x, lock=None): with lock: assert lock.name == 'x' return x + 1 lock = Lock('x') futures = c.map(f, range(10), lock=lock) yield c.gather(futures) lock2 = pickle.loads(pickle.dumps(lock)) assert lock2.name == lock.name assert lock2.client is lock.client
def process(self, events): from distributed import worker_client, Variable, Lock assert isinstance(self.proc, BaseProcessor) assert not isinstance(self.proc, _Preheater) s = self.proc.get_dataset(events).data_source d = self.prefix + s with worker_client(separate_thread=False) as c: v = Variable(d, c) l = Lock(d, c) if l.acquire(blocking=False): self.proc.process(events) cols = set() for col in events.materialized: col = col.replace("_", ".", 1) try: attrgetter(col)(events) except AttributeError: pass else: cols.add(col) cols = sorted(cols) v.set(cols) return dict_accumulator({s: set_accumulator(cols)}) else: cols = v.get() for ag in map(attrgetter, cols): data = ag(events) data = getattr(data, "content", data) if callable(getattr(data, "materialize")): data.materialize() return dict_accumulator({})
def test_timeout(c, s, a, b): locks = s.extensions['locks'] lock = Lock('x') yield lock.acquire() lock2 = Lock('x') assert lock.id != lock2.id start = time() with pytest.raises(gen.TimeoutError): yield lock2.acquire(timeout=0.1) stop = time() assert stop - start < 0.3 yield lock.release()
def _mosaic(im, ch, conf, pos, abs_err, imgtype, out): assert imgtype in ["raw", "pos_err", "overlap"] assert out is not None y0, x0 = pos yslice = slice(y0, y0 + im.shape[0]) xslice = slice(x0, x0 + im.shape[1]) lock_name = f"{imgtype}-{ch}" with Lock(lock_name): if imgtype == "raw": out[yslice, xslice] = out[yslice, xslice] + im * conf elif imgtype == "overlap": out[yslice, xslice] = out[yslice, xslice] + conf elif imgtype == "pos_err": out[yslice, xslice] = out[yslice, xslice] + conf * np.sum(np.array(abs_err)**2)
def fit(self, X, y=None): w = get_worker() dsk_lock = Lock(self.lock_name, client=w.client) dsk_counter = Variable(self.counter_name, client=w.client) dsk_killed_workers = Variable(self.killed_workers_name, client=w.client) for e in list(w.executing): should_die = False t = literal_eval(e) with dsk_lock: c = dsk_counter.get() dsk_counter.set(c + 1) killed_workers = dsk_killed_workers.get() if c > self.min_complete and t not in killed_workers: killed_workers[t] = True should_die = True dsk_killed_workers.set(killed_workers) if should_die: os.kill(os.getpid(), 9) return self
async def test_locks(c, s): async with Lock("x") as l1: l2 = Lock("x") assert l1.client is c assert l2.client is c assert await l2.acquire(timeout=0.01) is False
def test_errors(c, s, a, b): lock = Lock('x') with pytest.raises(ValueError): yield lock.release()
def test_timeout_sync(loop): with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as c: with Lock('x') as lock: assert Lock('x').acquire(timeout=0.1) is False
def test_timeout_sync(client): with Lock('x') as lock: assert Lock('x').acquire(timeout=0.1) is False
def test_timeout_sync(loop): with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as c: with Lock('x') as lock: with pytest.raises(gen.TimeoutError): Lock('x').acquire(timeout=0.1)
def get_chunklock(array: zarr.core.Array, client: Client) -> Dict[str, Lock]: result = {key: Lock(key, client=client) for key in get_chunk_keys(array)} return result
async def test_errors(c, s, a, b): lock = Lock("x") with pytest.raises(ValueError): await lock.release()