def test_no_workers_to_memory(c, s):
x = delayed(slowinc)(1, delay=0.4)
y = delayed(slowinc)(x, delay=0.4)
z = delayed(slowinc)(y, delay=0.4)
yy, zz = c.persist([y, z])
while not s.task_state:
yield gen.sleep(0.01)
w = Worker(s.ip, s.port, ncores=1)
w.put_key_in_memory(y.key, 3)
yield w._start()
start = time()
while not s.workers:
yield gen.sleep(0.01)
assert s.task_state[y.key] == 'memory'
assert s.task_state[x.key] == 'released'
assert s.task_state[z.key] == 'processing'
yield w._close()
def test_worker_arrives_with_processing_data(c, s, a, b):
x = delayed(slowinc)(1, delay=0.4)
y = delayed(slowinc)(x, delay=0.4)
z = delayed(slowinc)(y, delay=0.4)
yy, zz = c.persist([y, z])
while not any(w.processing for w in s.workers.values()):
yield gen.sleep(0.01)
w = Worker(s.ip, s.port, ncores=1)
w.put_key_in_memory(y.key, 3)
yield w._start()
start = time()
while len(s.workers) < 3:
yield gen.sleep(0.01)
assert s.get_task_status(keys={x.key, y.key, z.key}) == {
x.key: 'released',
y.key: 'memory',
z.key: 'processing',
}
yield w._close()
def test_no_workers_to_memory(c, s):
x = delayed(slowinc)(1, delay=0.4)
y = delayed(slowinc)(x, delay=0.4)
z = delayed(slowinc)(y, delay=0.4)
yy, zz = c.persist([y, z])
while not s.tasks:
yield gen.sleep(0.01)
w = Worker(s.ip, s.port, ncores=1)
w.put_key_in_memory(y.key, 3)
yield w._start()
start = time()
while not s.workers:
yield gen.sleep(0.01)
assert s.get_task_status(keys={x.key, y.key, z.key}) == {
x.key: 'released',
y.key: 'memory',
z.key: 'processing',
}
yield w._close()
def test_no_worker_to_memory_restrictions(c, s, a, b):
x = delayed(slowinc)(1, delay=0.4)
y = delayed(slowinc)(x, delay=0.4)
z = delayed(slowinc)(y, delay=0.4)
yy, zz = c.persist([y, z], workers={(x, y, z): 'alice'})
while not s.tasks:
yield gen.sleep(0.01)
w = Worker(s.ip, s.port, ncores=1, name='alice')
w.put_key_in_memory(y.key, 3)
yield w._start()
while len(s.workers) < 3:
yield gen.sleep(0.01)
yield gen.sleep(0.3)
assert s.get_task_status(keys={x.key, y.key, z.key}) == {
x.key: 'released',
y.key: 'memory',
z.key: 'processing',
}
yield w._close()
def test_worker_arrives_with_processing_data(c, s, a, b):
x = delayed(slowinc)(1, delay=0.4)
y = delayed(slowinc)(x, delay=0.4)
z = delayed(slowinc)(y, delay=0.4)
yy, zz = c.persist([y, z])
while not any(w.processing for w in s.workers.values()):
yield gen.sleep(0.01)
w = Worker(s.address, nthreads=1)
w.put_key_in_memory(y.key, 3)
yield w
start = time()
while len(s.workers) < 3:
yield gen.sleep(0.01)
assert s.get_task_status(keys={x.key, y.key, z.key}) == {
x.key: "released",
y.key: "memory",
z.key: "processing",
}
yield w.close()
def test_no_workers_to_memory(c, s):
x = delayed(slowinc)(1, delay=0.4)
y = delayed(slowinc)(x, delay=0.4)
z = delayed(slowinc)(y, delay=0.4)
yy, zz = c.persist([y, z])
while not s.tasks:
yield gen.sleep(0.01)
w = Worker(s.address, nthreads=1)
w.put_key_in_memory(y.key, 3)
yield w
start = time()
while not s.workers:
yield gen.sleep(0.01)
assert s.get_task_status(keys={x.key, y.key, z.key}) == {
x.key: "released",
y.key: "memory",
z.key: "processing",
}
yield w.close()
def test_no_worker_to_memory_restrictions(c, s, a, b):
x = delayed(slowinc)(1, delay=0.4)
y = delayed(slowinc)(x, delay=0.4)
z = delayed(slowinc)(y, delay=0.4)
yy, zz = c.persist([y, z], workers={(x, y, z): "alice"})
while not s.tasks:
yield gen.sleep(0.01)
w = Worker(s.address, nthreads=1, name="alice")
w.put_key_in_memory(y.key, 3)
yield w
while len(s.workers) < 3:
yield gen.sleep(0.01)
yield gen.sleep(0.3)
assert s.get_task_status(keys={x.key, y.key, z.key}) == {
x.key: "released",
y.key: "memory",
z.key: "processing",
}
yield w.close()
def test_no_worker_to_memory_restrictions(c, s, a, b):
x = delayed(slowinc)(1, delay=0.4)
y = delayed(slowinc)(x, delay=0.4)
z = delayed(slowinc)(y, delay=0.4)
yy, zz = c.persist([y, z], workers={(x, y, z): 'alice'})
while not s.tasks:
yield gen.sleep(0.01)
w = Worker(s.ip, s.port, ncores=1, name='alice')
w.put_key_in_memory(y.key, 3)
yield w._start()
while len(s.workers) < 3:
yield gen.sleep(0.01)
yield gen.sleep(0.3)
assert s.get_task_status(keys={x.key, y.key, z.key}) == {
x.key: 'released',
y.key: 'memory',
z.key: 'processing',
}
yield w._close()
def test_no_workers_to_memory(c, s):
x = delayed(slowinc)(1, delay=0.4)
y = delayed(slowinc)(x, delay=0.4)
z = delayed(slowinc)(y, delay=0.4)
yy, zz = c.persist([y, z])
while not s.tasks:
yield gen.sleep(0.01)
w = Worker(s.ip, s.port, ncores=1)
w.put_key_in_memory(y.key, 3)
yield w._start()
start = time()
while not s.workers:
yield gen.sleep(0.01)
assert s.get_task_status(keys={x.key, y.key, z.key}) == {
x.key: 'released',
y.key: 'memory',
z.key: 'processing',
}
yield w._close()
def test_worker_arrives_with_processing_data(c, s, a, b):
x = delayed(slowinc)(1, delay=0.4)
y = delayed(slowinc)(x, delay=0.4)
z = delayed(slowinc)(y, delay=0.4)
yy, zz = c.persist([y, z])
while not any(w.processing for w in s.workers.values()):
yield gen.sleep(0.01)
w = Worker(s.ip, s.port, ncores=1)
w.put_key_in_memory(y.key, 3)
yield w._start()
start = time()
while len(s.workers) < 3:
yield gen.sleep(0.01)
assert s.get_task_status(keys={x.key, y.key, z.key}) == {
x.key: 'released',
y.key: 'memory',
z.key: 'processing',
}
yield w._close()
def test_worker_arrives_with_processing_data(c, s, a, b):
x = delayed(slowinc)(1, delay=0.4)
y = delayed(slowinc)(x, delay=0.4)
z = delayed(slowinc)(y, delay=0.4)
yy, zz = c.persist([y, z])
while not s.processing:
yield gen.sleep(0.01)
w = Worker(s.ip, s.port, ncores=1, ip='127.0.0.1')
w.put_key_in_memory(y.key, 3)
yield w._start()
start = time()
while len(s.workers) < 3:
yield gen.sleep(0.01)
assert s.task_state[y.key] == 'memory'
assert s.task_state[x.key] == 'released'
assert s.task_state[z.key] == 'processing'
yield w._close()
