def testRecurse3(self):
"""
See if recursion fails for large number of processes.
Have this as last test as it ties up lots of threads
"""
res = rec_fib(15)
from PyDFlow.writeonce import WriteOnceVar
resslot = WriteOnceVar()
def waiter(resslot):
print "waiter running"
f = res.get()
print "waiter got %d" % f
resslot.set(f)
print "waiter done %s" % repr(resslot)
t = th.Thread(target=waiter, args=(resslot, ))
t.start()
# 10 seconds
print "waiting for fibonacci result"
for i in range(10):
#while True:
print resslot
if resslot.isSet():
self.assertEquals(resslot.get(), 610)
return
print ".",
time.sleep(1)
print(resslot.get())
self.fail("Ran out of time waiting for recursive fibonacci calc")
def testRecurse3(self):
"""
See if recursion fails for large number of processes.
Have this as last test as it ties up lots of threads
"""
res = rec_fib(15)
from PyDFlow.writeonce import WriteOnceVar
resslot = WriteOnceVar()
def waiter(resslot):
print "waiter running"
f = res.get()
print "waiter got %d" % f
resslot.set(f)
print "waiter done %s" % repr(resslot)
t = th.Thread(target=waiter, args=(resslot,))
t.start()
# 10 seconds
print "waiting for fibonacci result"
for i in range(10):
#while True:
print resslot
if resslot.isSet():
self.assertEquals(resslot.get(), 610)
return
print ".",
time.sleep(1)
print(resslot.get())
self.fail("Ran out of time waiting for recursive fibonacci calc")
def testSetGet(self):
x = WriteOnceVar()
setTh = th.Thread(target=(lambda: x.set(32)))
setTh.run()
self.assertEquals(x.get(), 32)
self.assertEquals(x.get(), 32)
self.assertEquals(x.get(), 32)
self.assertEquals(x.get(), 32)
def testSetGet(self):
x = WriteOnceVar()
setTh = th.Thread(target=(lambda: x.set(32)))
setTh.run()
self.assertEquals(x.get(), 32)
self.assertEquals(x.get(), 32)
self.assertEquals(x.get(), 32)
self.assertEquals(x.get(), 32)
def __init__(self, *args, **kwargs):
super(AtomicIvar, self).__init__(*args, **kwargs)
# Create the future
self._future = WriteOnceVar()
# __future stores a handle to the underlying data
# future will be set exactly when the underlying data is
# ready for reading: this way the get() function can block
# on the future
# Whether the backing storage is reliable
self._reliable = True
def testReply(self):
"""
Send reply ivar down request ivar
"""
requestCh = WriteOnceVar()
def t():
reply = requestCh.get()
reply.set("hello!!")
th.Thread(target=t).start()
replyCh = WriteOnceVar()
requestCh.set(replyCh)
self.assertEquals(replyCh.get(), "hello!!")
def testSetGet2(self):
"""
Swap data between three threads
"""
r1 = WriteOnceVar()
r2 = WriteOnceVar()
x = WriteOnceVar()
y = WriteOnceVar()
def t1():
x.set("one")
r1.set(y.get())
def t2():
y.set("two")
r2.set(x.get())
th1 = th.Thread(target=t1)
th1.start()
self.assertEquals(x.get(), "one")
self.assertTrue(x.isSet())
th2 = th.Thread(target=t2)
th2.start()
th1.join()
th2.join()
self.assertEquals(r1.get(), "two")
self.assertEquals(r2.get(), "one")
def __init__(self, *args, **kwargs):
super(AtomicIvar, self).__init__(*args, **kwargs)
# Create the future
self._future = WriteOnceVar()
# __future stores a handle to the underlying data
# future will be set exactly when the underlying data is
# ready for reading: this way the get() function can block
# on the future
# Whether the backing storage is reliable
self._reliable = True
def testReply(self):
'''
Send reply ivar down request ivar
'''
requestCh = WriteOnceVar()
def t():
reply = requestCh.get()
reply.set("hello!!")
th.Thread(target=t).start()
replyCh = WriteOnceVar()
requestCh.set(replyCh)
self.assertEquals(replyCh.get(), "hello!!")
class IvarPlaceholder(Placeholder, Ivar):
"""
This should have a composite task as input.
When this is forced, the composite task will construct a new task graph, with the corresponding
output of te task graph to be grafted into the task graph at the same place as this placeholder.
Assuming that the function does not cause an exception, the steps are:
1. Identify the ivar placeholders corresponding to the outputs of the composite task
2. Run the composite task to generate a new task graph, which has the same (or a subset) of the inputs
to the composite task, and a set of new output ivars corresponding to the placeholders.
3. Replace the placeholders with the composite tasks in the task graph.
The placeholders are updated to point to the original output ivars
A Composite
TODO: think about what happens if expanding function fails. This ivar should probably
go into a IVAR_ERROR state.
"""
def __init__(self, expected_class):
Placeholder.__init__(self, expected_class)
Ivar.__init__(self) # Inputs and outputs will be managed
self._proxy_for = WriteOnceVar()
def _check_real_ivar(self):
"""
a) check if the real ivar exists. return true if found
b) compress chain of pointers if there are multiple proxies
"""
ivar = self
next = self._proxy_for.get()
while isinstance(next, IvarPlaceholder):
ivar = next
if next._proxy_for.isSet():
next = next._proxy_for.get()
else:
self._proxy_for = next._proxy_for
raise Exception("no real ivar found")
self._proxy_for = ivar._proxy_for
def _replacewith(self, other):
"""
"""
assert(id(self) != id(other))
logging.debug("_replacewith %s %s" % (repr(self), repr(other)))
if not self._proxy_for.isSet():
self._proxy_for.set(other)
#TODO: does thsi correctly update inputs, outputs?
#for t in self._in_tasks:
# t._output_replace(self, other)
for out in self._out_tasks:
out._input_replace(self, other)
other._out_tasks = self._out_tasks
else:
raise Exception("should not be here yet")
#self._check_real_ivar()
#self._proxy_for.get()._replacewith(other)
def get(self):
#TODO: check?
with graph_mutex:
self._spark()
self._check_real_ivar()
next = self._proxy_for.get()
#TODO: this is a bit hacky...
return next.get()
def _expand(self, rec=True):
"""
If rec is true, expand until we hit a real ivar
Otherwise just do it once
"""
if not self._proxy_for.isSet():
assert len(self._in_tasks) == 1
in_task = self._in_tasks[0]
#own_ix = in_task._outputs.index(self)
in_task._state = T_QUEUED
graph_mutex.release()
try:
in_task._exec(None, None)
finally:
graph_mutex.acquire()
assert(self._proxy_for.isSet())
if rec:
last = self
ch = self._proxy_for.get()
while isinstance(ch, IvarPlaceholder):
last = ch
ch = ch._expand(rec=False)
# shorten chain
self._proxy_for = last._proxy_for
logging.debug("expanded %s, now points to: %s" % (repr(self), repr(self._proxy_for.get())))
else:
logging.debug("Proxy for %s" % repr(self._proxy_for))
return self._proxy_for.get()
def _spark(self, done_callback=None):
# input task should be a compound task.
# make sure the compound task is expanded
# TODO: less dreadful implementation
self._expand()
next_ivar = self._proxy_for.get()
next_ivar._spark(done_callback)
def __repr__(self):
if not self._proxy_for.isSet():
return "<Placeholder for ivar of type %s>" % repr(self._expected_class)
else:
return repr(self._proxy_for.get())
def state(self):
raise UnimplementedException("state not implemented")
#TODO
#TODO: do I need to imp
def _try_readable(self):
if self._proxy_for.isSet():
raise Exception("should not call _try_readable??")
else:
return False
def testSetGet2(self):
"""
Swap data between three threads
"""
r1 = WriteOnceVar()
r2 = WriteOnceVar()
x = WriteOnceVar()
y = WriteOnceVar()
def t1():
x.set("one")
r1.set(y.get())
def t2():
y.set("two")
r2.set(x.get())
th1 = th.Thread(target=t1)
th1.start()
self.assertEquals(x.get(), "one")
self.assertTrue(x.isSet())
th2 = th.Thread(target=t2)
th2.start()
th1.join()
th2.join()
self.assertEquals(r1.get(), "two")
self.assertEquals(r2.get(), "one")
def testIsSet(self):
x = WriteOnceVar()
self.assertFalse(x.isSet())
x.set("hello")
self.assertTrue(x.isSet())
def testGet(self):
x = WriteOnceVar()
x.set(32)
self.assertEquals(x.get(), 32)
def init():
global in_queue, resume_queue, workers, work_deques
logging.debug("Initialising thread pool")
in_queue = Queue.Queue()
resume_queue = Queue.Queue()
#TODO: this may not be a good memory layout, likely
# to cause cache conflict
work_deques = [deque() for i in range(NUM_THREADS)]
for i in range(NUM_THREADS):
t = WorkerThread(in_queue, resume_queue, i, work_deques[i])
workers.append(t)
t.start()
# Use future to ensure init() run exactly once
initFuture = WriteOnceVar(function=init)
def exec_async(ivar):
"""
Takes a ivar and fills it at some point in the future
"""
# Ensure workers are initialized
logging.debug("Entered exec_async")
initFuture.get()
logging.debug("Added ivar to work queue")
with idle_worker_cvar:
in_queue.put(ivar)
idle_worker_cvar.notifyAll()
logging.debug("Exiting exec_async")
class AtomicIvar(Ivar):
def __init__(self, *args, **kwargs):
super(AtomicIvar, self).__init__(*args, **kwargs)
# Create the future
self._future = WriteOnceVar()
# __future stores a handle to the underlying data
# future will be set exactly when the underlying data is
# ready for reading: this way the get() function can block
# on the future
# Whether the backing storage is reliable
self._reliable = True
def _register_input(self, input_task):
"""
Ivar can only be written to once, by a single writer.
"""
#TODO: proper exception types
if len(self._in_tasks) > 0:
raise Exception("Multiple tasks writing to an AtomicIvar")
elif self._state != IVAR_CLOSED:
raise Exception("Adding an input to an open AtomicIvar")
else:
self._in_tasks.append(input_task)
def _register_output(self, output_task):
"""
Don't need to track output task for notifications if ivar is reliable.
"""
done = self._state in [IVAR_OPEN_RW, IVAR_OPEN_R, IVAR_DONE_FILLED]
if done and self._reliable:
return True
else:
self._out_tasks.append(output_task)
#TODO: right?
return done
def _replacewith(self, other):
# Merge the futures to handle the case where a thread
# is block on the current thread's future
# other._future.merge_future(._future)
super(AtomicIvar, self)._replacewith(other)
def _prepare(self, mode):
"""
Set up the future variable to be written into.
"""
logging.debug("%s prepared" % (repr(self)))
if mode == M_READWRITE:
#TODO: exception type
raise Exception("M_READWRITE is not valid for atomic Ivars")
elif mode == M_WRITE:
#TODO: work out if it is bound to something, otherwise create a temp
# State will be open for writing until something
# is written into the file.
if self._state == IVAR_CLOSED or self._state == IVAR_CLOSED_WAITING:
self._open_write()
self._state = IVAR_OPEN_W
elif self._state == IVAR_OPEN_W or self._state == IVAR_OPEN_RW:
pass
else:
#TODO: type
raise Exception(
"Invalid state %d when trying to prepare for writing" %
self._state)
#TODO: what if the Ivar is destroyed?
elif mode == M_READ:
if self._state == IVAR_OPEN_R or self._state == IVAR_OPEN_RW:
pass
elif self._state == IVAR_DONE_FILLED:
self._open_read()
self._state = IVAR_OPEN_R
else:
#TODO: exception type
raise Exception(
"Read from Ivar which does not yet have data assoc")
else:
raise ValueError("Invalid mode to AtomicIvar._prepare %d" % mode)
def _open_write(self):
"""
Called when we want to prepare the Ivar for writing.
This does any required setup. Not responsible for
state-related logic, but is responsible for ensuring
that a write will proceed correctly.
Override to implement alternative logic.
"""
if self._future.isSet():
#TODO: exception type
raise Exception("Write to filled future Ivar")
def _open_read(self):
"""
Called when we want to prepare the Ivar for reading.
This does any required setup. Not responsible for
state-related logic, but is responsible for ensuring
that a read will proceed correctly.
Override to implement alternative logic.
"""
if not self._future.isSet():
#TODO: exception type
raise Exception("input Ivar has no data, cannot prepare")
def _set(self, val):
"""
Function to be called by input task when the data becomes available
"""
oldstate = self._state
if oldstate in (IVAR_OPEN_W, IVAR_OPEN_RW):
self._future.set(val)
self._state = IVAR_DONE_FILLED
logging.debug("%s set" % repr(self))
#update the state and notify output tasks
for t in self._out_tasks:
t._input_readable(self, oldstate, IVAR_DONE_FILLED)
if self._reliable:
self._in_tasks = None
self._out_tasks = None # Don't need to provide notification of any changes'
self._notify_done()
else:
#TODO: exception type
raise Exception("Invalid state when atomic_Ivar %s set" %
repr(self))
def get(self):
with graph_mutex:
res = self._get() # block on future
return res
def _get(self):
"""
For internal use only: get directly from local future, don't
bother forcing or locking or anything.
Should have graph lock first.
Only call when you are sure the Ivar has data ready for you.
"""
if LocalExecutor.isWorkerThread():
if not self._state in (IVAR_OPEN_R, IVAR_OPEN_RW,
IVAR_DONE_FILLED):
# This thread goes off and runs stuff recursively
# before blocking
LocalExecutor.spark_recursive(self)
else:
self._spark()
graph_mutex.release()
try:
res = self._future.get()
finally:
graph_mutex.acquire()
if res is ErrorVal and self._state == IVAR_ERROR:
raise self._exception
return res
def _error(self, *args, **kwargs):
self._future.set(None)
super(AtomicIvar, self)._error(*args, **kwargs)
def _has_data(self):
raise UnimplementedException("_has_data not overridden")
def _try_readable(self):
logging.debug("_try_readable on %s" % repr(self))
if self._state in (IVAR_DONE_FILLED, IVAR_OPEN_R, IVAR_OPEN_RW):
return True
elif self._state in (IVAR_CLOSED, IVAR_DONE_DESTROYED):
if len(self._in_tasks) == 0:
if self._bound is Unbound:
raise NoDataException(
"Unbound Ivar with no input tasks was forced.")
else:
if self._has_data():
# Data might be there, assume that binding was correct
self._prepare(M_WRITE)
self._set(self._bound)
return True
else:
raise NoDataException(
("Bound Ivar with no input tasks " +
"and no associated data was forced. " +
"Ivar was bound to" + repr(self._bound)))
elif self._state in (IVAR_ERROR, ):
return False
else:
return False
def _spark(self, done_callback=None):
"""
Should be called with lock held
Ensure that at some point in the future this Ivar will be filled
"""
logging.debug("Atomic Ivar sparked")
if done_callback is not None:
self._done_callbacks.append(done_callback)
if self._state in (IVAR_CLOSED, IVAR_DONE_DESTROYED):
if self._bound is not Unbound and len(self._in_tasks) == 0:
try:
self._try_readable()
except NoDataException, ex:
# Don't need to propagate error: this method only run if this
# Ivar is forced manually
self._fail([ex])
self._notify_done()
elif len(self._in_tasks) > 0:
# Enable task to be run, but
# input tasks should be run first
self._state = IVAR_CLOSED_WAITING
LocalExecutor.exec_async(self)
else:
# Nowhere for data to come from
#TODO: exception type
raise Exception(
"forcing Ivar which has no input tasks or bound data")
elif self._state in (IVAR_CLOSED_WAITING, IVAR_OPEN_W):
# Already sparked, just wait
pass
def testIsSet(self):
x = WriteOnceVar()
self.assertFalse(x.isSet())
x.set("hello")
self.assertTrue(x.isSet())
class AtomicIvar(Ivar):
def __init__(self, *args, **kwargs):
super(AtomicIvar, self).__init__(*args, **kwargs)
# Create the future
self._future = WriteOnceVar()
# __future stores a handle to the underlying data
# future will be set exactly when the underlying data is
# ready for reading: this way the get() function can block
# on the future
# Whether the backing storage is reliable
self._reliable = True
def _register_input(self, input_task):
"""
Ivar can only be written to once, by a single writer.
"""
#TODO: proper exception types
if len(self._in_tasks) > 0:
raise Exception("Multiple tasks writing to an AtomicIvar")
elif self._state != IVAR_CLOSED:
raise Exception("Adding an input to an open AtomicIvar")
else:
self._in_tasks.append(input_task)
def _register_output(self, output_task):
"""
Don't need to track output task for notifications if ivar is reliable.
"""
done = self._state in [IVAR_OPEN_RW, IVAR_OPEN_R, IVAR_DONE_FILLED]
if done and self._reliable:
return True
else:
self._out_tasks.append(output_task)
#TODO: right?
return done
def _replacewith(self, other):
# Merge the futures to handle the case where a thread
# is block on the current thread's future
# other._future.merge_future(._future)
super(AtomicIvar, self)._replacewith(other)
def _prepare(self, mode):
"""
Set up the future variable to be written into.
"""
logging.debug("%s prepared" % (repr(self)))
if mode == M_READWRITE:
#TODO: exception type
raise Exception("M_READWRITE is not valid for atomic Ivars")
elif mode == M_WRITE:
#TODO: work out if it is bound to something, otherwise create a temp
# State will be open for writing until something
# is written into the file.
if self._state == IVAR_CLOSED or self._state == IVAR_CLOSED_WAITING:
self._open_write()
self._state = IVAR_OPEN_W
elif self._state == IVAR_OPEN_W or self._state == IVAR_OPEN_RW:
pass
else:
#TODO: type
raise Exception("Invalid state %d when trying to prepare for writing" %
self._state)
#TODO: what if the Ivar is destroyed?
elif mode == M_READ:
if self._state == IVAR_OPEN_R or self._state == IVAR_OPEN_RW:
pass
elif self._state == IVAR_DONE_FILLED:
self._open_read()
self._state = IVAR_OPEN_R
else:
#TODO: exception type
raise Exception("Read from Ivar which does not yet have data assoc")
else:
raise ValueError("Invalid mode to AtomicIvar._prepare %d" % mode)
def _open_write(self):
"""
Called when we want to prepare the Ivar for writing.
This does any required setup. Not responsible for
state-related logic, but is responsible for ensuring
that a write will proceed correctly.
Override to implement alternative logic.
"""
if self._future.isSet():
#TODO: exception type
raise Exception("Write to filled future Ivar")
def _open_read(self):
"""
Called when we want to prepare the Ivar for reading.
This does any required setup. Not responsible for
state-related logic, but is responsible for ensuring
that a read will proceed correctly.
Override to implement alternative logic.
"""
if not self._future.isSet():
#TODO: exception type
raise Exception("input Ivar has no data, cannot prepare")
def _set(self, val):
"""
Function to be called by input task when the data becomes available
"""
oldstate = self._state
if oldstate in (IVAR_OPEN_W, IVAR_OPEN_RW):
self._future.set(val)
self._state = IVAR_DONE_FILLED
logging.debug("%s set" % repr(self))
#update the state and notify output tasks
for t in self._out_tasks:
t._input_readable(self, oldstate, IVAR_DONE_FILLED)
if self._reliable:
self._in_tasks = None
self._out_tasks = None # Don't need to provide notification of any changes'
self._notify_done()
else:
#TODO: exception type
raise Exception("Invalid state when atomic_Ivar %s set" % repr(self))
def get(self):
with graph_mutex:
res = self._get() # block on future
return res
def _get(self):
"""
For internal use only: get directly from local future, don't
bother forcing or locking or anything.
Should have graph lock first.
Only call when you are sure the Ivar has data ready for you.
"""
if LocalExecutor.isWorkerThread():
if not self._state in (IVAR_OPEN_R, IVAR_OPEN_RW, IVAR_DONE_FILLED):
# This thread goes off and runs stuff recursively
# before blocking
LocalExecutor.spark_recursive(self)
else:
self._spark()
graph_mutex.release()
try:
res = self._future.get()
finally:
graph_mutex.acquire()
if res is ErrorVal and self._state == IVAR_ERROR:
raise self._exception
return res
def _error(self, *args, **kwargs):
self._future.set(None)
super(AtomicIvar, self)._error(*args, **kwargs)
def _has_data(self):
raise UnimplementedException("_has_data not overridden")
def _try_readable(self):
logging.debug("_try_readable on %s" % repr(self))
if self._state in (IVAR_DONE_FILLED, IVAR_OPEN_R, IVAR_OPEN_RW):
return True
elif self._state in (IVAR_CLOSED, IVAR_DONE_DESTROYED):
if len(self._in_tasks) == 0:
if self._bound is Unbound:
raise NoDataException("Unbound Ivar with no input tasks was forced.")
else:
if self._has_data():
# Data might be there, assume that binding was correct
self._prepare(M_WRITE)
self._set(self._bound)
return True
else:
raise NoDataException(("Bound Ivar with no input tasks " +
"and no associated data was forced. " +
"Ivar was bound to" + repr(self._bound)))
elif self._state in (IVAR_ERROR,):
return False
else:
return False
def _spark(self, done_callback=None):
"""
Should be called with lock held
Ensure that at some point in the future this Ivar will be filled
"""
logging.debug("Atomic Ivar sparked")
if done_callback is not None:
self._done_callbacks.append(done_callback)
if self._state in (IVAR_CLOSED, IVAR_DONE_DESTROYED):
if self._bound is not Unbound and len(self._in_tasks) == 0:
try:
self._try_readable()
except NoDataException, ex:
# Don't need to propagate error: this method only run if this
# Ivar is forced manually
self._fail([ex])
self._notify_done()
elif len(self._in_tasks) > 0:
# Enable task to be run, but
# input tasks should be run first
self._state = IVAR_CLOSED_WAITING
LocalExecutor.exec_async(self)
else:
# Nowhere for data to come from
#TODO: exception type
raise Exception("forcing Ivar which has no input tasks or bound data")
elif self._state in (IVAR_CLOSED_WAITING, IVAR_OPEN_W):
# Already sparked, just wait
pass
def __init__(self, expected_class):
Placeholder.__init__(self, expected_class)
Ivar.__init__(self) # Inputs and outputs will be managed
self._proxy_for = WriteOnceVar()
def testGet(self):
x = WriteOnceVar()
x.set(32)
self.assertEquals(x.get(), 32)