def run(self):
"""Runs the Work, executing all decorated methods in the order they
were specified. A few kwargs passed to Work() will be forwarded to
job_stream.run().
"""
runKwargs = dict(self._runKwargs)
# Hack in a finish() that returns the results, if no finish or results
# is specified
result = [None]
if not self._hasFinish and self._resultHandler is None:
@self.finish
def returnResults(results):
return results
if self._hasFinish:
if self._resultHandler is not None:
raise ValueError("finish() and result()?")
def handleSingleResult(onlyResult):
if result[0] is not None:
raise ValueError("Got multiple results?")
result[0] = onlyResult
runKwargs['handleResult'] = handleSingleResult
else:
if self._resultHandler is None:
raise ValueError("No finish() nor result()?")
runKwargs['handleResult'] = self._resultHandler
# Run init functions, if it's the first execution of this stream
isFirst = True
# Is this a continuation?
if runKwargs.get('checkpointFile'):
if os.path.lexists(runKwargs['checkpointFile']):
isFirst = False
# Init?
if isFirst and job_stream.getRank() == 0:
# Call initial functions
def addToInitial(w):
self._initialWork.append(w)
for init in self._inits:
# Remember, anything returned from init() adds to our initial
# work.
self._listCall(addToInitial, init())
# Bury our initial work appropriately
for i in range(self._initialWorkDepth):
self._initialWork = [self._initialWork]
job_stream.work = self._initialWork
job_stream.run(self._config, **runKwargs)
return result[0]
def run(self):
"""Runs the Work, executing all decorated methods in the order they
were specified. A few kwargs passed to Work() will be forwarded to
job_stream.run().
"""
runKwargs = dict(self._runKwargs)
# Hack in a finish() that returns the results, if no finish or results
# is specified
result = [ None ]
if not self._hasFinish and self._resultHandler is None:
@self.finish
def returnResults(results):
return results
if self._hasFinish:
if self._resultHandler is not None:
raise ValueError("finish() and result()?")
def handleSingleResult(onlyResult):
if result[0] is not None:
raise ValueError("Got multiple results?")
result[0] = onlyResult
runKwargs['handleResult'] = handleSingleResult
else:
if self._resultHandler is None:
raise ValueError("No finish() nor result()?")
runKwargs['handleResult'] = self._resultHandler
# Run init functions, if it's the first execution of this stream
isFirst = True
# Is this a continuation?
if runKwargs.get('checkpointFile'):
if os.path.lexists(runKwargs['checkpointFile']):
isFirst = False
# Init?
if isFirst and job_stream.getRank() == 0:
# Call initial functions
def addToInitial(w):
self._initialWork.append(w)
for init in self._inits:
# Remember, anything returned from init() adds to our initial
# work.
self._listCall(addToInitial, init())
# Bury our initial work appropriately
for i in range(self._initialWorkDepth):
self._initialWork = [ self._initialWork ]
job_stream.work = self._initialWork
job_stream.run(self._config, **runKwargs)
return result[0]
import job_stream
import os
exDir = os.path.join(os.path.dirname(__file__), '../../../../example')
class addOne(job_stream.Job):
USE_MULTIPROCESSING = False
def postSetup(self):
print("addOne setup")
def handleWork(self, w):
self.emit(w + 1)
if __name__ == '__main__':
job_stream.work = [ 1, 2, 4, 8 ]
job_stream.run(os.path.join(exDir, "adder.yaml"))
def handleWork(self, w):
self.emit(w + 1)
class runExperiments(job_stream.Frame):
# Only call postSetup once
USE_MULTIPROCESSING = False
def postSetup(self):
print("runExperiments setup")
def handleFirst(self, store, work):
store.values = []
for i in work:
self.recur(ord(i))
def handleNext(self, store, work):
store.values.append(work)
def handleDone(self, store):
self.emit(sum(store.values) // len(store.values))
if __name__ == '__main__':
job_stream.work.append("aec")
job_stream.run(os.path.join(exDir, "example5.yaml"))
import job_stream
import sys
class addOneAndQuit(job_stream.Job):
def handleWork(self, w):
self._forceCheckpoint(True)
self.emit(w + 1)
if __name__ == '__main__':
job_stream.work = [ 1 ]
job_stream.run({
'jobs': [
{ 'type': addOneAndQuit },
{ 'type': addOneAndQuit },
{ 'type': addOneAndQuit },
]
}, checkpointFile = sys.argv[-1], checkpointSyncInterval = 0)
def handleDone(self, store):
self.emit(store.value)
class runExperiments(job_stream.Frame):
def handleFirst(self, store, work):
store.values = []
for i in work:
self.recur(ord(i))
def handleNext(self, store, work):
store.values.append(work)
def handleDone(self, store):
self.emit(_sum(store.values) / len(store.values))
if __name__ == '__main__':
# TODO - Transform config into python-readable equivalent. Is one
# directional ok? That is, from the C program's view, python changes would
# be read-only.
# TODO - Make programmatic config work:
config = {
'jobs': [
{ 'type': addOne },
{ 'type': addOne },
]
}
job_stream.work.append("abc")
job_stream.run(os.path.join(curDir, "example5.yaml"))
def handleWork(self, w):
self.emit(w + 1)
class sum(job_stream.Reducer):
USE_MULTIPROCESSING = False
def postSetup(self):
print("sum setup")
def handleInit(self, store):
store.value = 0
def handleAdd(self, store, work):
store.value += work
def handleJoin(self, store, other):
store.value += other.value
def handleDone(self, store):
self.emit(store.value)
if __name__ == '__main__':
job_stream.work = [ 3, 4 ]
job_stream.run(os.path.join(exDir, "example1.yaml"))
import job_stream
import sys
class addOneAndQuit(job_stream.Job):
def handleWork(self, w):
self._forceCheckpoint(True)
self.emit(w + 1)
if __name__ == '__main__':
job_stream.work = [1]
job_stream.run(
{
'jobs': [
{
'type': addOneAndQuit
},
{
'type': addOneAndQuit
},
{
'type': addOneAndQuit
},
]
},
checkpointFile=sys.argv[-1],
checkpointSyncInterval=0)
job_stream.work = [1, 2, 3, 4]
job_stream.run(
{
'jobs': [
{
'frame': getToFifty,
'jobs': [
{
'type': addOneAndQuit
},
]
},
{
'type': addOneAndQuit
},
{
'frame': getToHundred,
'jobs': [
{
'type': addOneAndQuit
},
]
},
{
'type': addOneAndQuit
},
]
},
checkpointFile=sys.argv[-1],
checkpointSyncInterval=0)
import job_stream
class addTwo(job_stream.Job):
def handleWork(self, w):
self.emit(w + 2)
class multiply(job_stream.Reducer):
def handleInit(self, stash):
stash.value = 1
def handleAdd(self, stash, work):
stash.value *= work
def handleJoin(self, stash, other):
stash.value *= other.value
def handleDone(self, stash):
self.emit(stash.value)
if __name__ == '__main__':
job_stream.work = [ 1, 5 ]
# Adds 4 and multiplies numbers; so 5 * 9 = 45
job_stream.run({
'reducer': multiply,
'jobs': [
{ 'type': addTwo }, { 'type': 'addTwo' }
] })
class addTwo(job_stream.Job):
def handleWork(self, w):
self.emit(w + 2)
class multiply(job_stream.Reducer):
def handleInit(self, stash):
stash.value = 1
def handleAdd(self, stash, work):
stash.value *= work
def handleJoin(self, stash, other):
stash.value *= other.value
def handleDone(self, stash):
self.emit(stash.value)
if __name__ == '__main__':
job_stream.work = [1, 5]
# Adds 4 and multiplies numbers; so 5 * 9 = 45
job_stream.run({
'reducer': multiply,
'jobs': [{
'type': addTwo
}, {
'type': 'addTwo'
}]
})
class getToHundred(getToFifty):
CAP = 100
if __name__ == '__main__':
# 1 + 1 = 2 + 2 = 4 + 3 = 7 + 4 = 11 + 6 = 17 + 9 = 26 + 14 = 40 + 21 = 61
# 61 + 1 = 62 + 32 = 94 + 48 = 142 + 1 = 143
# 5 is the only one that doesn't match this, so:
# 5 + 3 = 8 + 5 = 13 + 7 = 20 + 11 = 31 + 16 = 47 + 24 = 71
# 71 + 1 = 72 + 37 = 109 + 1 = 110
# So, 3 143s and 1 110 are expected.
job_stream.work = [ 1, 2, 3, 4 ]
job_stream.run({
'jobs': [
{
'frame': getToFifty,
'jobs': [
{ 'type': addOneAndQuit },
]
},
{ 'type': addOneAndQuit },
{
'frame': getToHundred,
'jobs': [
{ 'type': addOneAndQuit },
]
},
{ 'type': addOneAndQuit },
]
}, checkpointFile = sys.argv[-1], checkpointSyncInterval = 0)
import job_stream
import os
exDir = os.path.join(os.path.dirname(__file__), '../../../../example')
class addOne(job_stream.Job):
USE_MULTIPROCESSING = False
def postSetup(self):
print("addOne setup")
def handleWork(self, w):
self.emit(w + 1)
if __name__ == '__main__':
job_stream.work = [1, 2, 4, 8]
job_stream.run(os.path.join(exDir, "adder.yaml"))