class PostprocessingState(object):
"""
A job that is trying to postprocess
"""
def __init__(self, job):
self.job = job
def is_alive(self):
return False
def initiate_postprocessing(self, job):
raise RuntimeError, "initiate_postprocess called second time"
def perform_postprocessing(self, job):
try:
value = result.Success( value = job.unit.finalize( job = self.job ) )
except ProcessingException, e:
raise
except Exception, e:
value = result.Error( exception = e )
def initiate_postprocessing(self, job):
self.job.join()
exit_code = getattr( self.job, "exitcode", 0 ) # Thread has no "exitcode" attribute
if exit_code == 0:
job.status = PostprocessingState( job = self.job )
else:
err = getattr( self.job, "err", RuntimeError( "exit code = %s" % exit_code ) )
job.status = ValueState( value = result.Error( exception = err ) )
def serve(self):
while self.active:
command = pickle.load(self.instream)
try:
response = command(server=self)
except Exception, e:
pickle.dump(result.Error(exception=e), self.outstream, 0)
else:
pickle.dump(result.Success(value=response), self.outstream, 0)
self.outstream.flush()
def poll(self, block):
if self.exitcode is None:
from Queue import Empty
try:
self.result = self.worker.get( block = block )
self.exitcode = 0
except Empty:
return True
except Exception, e:
self.result = result.Error( exception = e )
self.exitcode = 1
self.err = e
def results(self, raw):
results = []
try:
res = raw()
except Exception, e:
results.extend(
[
Result(
identifier = i,
value = result.Error( exception = e ),
)
for i in self.identifiers
]
)
def poll(self):
self.manage_workers()
if self.identifier_for:
if self.job_count == 0:
for identifier in self.identifier_for.keys():
iden = self.identifier_for[ identifier ]
res = Result(
identifier = self.identifier_for[ identifier ],
value = result.Error( exception = TerminatedException( "Worker died" ) ),
)
self.completed_results.append( res )
del self.identifier_for[ identifier ]
self.dead = 0
else:
self.process_next()
[
Result(
identifier = i,
value = result.Error( exception = e ),
)
for i in self.identifiers
]
)
else:
assert len( res ) == len( self.identifiers )
for ( identifier, ( failure, data ) ) in zip( self.identifiers, res ):
if failure:
results.append(
Result( identifier = identifier, value = result.Error( exception = data ) )
)
else:
results.append(
Result( identifier = identifier, value = result.Success( value = data ) )
)
return results
class Pooler(object):
"""
Pools up a number of jobs and runs this way
pool - number of jobs to pool