def amap(self, f, *args, **kwds):
AbstractWorkerPool._AbstractWorkerPool__map(self, f, *args, **kwds)
def submit(*argz):
"""send a job to the server"""
#print "using", __STATE['server'].get_ncpus(), 'local workers'
return __STATE['server'].submit(f, argz, globals=globals())
override = True if kwds.has_key('size') else False
elem_size = kwds.pop('size', 2)
args = zip(*args)
# submit all jobs, to be collected later with 'get()'
tasks = [submit(*task) for task in args]
tasks = [ApplyResult(task) for task in tasks]
# build a correctly sized results object
length = len(args)
nodes = self.nodes
if self.nodes in ['*','autodetect',None]:
nodes = __STATE['server'].get_ncpus() #XXX: local workers only?
# try to quickly find a small chunksize that gives good results
maxsize = 2**62 #XXX: HOPEFULLY, this will never be reached...
chunksize = 1
while chunksize < maxsize:
chunksize, extra = divmod(length, nodes * elem_size)
if override: break # the user *wants* to override this loop
if extra >= length: break # we found something that 'works'
elem_size = elem_size * 2
if extra: chunksize += 1
m = MapResult((chunksize,length))
# queue the tasks
m.queue(*tasks)
return m
def amap(self, f, *args, **kwds):
AbstractWorkerPool._AbstractWorkerPool__map(self, f, *args, **kwds)
def submit(*argz):
"""send a job to the server"""
_pool = self._serve()
#print("using %s local workers" % _pool.get_ncpus())
try:
return _pool.submit(f, argz, globals=globals())
except pp.DestroyedServerError:
self._is_alive(None)
override = True if 'size' in kwds else False
elem_size = kwds.pop('size', 2)
length = min(len(task) for task in args)
args = zip(*args) #XXX: zip iterator ok? or should be list?
# submit all jobs, to be collected later with 'get()'
tasks = [submit(*task) for task in args]
tasks = [ApplyResult(task) for task in tasks]
# build a correctly sized results object
nodes = self.nodes
if self.nodes in ['*','autodetect',None]:
_pool = self._serve()
nodes = _pool.get_ncpus() #XXX: local workers only?
# try to quickly find a small chunksize that gives good results
maxsize = 2**62 #XXX: HOPEFULLY, this will never be reached...
chunksize = 1
while chunksize < maxsize:
chunksize, extra = divmod(length, nodes * elem_size)
if override: break # the user *wants* to override this loop
if extra >= length: break # we found something that 'works'
elem_size = elem_size * 2
if extra: chunksize += 1
m = MapResult((chunksize,length))
# queue the tasks
m.queue(*tasks)
return m
def amap(self, f, *args, **kwds):
AbstractWorkerPool._AbstractWorkerPool__map(self, f, *args, **kwds)
def submit(*argz):
"""send a job to the server"""
_pool = self._serve()
#print("using %s local workers" % _pool.get_ncpus())
try:
return _pool.submit(f, argz, globals=globals())
except pp.DestroyedServerError:
self._is_alive(None)
override = True if 'size' in kwds else False
elem_size = kwds.pop('size', 2)
length = min(len(task) for task in args)
args = zip(*args) #XXX: zip iterator ok? or should be list?
# submit all jobs, to be collected later with 'get()'
tasks = [submit(*task) for task in args]
tasks = [ApplyResult(task) for task in tasks]
# build a correctly sized results object
nodes = self.nodes
if self.nodes in ['*','autodetect',None]:
_pool = self._serve()
nodes = _pool.get_ncpus() #XXX: local workers only?
# try to quickly find a small chunksize that gives good results
maxsize = 2**62 #XXX: HOPEFULLY, this will never be reached...
chunksize = 1
while chunksize < maxsize:
chunksize, extra = divmod(length, nodes * elem_size)
if override: break # the user *wants* to override this loop
if extra >= length: break # we found something that 'works'
elem_size = elem_size * 2
if extra: chunksize += 1
m = MapResult((chunksize,length))
# queue the tasks
m.queue(*tasks)
return m
def amap(self, f, *args, **kwds):
AbstractWorkerPool._AbstractWorkerPool__map(self, f, *args, **kwds)
def submit(*argz):
"""send a job to the server"""
#print "using", __STATE['server'].get_ncpus(), 'local workers'
return __STATE['server'].submit(f, argz, globals=globals())
elem_size = kwds.pop('size', 8) #FIXME: should be size of output type
args = zip(*args)
# submit all jobs, to be collected later with 'get()'
tasks = [submit(*task) for task in args]
tasks = [ApplyResult(task) for task in tasks]
# build a correctly sized results object
length = len(args)
nodes = self.nodes
if self.nodes in ['*','autodetect',None]:
nodes = __STATE['server'].get_ncpus() #XXX: local workers only?
chunksize, extra = divmod(length, nodes * elem_size)
if extra: chunksize += 1
m = MapResult((chunksize,length))
# queue the tasks
m.queue(*tasks)
return m