Пример #1
0
 def amap(self, f, *args, **kwds):
     AbstractWorkerPool._AbstractWorkerPool__map(self, f, *args, **kwds)
     def submit(*argz):
         """send a job to the server"""
        #print "using", __STATE['server'].get_ncpus(), 'local workers'
         return __STATE['server'].submit(f, argz, globals=globals())
     override = True if kwds.has_key('size') else False
     elem_size = kwds.pop('size', 2)
     args = zip(*args)
     # submit all jobs, to be collected later with 'get()'
     tasks = [submit(*task) for task in args]
     tasks = [ApplyResult(task) for task in tasks]
     # build a correctly sized results object
     length = len(args)
     nodes = self.nodes
     if self.nodes in ['*','autodetect',None]:
         nodes = __STATE['server'].get_ncpus() #XXX: local workers only?
     # try to quickly find a small chunksize that gives good results
     maxsize = 2**62 #XXX: HOPEFULLY, this will never be reached...
     chunksize = 1
     while chunksize < maxsize:
         chunksize, extra = divmod(length, nodes * elem_size)
         if override: break # the user *wants* to override this loop
         if extra >= length: break # we found something that 'works'
         elem_size = elem_size * 2
     if extra: chunksize += 1
     m = MapResult((chunksize,length))
     # queue the tasks
     m.queue(*tasks)
     return m
Пример #2
0
 def amap(self, f, *args, **kwds):
     AbstractWorkerPool._AbstractWorkerPool__map(self, f, *args, **kwds)
     def submit(*argz):
         """send a job to the server"""
         _pool = self._serve()
        #print("using %s local workers" % _pool.get_ncpus())
         try:
             return _pool.submit(f, argz, globals=globals())
         except pp.DestroyedServerError:
             self._is_alive(None)
     override = True if 'size' in kwds else False
     elem_size = kwds.pop('size', 2)
     length = min(len(task) for task in args)
     args = zip(*args)  #XXX: zip iterator ok? or should be list?
     # submit all jobs, to be collected later with 'get()'
     tasks = [submit(*task) for task in args]
     tasks = [ApplyResult(task) for task in tasks]
     # build a correctly sized results object
     nodes = self.nodes
     if self.nodes in ['*','autodetect',None]:
         _pool = self._serve()
         nodes = _pool.get_ncpus() #XXX: local workers only?
     # try to quickly find a small chunksize that gives good results
     maxsize = 2**62 #XXX: HOPEFULLY, this will never be reached...
     chunksize = 1
     while chunksize < maxsize:
         chunksize, extra = divmod(length, nodes * elem_size)
         if override: break # the user *wants* to override this loop
         if extra >= length: break # we found something that 'works'
         elem_size = elem_size * 2
     if extra: chunksize += 1
     m = MapResult((chunksize,length))
     # queue the tasks
     m.queue(*tasks)
     return m
Пример #3
0
 def amap(self, f, *args, **kwds):
     AbstractWorkerPool._AbstractWorkerPool__map(self, f, *args, **kwds)
     def submit(*argz):
         """send a job to the server"""
         _pool = self._serve()
        #print("using %s local workers" % _pool.get_ncpus())
         try:
             return _pool.submit(f, argz, globals=globals())
         except pp.DestroyedServerError:
             self._is_alive(None)
     override = True if 'size' in kwds else False
     elem_size = kwds.pop('size', 2)
     length = min(len(task) for task in args)
     args = zip(*args)  #XXX: zip iterator ok? or should be list?
     # submit all jobs, to be collected later with 'get()'
     tasks = [submit(*task) for task in args]
     tasks = [ApplyResult(task) for task in tasks]
     # build a correctly sized results object
     nodes = self.nodes
     if self.nodes in ['*','autodetect',None]:
         _pool = self._serve()
         nodes = _pool.get_ncpus() #XXX: local workers only?
     # try to quickly find a small chunksize that gives good results
     maxsize = 2**62 #XXX: HOPEFULLY, this will never be reached...
     chunksize = 1
     while chunksize < maxsize:
         chunksize, extra = divmod(length, nodes * elem_size)
         if override: break # the user *wants* to override this loop
         if extra >= length: break # we found something that 'works'
         elem_size = elem_size * 2
     if extra: chunksize += 1
     m = MapResult((chunksize,length))
     # queue the tasks
     m.queue(*tasks)
     return m
Пример #4
0
 def amap(self, f, *args, **kwds):
     AbstractWorkerPool._AbstractWorkerPool__map(self, f, *args, **kwds)
     def submit(*argz):
         """send a job to the server"""
        #print "using", __STATE['server'].get_ncpus(), 'local workers'
         return __STATE['server'].submit(f, argz, globals=globals())
     elem_size = kwds.pop('size', 8) #FIXME: should be size of output type
     args = zip(*args)
     # submit all jobs, to be collected later with 'get()'
     tasks = [submit(*task) for task in args]
     tasks = [ApplyResult(task) for task in tasks]
     # build a correctly sized results object
     length = len(args)
     nodes = self.nodes
     if self.nodes in ['*','autodetect',None]:
         nodes = __STATE['server'].get_ncpus() #XXX: local workers only?
     chunksize, extra = divmod(length, nodes * elem_size)
     if extra: chunksize += 1
     m = MapResult((chunksize,length))
     # queue the tasks
     m.queue(*tasks)
     return m