def isValidFunc(function): res = dill.pickles(function) if res: return True else: logger = FlockLogger() logger.error("Your function is not pickable") print("Function not pickable") return False
def isValidFunc(function): test1 = dill.pickles(function) test2 = isfunction(function) if all([test1, test2]): return True else: logger = FlockLogger() logger.error("Your function is not pickable") print("Function not pickable") return False
def apply(self, func, iterator): logger = FlockLogger() if not isValidFunc(func): print( "There is an error with your function. Look at the logging files." ) return tasks = Queue() manager = Manager() results = manager.list() listExSize = min(self.numProcesses, len(iterator)) parentPipe, childPipe = Pipe() progress = self.getProgressBar(sizeIter=len(iterator)) executors = [ Executor(tasks, results, self.setups, childPipe, progress) for _ in range(listExSize) ] for ex in executors: ex.start() self.sendInputs(tasks, func, iterator) # kill each executor processes poisonPills = [(None, None)] * listExSize for pill in poisonPills: tasks.put(pill) # wait for all the results to end for _ in range(listExSize): parentPipe.recv() if self.checkProgress: # finish progress bar queue progress.put(None) if len(results) != len(iterator): logger.error( "The return list object does not have the same size as your input iterator." ) # headshot remaining zombies for ex in executors: ex.terminate() return results
def logErrors(self, localQueue): """ :func:`logErrors` takes as argument the queue to be processed and assess if the process has timed-out :param list localQueue: list of process to be executed """ for asyncRes in localQueue: try: retQ = asyncRes.get(timeout=self.timeOutError) del retQ except Exception as e: fl = FlockLogger() fl.error('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e) print("Some error occur - Check the logs.")
def apply(self, func, iterator): logger = FlockLogger() if not isValidFunc(func): self._anyErrors = True raise FunctionNotPickled tasks = Queue() manager = Manager() results = manager.list() listExSize = min(self.numProcesses, len(iterator)) parentPipe, childPipe = Pipe() progress = self.getProgressBar(sizeIter=len(iterator)) executors = [Executor(tasks, results, self.setups, childPipe, progress) for _ in range(listExSize)] for ex in executors: ex.start() self.sendInputs(tasks, func, iterator) # kill each executor processes poisonPills = [(None, None)] * listExSize for pill in poisonPills: tasks.put(pill) # wait for all the results to end for _ in range(listExSize): parentPipe.recv() if self.checkProgress: # finish progress bar queue progress.put(None) if not self.checkLength(results, len(iterator)): self._anyErrors = True logger.error("The return list object does not have the same size as your input iterator.") # headshot remaining zombies for ex in executors: ex.terminate() # logging some info after running the apply self._outputSize = len(results) self._numUsedProcess = listExSize return results
def split_chunks(objc, _type=None, size=100): # _type is not relevant anymore, how do I handle this case without breaking the interface? # for now I will only ignore the input and send a warning to the user if isinstance(objc, DataFrame): size = min(objc.shape[0], size) _it = array_split(objc, size) elif (isinstance(objc, list)): size = min(len(objc), size) _it = [objc[i::size] for i in range(size)] else: raise Exception("Not supported type") if _type: logger = FlockLogger() logger.warning("The argument _type is not necessary inside the function split_chunks anymore.") return _it
def run(self): # setting up the environment kwargs = {} try: if self.databaseSetup is not None: if not isinstance(self.databaseSetup, list): self.databaseSetup = [self.databaseSetup] for inst in self.databaseSetup: dbPar = inst.parameters parName = inst.name con = inst.server(**dbPar) if parName is not None: kwargs[parName] = con flag = True while True: try: _function, args = self.taskQueue.get() if _function is None: flag = False break else: res = _function(*args, **kwargs) self.sendData(res) except Exception as e: logger = FlockLogger() logger.error("Function failed! Line {} {} {}".format( sys.exc_info()[-1].tb_lineno, type(e).__name__, e)) finally: if (self.progressQueue is not None) and (flag is True): self.progressQueue.put(self.SENTINEL) except Exception as e: logger = FlockLogger() logger.error("Worker failed! - {} - {}".format( type(e).__name__, e)) self.childPipe.send('Job done!') return True
def apply(cls, iterator, function, poolSize=5): """ Method :func:`apply` executes a function asynchronously given an iterator. :param iter iterator: variable in which the function will be applied. :param func function: a function in which is desired to be ran multi-processed. Returns the list with the results of function(iterator). """ logger = FlockLogger() bp = BaseMultiProc(poolSize=poolSize) if not isIter(iterator): logger.error("Variable `iterator` should be iterable.") return res = bp.executeAsync(function, iterator) logger.info("Successful execution of function {}".format( function.__name__)) return res