def isValidFunc(function):
res = dill.pickles(function)
if res:
return True
else:
logger = FlockLogger()
logger.error("Your function is not pickable")
print("Function not pickable")
return False
def isValidFunc(function):
test1 = dill.pickles(function)
test2 = isfunction(function)
if all([test1, test2]):
return True
else:
logger = FlockLogger()
logger.error("Your function is not pickable")
print("Function not pickable")
return False
def apply(self, func, iterator):
logger = FlockLogger()
if not isValidFunc(func):
print(
"There is an error with your function. Look at the logging files."
)
return
tasks = Queue()
manager = Manager()
results = manager.list()
listExSize = min(self.numProcesses, len(iterator))
parentPipe, childPipe = Pipe()
progress = self.getProgressBar(sizeIter=len(iterator))
executors = [
Executor(tasks, results, self.setups, childPipe, progress)
for _ in range(listExSize)
]
for ex in executors:
ex.start()
self.sendInputs(tasks, func, iterator)
# kill each executor processes
poisonPills = [(None, None)] * listExSize
for pill in poisonPills:
tasks.put(pill)
# wait for all the results to end
for _ in range(listExSize):
parentPipe.recv()
if self.checkProgress:
# finish progress bar queue
progress.put(None)
if len(results) != len(iterator):
logger.error(
"The return list object does not have the same size as your input iterator."
)
# headshot remaining zombies
for ex in executors:
ex.terminate()
return results
def logErrors(self, localQueue):
"""
:func:`logErrors` takes as argument the queue to be processed and assess if the process has timed-out
:param list localQueue: list of process to be executed
"""
for asyncRes in localQueue:
try:
retQ = asyncRes.get(timeout=self.timeOutError)
del retQ
except Exception as e:
fl = FlockLogger()
fl.error('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
print("Some error occur - Check the logs.")
def apply(self, func, iterator):
logger = FlockLogger()
if not isValidFunc(func):
self._anyErrors = True
raise FunctionNotPickled
tasks = Queue()
manager = Manager()
results = manager.list()
listExSize = min(self.numProcesses, len(iterator))
parentPipe, childPipe = Pipe()
progress = self.getProgressBar(sizeIter=len(iterator))
executors = [Executor(tasks, results, self.setups, childPipe, progress) for _ in range(listExSize)]
for ex in executors:
ex.start()
self.sendInputs(tasks, func, iterator)
# kill each executor processes
poisonPills = [(None, None)] * listExSize
for pill in poisonPills:
tasks.put(pill)
# wait for all the results to end
for _ in range(listExSize):
parentPipe.recv()
if self.checkProgress:
# finish progress bar queue
progress.put(None)
if not self.checkLength(results, len(iterator)):
self._anyErrors = True
logger.error("The return list object does not have the same size as your input iterator.")
# headshot remaining zombies
for ex in executors:
ex.terminate()
# logging some info after running the apply
self._outputSize = len(results)
self._numUsedProcess = listExSize
return results
def split_chunks(objc, _type=None, size=100):
# _type is not relevant anymore, how do I handle this case without breaking the interface?
# for now I will only ignore the input and send a warning to the user
if isinstance(objc, DataFrame):
size = min(objc.shape[0], size)
_it = array_split(objc, size)
elif (isinstance(objc, list)):
size = min(len(objc), size)
_it = [objc[i::size] for i in range(size)]
else:
raise Exception("Not supported type")
if _type:
logger = FlockLogger()
logger.warning("The argument _type is not necessary inside the function split_chunks anymore.")
return _it
def run(self):
# setting up the environment
kwargs = {}
try:
if self.databaseSetup is not None:
if not isinstance(self.databaseSetup, list):
self.databaseSetup = [self.databaseSetup]
for inst in self.databaseSetup:
dbPar = inst.parameters
parName = inst.name
con = inst.server(**dbPar)
if parName is not None:
kwargs[parName] = con
flag = True
while True:
try:
_function, args = self.taskQueue.get()
if _function is None:
flag = False
break
else:
res = _function(*args, **kwargs)
self.sendData(res)
except Exception as e:
logger = FlockLogger()
logger.error("Function failed! Line {} {} {}".format(
sys.exc_info()[-1].tb_lineno,
type(e).__name__, e))
finally:
if (self.progressQueue is not None) and (flag is True):
self.progressQueue.put(self.SENTINEL)
except Exception as e:
logger = FlockLogger()
logger.error("Worker failed! - {} - {}".format(
type(e).__name__, e))
self.childPipe.send('Job done!')
return True
def apply(cls, iterator, function, poolSize=5):
"""
Method :func:`apply` executes a function asynchronously given an iterator.
:param iter iterator: variable in which the function will be applied.
:param func function: a function in which is desired to be ran multi-processed.
Returns the list with the results of function(iterator).
"""
logger = FlockLogger()
bp = BaseMultiProc(poolSize=poolSize)
if not isIter(iterator):
logger.error("Variable `iterator` should be iterable.")
return
res = bp.executeAsync(function, iterator)
logger.info("Successful execution of function {}".format(
function.__name__))
return res