def parallel_map(task, execute=True):
tools = task.AllETLTools;
index = extends.getindex(tools, lambda d: isinstance(d, ToListTF));
if index == -1:
index = 0;
tool = tools[index];
generator = tool.process(None);
else:
generator = generate(tools[:index],None, execute=execute);
return generator;
def parallel_map(task, execute=True):
tools = task.AllETLTools;
index = extends.getindex(tools, lambda d: isinstance(d, ToListTF));
if index == -1:
index = 0;
tool = tools[index];
generator = tool.process(None);
else:
generator = generate(tools[:index],None, execute=execute);
return generator;
def mThreadExecute(self, threadcount=10):
import threadpool
pool = threadpool.ThreadPool(threadcount)
tools = [tool for tool in self.AllETLTools if tool.Enabled];
index = extends.getindex(tools, lambda d: d.Type == 'ToListTF');
if index == -1:
index = 0;
tool = tools[index];
generator = tool.Func(tool, None);
else:
generator = self.__generate__(tools[:index]);
def Funcs(item):
mgenerator = self.__generate__(tools[index + 1:], (r for r in [item]), True);
for r in mgenerator:
pass;
print('finish' + str(item));
requests = threadpool.makeRequests(Funcs, generator);
[pool.putRequest(req) for req in requests]
pool.wait()
def mThreadExecute(self, threadcount=10):
import threadpool
pool = threadpool.ThreadPool(threadcount)
tools = [tool for tool in self.AllETLTools if tool.Enabled];
index = extends.getindex(tools, lambda d: d.Type == 'ToListTF');
if index == -1:
index = 0;
tool = tools[index];
generator = tool.Func(tool, None);
else:
generator = self.__generate__(tools[:index]);
def Funcs(item):
mgenerator = self.__generate__(tools[index + 1:], (r for r in [item]), True);
for r in mgenerator:
pass;
print('finish' + str(item));
requests = threadpool.makeRequests(Funcs, generator);
[pool.putRequest(req) for req in requests]
pool.wait()
def parallel_reduce(task,generator=None, execute=True):
tools = task.AllETLTools;
index = extends.getindex(tools, lambda d: isinstance(d,ToListTF));
index =0 if index==-1 else index;
generator = generate(tools[index + 1:], generator, execute);
return generator;
def parallel_reduce(task,generator=None, execute=True):
tools = task.AllETLTools;
index = extends.getindex(tools, lambda d: isinstance(d,ToListTF));
index =0 if index==-1 else index;
generator = generate(tools[index + 1:], generator, execute);
return generator;