def execute(self, call, inputs, outputs, priority=1, collating=True):
"""
Schedules task `call inputs outputs`, either a single collating call or one call per input
"""
try:
self._mutex.acquire()
if self._all_inputs_available(inputs):
inputPaths = self._paths_of(inputs)
if collating:
self._created += 1
request = threadpool.WorkRequest(self._process_step, [call, self._created, inputPaths, outputs], priority=priority)
self._pool.putRequest(request)
else:
self._counts[outputs[0]] = len(inputPaths)
for input in inputPaths:
self._created += 1
request = threadpool.WorkRequest(self._process_noncollating_step, [call, self._created, [input], outputs], priority=priority)
self._pool.putRequest(request)
else:
if collating:
handler = self._process_step
else:
handler = self._process_noncollating_step
self._created += 1
request = threadpool.WorkRequest(handler, [call, self._created, inputs, outputs], priority=priority)
self._backlog.append(request)
finally:
self._mutex.release()
def updateHoldings():
pool=threadpool.ThreadPool(5)
for i in insts[0:5]:
pool.putRequest(threadpool.WorkRequest(od.updateCOT,[i]))
pool.putRequest(threadpool.WorkRequest(od.updateHPR,[i]))
pool.wait()
def execute(self, call, inputs, outputs, parameters=list(), priority=1, collating=True, log_prefix=None):
"""
Schedules task `call parameters inputs outputs`, either a single collating call or one call per input
"""
try:
self._mutex.acquire()
if log_prefix is None:
log_prefix = call[call.rfind('/') + 1:]
if log_prefix.endswith('.sh'):
log_prefix = log_prefix[:-3]
self._created += 1
request = threadpool.WorkRequest(None, [call, self._created, parameters, None, outputs, None, log_prefix],
priority=priority, request_id=self._created)
for o in outputs:
if o in self._counts:
self._counts[o] += 1
else:
self._counts[o] = 1
if self._all_inputs_available(inputs) and self._constraints_fulfilled(request):
input_paths = self._paths_of(inputs)
if collating:
request.callable = self._process_step
request.args[3] = input_paths
self._pool.put_request(request)
if self._delay is not None:
time.sleep(self._delay)
else:
for i in range(len(input_paths)):
if i == 0:
request.callable = self._process_step
request.args[3] = input_paths[0:1]
else:
self._created += 1
request = threadpool.WorkRequest(self._process_step,
[call, self._created, parameters, input_paths[i:i + 1],
outputs, None, log_prefix],
priority=priority, request_id=self._created)
for o in outputs:
self._counts[o] += 1
if i == 0 or self._constraints_fulfilled(request):
self._pool.put_request(request)
if self._delay is not None:
time.sleep(self._delay)
else:
self._backlog.append(request)
else:
if collating:
request.callable = self._translate_step
else:
request.callable = self._expand_step
request.args[3] = inputs
self._backlog.append(request)
finally:
self._mutex.release()
def get_restore_list(self):
request_list = []
for action in self.restore_list.keys():
if action == 'maintenance':
for host in self.restore_list[action]:
request_list.append(
threadpool.WorkRequest(self.call_func,
args=(host, action, ('exit',))))
continue
if action == 'disconnect':
for host in self.restore_list[action]:
request_list.append(
threadpool.WorkRequest(self.call_func,
args=(host, 'reconnect', ())))
return request_list
def onRunClicked(self):
self.widgetSuc.clear()
self.widgetFai.clear()
tp, cmdText, matchCondition = threadpool.ThreadPool(min(len(self.labelShells), 20)), self.editCmd.toPlainText(), self.editSc.toPlainText()
for labelShell in self.labelShells:
tp.putRequest(threadpool.WorkRequest(self.request, (cmdText, matchCondition, labelShell)))
tp.wait()
def _check_for_mature_tasks(self):
"""
Checks tasks in backlog whether inputs are (now) all bound in products map
adds these mature tasks to thread pool, removes them from backlog
distinguishes collocating and non-collocating asks by the callable used
generates one task per input for non-collocating tasks
"""
matureTasks = []
for task in self._backlog:
if self._all_inputs_available(task.args[2]):
inputPaths = self._paths_of(task.args[2])
if task.callable == self._process_step:
task.args[2] = inputPaths
self._pool.putRequest(task)
else:
self._counts[task.args[3][0]] = len(inputPaths)
for input in inputPaths:
self._created += 1
request = threadpool.WorkRequest(self._process_noncollating_step, \
[task.args[0], self._created, [input], task.args[3]], \
priority=task.priority)
self._pool.putRequest(request)
matureTasks.append(task)
for task in matureTasks:
self._backlog.remove(task)
def map_domains():
resolves = {}
pool = threadpool.ThreadPool(10)
domains = fetch_gfw_domains()
for d in domains:
worker = threadpool.WorkRequest(callable_=resolve_domain,
args=(d, resolves))
pool.putRequest(worker)
pool.wait()
iptables = open("iptables.sh", "w")
dnsmasq = open("dnsmasq.conf", "w")
iptables.write("#!/bin/sh\n")
iptables.write("# HOST2IPTABLES BY SOL\n")
iptables.write("# %s\n" % datetime.datetime.now())
iptables.write("iptables -t nat -N SHADOWSOCKS\n")
iptables.write("iptables -t nat -A PREROUTING -p tcp -j SHADOWSOCKS\n")
dnsmasq.write("# HOST2DNSMASQ BY SOL\n")
dnsmasq.write("# %s\n" % datetime.datetime.now())
for k in resolves.keys():
iptables.write(
"iptables -t nat -A SHADOWSOCKS -d %s -p tcp -j REDIRECT --to-ports 1080\n"
% resolves[k])
dnsmasq.write("address=/%s/%s\n" % (k, resolves[k]))
iptables.flush()
dnsmasq.flush()
iptables.close()
dnsmasq.close()
def run_step(self, name, fns, *args, **kwargs):
# Default threads #
if '.' in name: threads = kwargs.pop('threads', False)
else: threads = kwargs.pop('threads', True)
# Start timer #
start_time = time.time()
# Message #
if self.color: print("Running step: " + Color.f_grn + name + Color.end)
else: print("Running step: " + name)
sys.stdout.flush()
# Threads #
if threads and len(list(fns)) > 1:
import threadpool
self.thpool = threadpool.ThreadPool(8)
for fn in fns: self.thpool.putRequest(threadpool.WorkRequest(fn))
self.thpool.wait()
self.thpool.dismissWorkers(8)
del self.thpool
else:
for fn in fns: fn(*args, **kwargs)
# Stop timer #
run_time = datetime.timedelta(seconds=round(time.time() - start_time))
if self.color: print(Color.ylw + "Run time: '%s'" % (run_time) + Color.end)
else: print("Run time: '%s'" % (run_time))
sys.stdout.flush()
def method1():
pool = threadpool.ThreadPool(10)
for i in xrange(0,1000):
wr = threadpool.WorkRequest(testMysql, args=[{"one":1, "two":3.14}], callback=workDone)
pool.putRequest(wr)
pool.wait()
def test_full_pipeline(self):
batch_size = 200
base = os.getcwd() + "/"
seq_format = "fasta"
db_dir = "/zenodotus/dat01/mason_lab_scratch_reference/cmlab/GENOMES/BLAST/"
pool = threadpool.ThreadPool(1)
ids = []
try:
ids = process_seqs.fetch_assembly_ids(batch_size)
except Exception as inst:
sys.stderr.write(
"Error converting non-virus assembly ids to bioproject ids\n")
classifications = process_seqs.fetch_classifications(ids[:30])
self.assertEqual(len(classifications), 30)
# get the asids that have classifications and ignore rest
asids = classifications.keys()[:6]
for ncbi_asid in asids:
request = threadpool.WorkRequest(process_seqs.process_genomes,
args=[
base, ncbi_asid,
classifications, seq_format,
db_dir, True, 0, 2
])
pool.putRequest(request)
pool.wait()
for asid in asids:
shutil.rmtree('./%s' % asid)
def DownloadImage(url):
bs = helper.GetBs(url)
#print(bs)
if not bs:
#print("下载错误")
return False
try:
dir_name = bs.select('.main-title')[0].text
#print(dir_name)
dir_tmp = dir_name
try_count = 1
while os.path.exists(dir_tmp):
dir_tmp = "%s%d" % (dir_name, try_count)
try_count += 1
dir_name = dir_tmp
try:
os.mkdir(dir_name)
except Exception:
os.mkdir(str(uuid.uuid1()))
img_num = int(bs.select('.pagenavi a')[-2].string)
#print(img_num)
pool = threadpool.ThreadPool(8)
for i in range(img_num):
img_url = "%s/%d" % (url, i + 1)
#print(img_url)
req = threadpool.WorkRequest(__task, [img_url, dir_name, i])
pool.putRequest(req)
pool.wait()
print(url, "下载完成")
return True
except Exception as ex:
print("error", ex, "\t", url)
return False
def __decorator(*arg):
list_var1 = arg
reult = None
request = threadpool.WorkRequest(func, list_var1, callback=reult)
print("start by thread")
pool.putRequest(request)
pool.wait()
return reult
def add_job(self, job):
'''Add a new callable to the queue.
Args:
* j: A thunk (no-variable callable) that will be put in the
work queue.
'''
self.pool.putRequest(threadpool.WorkRequest(job))
def add_job(self, feed):
""" adds a feed processing job to the pool
"""
if self.tpool:
req = threadpool.WorkRequest(self.process_feed_wrapper, (feed, ))
self.tpool.putRequest(req)
else:
# no threadpool module, just run the job
self.process_feed_wrapper(feed)
def _check_for_mature_tasks(self):
"""
Checks tasks in backlog whether inputs are (now) all bound in products map
adds these mature tasks to thread pool, removes them from backlog
distinguishes collating and non-collating tasks by the callable used
generates one task per input for non-collating tasks
"""
mature_tasks = []
for task in self._backlog:
if task.callable == self._process_step or self._all_inputs_available(task.args[3]):
if self._constraints_fulfilled(task):
mature_tasks.append(task)
if task.callable == self._process_step:
input_paths = task.args[3]
else:
input_paths = self._paths_of(task.args[3])
if task.callable == self._translate_step or task.callable == self._process_step:
task.callable = self._process_step
task.args[3] = input_paths
self._pool.put_request(task)
if self._delay is not None:
time.sleep(self._delay)
else:
new_tasks = []
pos = self._backlog.index(task)
for i in range(len(input_paths)):
if i == 0:
task.callable = self._process_step
task.args[3] = input_paths[0:1]
else:
self._created += 1
task = threadpool.WorkRequest(self._process_step,
[task.args[0], self._created, task.args[2],
input_paths[i:i + 1], task.args[4], None, task.args[6]],
priority=task.priority, request_id=self._created)
self._counts[task.args[4][0]] += 1
if i == 0 or self._constraints_fulfilled(task):
self._pool.put_request(task)
if self._delay is not None:
time.sleep(self._delay)
else:
new_tasks.insert(0, task)
if len(new_tasks) > 0:
for newTask in new_tasks:
self._backlog.insert(pos + 1, newTask)
# be unfair!
pass
# be fair!
# break
else:
# be unfair!
pass
# be fair!
# break
for task in mature_tasks:
self._backlog.remove(task)
def _get_request(self, host, action):
func_dict = {'maintenance': (),
'reboot': (),
'disconnect': ()}
if action != 'reboot':
if action not in self.restore_list.keys():
self.restore_list[action] = []
self.restore_list[action].append(host)
return threadpool.WorkRequest(self.call_func,
args=(host, action, func_dict[action]))
def updateHKStock():
table_names=open('ini/HKStockList.txt').read().split(',')
pool=threadpool.ThreadPool(5)
for name in table_names:
path='%s/%s.db' % (hs.savePath,name)
print(path)
workrequest=threadpool.WorkRequest(hs.updateStockData,[name])
pool.putRequest(workrequest)
pool.wait()
def test_steady_proxy(proxy_list, log):
log.info('开始进行代理测试..')
start_time = time.time()
pool = threadpool.ThreadPool(50)
for index, item in enumerate(proxy_list):
req = threadpool.WorkRequest(test_run_task, [item, index, log])
pool.putRequest(req)
pool.wait()
end_time = time.time()
log.info('代理测试完成..')
log.info('use time: {use_time}'.format(use_time=end_time - start_time))
def upload(self):
files = self.uploadFiles.toPlainText().split("\n")
if len(files) < 1:
QtWidgets.QMessageBox(None, "Tooltip", "Please select upload file at first!").exec_()
return
uploadDir = self.editDst.toPlainText()
self.widgetSuc.clear()
self.widgetFai.clear()
tp = threadpool.ThreadPool(min(len(self.labelShells), 20))
for labelShell in self.labelShells:
tp.putRequest(threadpool.WorkRequest(self.request, (files, uploadDir, labelShell)))
tp.wait()
def callbackfunc(request,result):
res,resource,pagebuf = result
if pagebuf == None :
return
hreflist = fetchPage.parsePage(pagebuf, resource)
for href in hreflist :
if PAGESCache.get(href,None) == None : PAGESCache[href] = True
else : continue
hostname,filename = fetchPage.parse(href)
main.putRequest(threadpool.WorkRequest(fetchPage.downPage,args=[hostname,filename],kwds={},callback=callbackfunc))
fetchPage.dealwithResult(res,resource)
def run(self):
"""连续运行"""
p = threadpool.ThreadPool(self.POOL_SIZE)
while self.active:
for url, callback in self.taskList:
req = threadpool.WorkRequest(self._excutor, (url, callback))
p.putRequest(req)
st = time()
p.wait()
elapse = time() - st
need_sleep = max(self.taskInterval - elapse, 0)
print '####', elapse, need_sleep
sleep(need_sleep)
def _get_request(self, vm, action):
if action == 'migrate':
host = vm.get_host()
dss = host.get_datastores()
size = vm.get_size()
vm_ds = None
random.shuffle(dss)
for ds in dss:
if ds.get_freespace() > size:
vm_ds = ds
break
if vm_ds is None:
vm_ds = vm.get_datastores[0]
return threadpool.WorkRequest(self.call_func,
args=(vm, action, (
host,
vm_ds,
)))
randstr = utils.get_randstr(3)
func_dict = {
'clone': (vm.name() + '_cln_' + randstr, ),
'snapshot': (vm.name() + '_snp_' + randstr, ),
'poweron': (),
'poweroff': (),
'reset': (),
'destroy': (),
'unregister': ()
}
if action not in self.restore_list.keys():
self.restore_list[action] = []
if action == 'snapshot' or action == 'poweron' or action == 'poweroff':
self.restore_list[action].append(vm)
elif action == 'clone':
self.restore_list[action].append(vm.name() + '_cln_' + randstr)
return threadpool.WorkRequest(self.call_func,
args=(vm, action, func_dict[action]))
def send_apn(self, token):
token64 = base64.encodestring(token)
udid = self.devices.get(token64)
if udid is None:
logMsg(_MODULE_ID, LOG_WARING,
"mdmapn invalid token, [%s]" % token64)
else:
#this is user the worker to send
#
# apn_worker = self.choose_worker()
# req = threadpool.WorkRequest(apn_worker.send_apn, (udid,))
# self.pool.putRequest(req)
#THis is just use the function to send apn
req = threadpool.WorkRequest(send_apn_worker, (udid, ))
self.pool.putRequest(req)
def serve_forever(self, poll_interval=0.5):
pool_size = cpu_count() * 2 + 1
self.pool = threadpool.ThreadPool(pool_size)
self.pool.putRequest(
threadpool.WorkRequest(self.serve_forever_thread,
args=[poll_interval]))
try:
while True:
time.sleep(0.001)
self.pool.poll()
except KeyboardInterrupt:
global srvr
srvr.shutdown()
srvr.server_close()
srvr = None
finally:
print("destory all threads before exit...")
self.pool.dismissWorkers(pool_size, do_join=True)
def addTask(self, id, callable_, args):
self.mutex.acquire()
taskControler = None
if id in self.taskControlerMap:
taskControler = self.taskControlerMap[id]
else:
taskControler = TaskContoler()
self.taskControlerMap[id] = taskControler
reqId = time.clock()
taskControler.addWaiter(reqId)
taskControler.setOwner(reqId)
reqArgs = [reqId, taskControler, callable_, args]
req = threadpool.WorkRequest(_do_task, reqArgs, None, callback=_task_return,
exc_callback=_handle_task_exception)
self.pool.putRequest(req)
self.mutex.release()
def DownloadImagePage(url):
bs = helper.GetBs(url)
#print(bs)
if not bs: return False
try:
all_li = bs.select('ul[id="pins"]>li')
pool = threadpool.ThreadPool(8)
for li in all_li:
if li.get('class') is None: #屏蔽广告
href = li.a.get('href')
title = li.a.img.get('alt')
#print(href,title)
req = threadpool.WorkRequest(__task, [href])
pool.putRequest(req)
pool.wait()
print(url, "整页下载完成")
return True
except Exception as ex:
print(ex)
return False
def updateInstrument():
global errors
errors={}
pool=threadpool.ThreadPool(5)
def callBack(req,out):
if len(out)>0:
errors[req.kwds['instrument']]=out
errorlog=open('error_instrument.txt','w')
for i in insts:
print(i)
req=threadpool.WorkRequest(od.update,kwds={'instrument':i},callback=callBack)
pool.putRequest(req)
pool.wait()
errorlog.write(json.dumps(errors))
errorlog.close()
def usingThreadpool(limit,num_thread):
urlset = open(config.SEED_FILE, "r")
start = datetime.datetime.now()
for url in urlset :
try :
if PAGESCache.get(url,None) == None : PAGESCache[url] = True
else : continue
hostname , filename = fetchPage.parse(url)
req = threadpool.WorkRequest(fetchPage.downPage,args=[hostname,filename],kwds={},callback=callbackfunc)
main.putRequest(req)
except Exception:
pass
while True:
try:
main.poll()
if config.total_url >= limit : break
except threadpool.NoResultsPending:
print "no pending results"
break
except Exception ,e:
pass
def execute(self):
log.debug("开始抓取夜不语小说...")
load_engine = LoadEngine(__baseUrl__)
chapters = YebuyuMuLuResolve(load_engine.execute()).execute()
self.note.chapters = chapters
has_exist = database.insert_note_if_not_exist(self.connection,
self.note)
if has_exist:
chapters = self.find_error_and_update_chapters()
requests = list()
if chapters is not None and len(chapters) != 0:
for chapter in chapters:
requests.append(
threadpool.WorkRequest(YebuyuEngine.__resolve_chapter,
(self, chapter)))
[g_pool.putRequest(req) for req in requests]
g_pool.wait()
log.debug("结束抓取夜不语小说" + self.note.simple_str())
return self.note
def vmotion(dc, vm_reg, host_name, ds_name, concurrency=10):
host = dc.get_host_by_name(host_name)
ds = dc.get_datastore_by_name(ds_name)
if host is None:
print 'Host {} not exist on DC.'.format(host_name)
exit(1)
if ds is None:
print 'Datastore {} not exist on DC.'.format(ds_name)
exit(1)
vms = dc.get_vms_by_regex(utils.get_items(vm_reg))
vm_num = len(vms)
import threadpool
size = concurrency if concurrency < vm_num else vm_num
pool = threadpool.ThreadPool(size)
reqs = [
threadpool.WorkRequest(call_func, args=(vm, 'migrate', (host, ds)))
for vm in vms
]
for req in reqs:
pool.putRequest(req)
pool.wait()
exit(0)
