def execute(self, call, inputs, outputs, priority=1, collating=True): """ Schedules task `call inputs outputs`, either a single collating call or one call per input """ try: self._mutex.acquire() if self._all_inputs_available(inputs): inputPaths = self._paths_of(inputs) if collating: self._created += 1 request = threadpool.WorkRequest(self._process_step, [call, self._created, inputPaths, outputs], priority=priority) self._pool.putRequest(request) else: self._counts[outputs[0]] = len(inputPaths) for input in inputPaths: self._created += 1 request = threadpool.WorkRequest(self._process_noncollating_step, [call, self._created, [input], outputs], priority=priority) self._pool.putRequest(request) else: if collating: handler = self._process_step else: handler = self._process_noncollating_step self._created += 1 request = threadpool.WorkRequest(handler, [call, self._created, inputs, outputs], priority=priority) self._backlog.append(request) finally: self._mutex.release()
def updateHoldings(): pool=threadpool.ThreadPool(5) for i in insts[0:5]: pool.putRequest(threadpool.WorkRequest(od.updateCOT,[i])) pool.putRequest(threadpool.WorkRequest(od.updateHPR,[i])) pool.wait()
def execute(self, call, inputs, outputs, parameters=list(), priority=1, collating=True, log_prefix=None): """ Schedules task `call parameters inputs outputs`, either a single collating call or one call per input """ try: self._mutex.acquire() if log_prefix is None: log_prefix = call[call.rfind('/') + 1:] if log_prefix.endswith('.sh'): log_prefix = log_prefix[:-3] self._created += 1 request = threadpool.WorkRequest(None, [call, self._created, parameters, None, outputs, None, log_prefix], priority=priority, request_id=self._created) for o in outputs: if o in self._counts: self._counts[o] += 1 else: self._counts[o] = 1 if self._all_inputs_available(inputs) and self._constraints_fulfilled(request): input_paths = self._paths_of(inputs) if collating: request.callable = self._process_step request.args[3] = input_paths self._pool.put_request(request) if self._delay is not None: time.sleep(self._delay) else: for i in range(len(input_paths)): if i == 0: request.callable = self._process_step request.args[3] = input_paths[0:1] else: self._created += 1 request = threadpool.WorkRequest(self._process_step, [call, self._created, parameters, input_paths[i:i + 1], outputs, None, log_prefix], priority=priority, request_id=self._created) for o in outputs: self._counts[o] += 1 if i == 0 or self._constraints_fulfilled(request): self._pool.put_request(request) if self._delay is not None: time.sleep(self._delay) else: self._backlog.append(request) else: if collating: request.callable = self._translate_step else: request.callable = self._expand_step request.args[3] = inputs self._backlog.append(request) finally: self._mutex.release()
def get_restore_list(self): request_list = [] for action in self.restore_list.keys(): if action == 'maintenance': for host in self.restore_list[action]: request_list.append( threadpool.WorkRequest(self.call_func, args=(host, action, ('exit',)))) continue if action == 'disconnect': for host in self.restore_list[action]: request_list.append( threadpool.WorkRequest(self.call_func, args=(host, 'reconnect', ()))) return request_list
def onRunClicked(self): self.widgetSuc.clear() self.widgetFai.clear() tp, cmdText, matchCondition = threadpool.ThreadPool(min(len(self.labelShells), 20)), self.editCmd.toPlainText(), self.editSc.toPlainText() for labelShell in self.labelShells: tp.putRequest(threadpool.WorkRequest(self.request, (cmdText, matchCondition, labelShell))) tp.wait()
def _check_for_mature_tasks(self): """ Checks tasks in backlog whether inputs are (now) all bound in products map adds these mature tasks to thread pool, removes them from backlog distinguishes collocating and non-collocating asks by the callable used generates one task per input for non-collocating tasks """ matureTasks = [] for task in self._backlog: if self._all_inputs_available(task.args[2]): inputPaths = self._paths_of(task.args[2]) if task.callable == self._process_step: task.args[2] = inputPaths self._pool.putRequest(task) else: self._counts[task.args[3][0]] = len(inputPaths) for input in inputPaths: self._created += 1 request = threadpool.WorkRequest(self._process_noncollating_step, \ [task.args[0], self._created, [input], task.args[3]], \ priority=task.priority) self._pool.putRequest(request) matureTasks.append(task) for task in matureTasks: self._backlog.remove(task)
def map_domains(): resolves = {} pool = threadpool.ThreadPool(10) domains = fetch_gfw_domains() for d in domains: worker = threadpool.WorkRequest(callable_=resolve_domain, args=(d, resolves)) pool.putRequest(worker) pool.wait() iptables = open("iptables.sh", "w") dnsmasq = open("dnsmasq.conf", "w") iptables.write("#!/bin/sh\n") iptables.write("# HOST2IPTABLES BY SOL\n") iptables.write("# %s\n" % datetime.datetime.now()) iptables.write("iptables -t nat -N SHADOWSOCKS\n") iptables.write("iptables -t nat -A PREROUTING -p tcp -j SHADOWSOCKS\n") dnsmasq.write("# HOST2DNSMASQ BY SOL\n") dnsmasq.write("# %s\n" % datetime.datetime.now()) for k in resolves.keys(): iptables.write( "iptables -t nat -A SHADOWSOCKS -d %s -p tcp -j REDIRECT --to-ports 1080\n" % resolves[k]) dnsmasq.write("address=/%s/%s\n" % (k, resolves[k])) iptables.flush() dnsmasq.flush() iptables.close() dnsmasq.close()
def run_step(self, name, fns, *args, **kwargs): # Default threads # if '.' in name: threads = kwargs.pop('threads', False) else: threads = kwargs.pop('threads', True) # Start timer # start_time = time.time() # Message # if self.color: print("Running step: " + Color.f_grn + name + Color.end) else: print("Running step: " + name) sys.stdout.flush() # Threads # if threads and len(list(fns)) > 1: import threadpool self.thpool = threadpool.ThreadPool(8) for fn in fns: self.thpool.putRequest(threadpool.WorkRequest(fn)) self.thpool.wait() self.thpool.dismissWorkers(8) del self.thpool else: for fn in fns: fn(*args, **kwargs) # Stop timer # run_time = datetime.timedelta(seconds=round(time.time() - start_time)) if self.color: print(Color.ylw + "Run time: '%s'" % (run_time) + Color.end) else: print("Run time: '%s'" % (run_time)) sys.stdout.flush()
def method1(): pool = threadpool.ThreadPool(10) for i in xrange(0,1000): wr = threadpool.WorkRequest(testMysql, args=[{"one":1, "two":3.14}], callback=workDone) pool.putRequest(wr) pool.wait()
def test_full_pipeline(self): batch_size = 200 base = os.getcwd() + "/" seq_format = "fasta" db_dir = "/zenodotus/dat01/mason_lab_scratch_reference/cmlab/GENOMES/BLAST/" pool = threadpool.ThreadPool(1) ids = [] try: ids = process_seqs.fetch_assembly_ids(batch_size) except Exception as inst: sys.stderr.write( "Error converting non-virus assembly ids to bioproject ids\n") classifications = process_seqs.fetch_classifications(ids[:30]) self.assertEqual(len(classifications), 30) # get the asids that have classifications and ignore rest asids = classifications.keys()[:6] for ncbi_asid in asids: request = threadpool.WorkRequest(process_seqs.process_genomes, args=[ base, ncbi_asid, classifications, seq_format, db_dir, True, 0, 2 ]) pool.putRequest(request) pool.wait() for asid in asids: shutil.rmtree('./%s' % asid)
def DownloadImage(url): bs = helper.GetBs(url) #print(bs) if not bs: #print("下载错误") return False try: dir_name = bs.select('.main-title')[0].text #print(dir_name) dir_tmp = dir_name try_count = 1 while os.path.exists(dir_tmp): dir_tmp = "%s%d" % (dir_name, try_count) try_count += 1 dir_name = dir_tmp try: os.mkdir(dir_name) except Exception: os.mkdir(str(uuid.uuid1())) img_num = int(bs.select('.pagenavi a')[-2].string) #print(img_num) pool = threadpool.ThreadPool(8) for i in range(img_num): img_url = "%s/%d" % (url, i + 1) #print(img_url) req = threadpool.WorkRequest(__task, [img_url, dir_name, i]) pool.putRequest(req) pool.wait() print(url, "下载完成") return True except Exception as ex: print("error", ex, "\t", url) return False
def __decorator(*arg): list_var1 = arg reult = None request = threadpool.WorkRequest(func, list_var1, callback=reult) print("start by thread") pool.putRequest(request) pool.wait() return reult
def add_job(self, job): '''Add a new callable to the queue. Args: * j: A thunk (no-variable callable) that will be put in the work queue. ''' self.pool.putRequest(threadpool.WorkRequest(job))
def add_job(self, feed): """ adds a feed processing job to the pool """ if self.tpool: req = threadpool.WorkRequest(self.process_feed_wrapper, (feed, )) self.tpool.putRequest(req) else: # no threadpool module, just run the job self.process_feed_wrapper(feed)
def _check_for_mature_tasks(self): """ Checks tasks in backlog whether inputs are (now) all bound in products map adds these mature tasks to thread pool, removes them from backlog distinguishes collating and non-collating tasks by the callable used generates one task per input for non-collating tasks """ mature_tasks = [] for task in self._backlog: if task.callable == self._process_step or self._all_inputs_available(task.args[3]): if self._constraints_fulfilled(task): mature_tasks.append(task) if task.callable == self._process_step: input_paths = task.args[3] else: input_paths = self._paths_of(task.args[3]) if task.callable == self._translate_step or task.callable == self._process_step: task.callable = self._process_step task.args[3] = input_paths self._pool.put_request(task) if self._delay is not None: time.sleep(self._delay) else: new_tasks = [] pos = self._backlog.index(task) for i in range(len(input_paths)): if i == 0: task.callable = self._process_step task.args[3] = input_paths[0:1] else: self._created += 1 task = threadpool.WorkRequest(self._process_step, [task.args[0], self._created, task.args[2], input_paths[i:i + 1], task.args[4], None, task.args[6]], priority=task.priority, request_id=self._created) self._counts[task.args[4][0]] += 1 if i == 0 or self._constraints_fulfilled(task): self._pool.put_request(task) if self._delay is not None: time.sleep(self._delay) else: new_tasks.insert(0, task) if len(new_tasks) > 0: for newTask in new_tasks: self._backlog.insert(pos + 1, newTask) # be unfair! pass # be fair! # break else: # be unfair! pass # be fair! # break for task in mature_tasks: self._backlog.remove(task)
def _get_request(self, host, action): func_dict = {'maintenance': (), 'reboot': (), 'disconnect': ()} if action != 'reboot': if action not in self.restore_list.keys(): self.restore_list[action] = [] self.restore_list[action].append(host) return threadpool.WorkRequest(self.call_func, args=(host, action, func_dict[action]))
def updateHKStock(): table_names=open('ini/HKStockList.txt').read().split(',') pool=threadpool.ThreadPool(5) for name in table_names: path='%s/%s.db' % (hs.savePath,name) print(path) workrequest=threadpool.WorkRequest(hs.updateStockData,[name]) pool.putRequest(workrequest) pool.wait()
def test_steady_proxy(proxy_list, log): log.info('开始进行代理测试..') start_time = time.time() pool = threadpool.ThreadPool(50) for index, item in enumerate(proxy_list): req = threadpool.WorkRequest(test_run_task, [item, index, log]) pool.putRequest(req) pool.wait() end_time = time.time() log.info('代理测试完成..') log.info('use time: {use_time}'.format(use_time=end_time - start_time))
def upload(self): files = self.uploadFiles.toPlainText().split("\n") if len(files) < 1: QtWidgets.QMessageBox(None, "Tooltip", "Please select upload file at first!").exec_() return uploadDir = self.editDst.toPlainText() self.widgetSuc.clear() self.widgetFai.clear() tp = threadpool.ThreadPool(min(len(self.labelShells), 20)) for labelShell in self.labelShells: tp.putRequest(threadpool.WorkRequest(self.request, (files, uploadDir, labelShell))) tp.wait()
def callbackfunc(request,result): res,resource,pagebuf = result if pagebuf == None : return hreflist = fetchPage.parsePage(pagebuf, resource) for href in hreflist : if PAGESCache.get(href,None) == None : PAGESCache[href] = True else : continue hostname,filename = fetchPage.parse(href) main.putRequest(threadpool.WorkRequest(fetchPage.downPage,args=[hostname,filename],kwds={},callback=callbackfunc)) fetchPage.dealwithResult(res,resource)
def run(self): """连续运行""" p = threadpool.ThreadPool(self.POOL_SIZE) while self.active: for url, callback in self.taskList: req = threadpool.WorkRequest(self._excutor, (url, callback)) p.putRequest(req) st = time() p.wait() elapse = time() - st need_sleep = max(self.taskInterval - elapse, 0) print '####', elapse, need_sleep sleep(need_sleep)
def _get_request(self, vm, action): if action == 'migrate': host = vm.get_host() dss = host.get_datastores() size = vm.get_size() vm_ds = None random.shuffle(dss) for ds in dss: if ds.get_freespace() > size: vm_ds = ds break if vm_ds is None: vm_ds = vm.get_datastores[0] return threadpool.WorkRequest(self.call_func, args=(vm, action, ( host, vm_ds, ))) randstr = utils.get_randstr(3) func_dict = { 'clone': (vm.name() + '_cln_' + randstr, ), 'snapshot': (vm.name() + '_snp_' + randstr, ), 'poweron': (), 'poweroff': (), 'reset': (), 'destroy': (), 'unregister': () } if action not in self.restore_list.keys(): self.restore_list[action] = [] if action == 'snapshot' or action == 'poweron' or action == 'poweroff': self.restore_list[action].append(vm) elif action == 'clone': self.restore_list[action].append(vm.name() + '_cln_' + randstr) return threadpool.WorkRequest(self.call_func, args=(vm, action, func_dict[action]))
def send_apn(self, token): token64 = base64.encodestring(token) udid = self.devices.get(token64) if udid is None: logMsg(_MODULE_ID, LOG_WARING, "mdmapn invalid token, [%s]" % token64) else: #this is user the worker to send # # apn_worker = self.choose_worker() # req = threadpool.WorkRequest(apn_worker.send_apn, (udid,)) # self.pool.putRequest(req) #THis is just use the function to send apn req = threadpool.WorkRequest(send_apn_worker, (udid, )) self.pool.putRequest(req)
def serve_forever(self, poll_interval=0.5): pool_size = cpu_count() * 2 + 1 self.pool = threadpool.ThreadPool(pool_size) self.pool.putRequest( threadpool.WorkRequest(self.serve_forever_thread, args=[poll_interval])) try: while True: time.sleep(0.001) self.pool.poll() except KeyboardInterrupt: global srvr srvr.shutdown() srvr.server_close() srvr = None finally: print("destory all threads before exit...") self.pool.dismissWorkers(pool_size, do_join=True)
def addTask(self, id, callable_, args): self.mutex.acquire() taskControler = None if id in self.taskControlerMap: taskControler = self.taskControlerMap[id] else: taskControler = TaskContoler() self.taskControlerMap[id] = taskControler reqId = time.clock() taskControler.addWaiter(reqId) taskControler.setOwner(reqId) reqArgs = [reqId, taskControler, callable_, args] req = threadpool.WorkRequest(_do_task, reqArgs, None, callback=_task_return, exc_callback=_handle_task_exception) self.pool.putRequest(req) self.mutex.release()
def DownloadImagePage(url): bs = helper.GetBs(url) #print(bs) if not bs: return False try: all_li = bs.select('ul[id="pins"]>li') pool = threadpool.ThreadPool(8) for li in all_li: if li.get('class') is None: #屏蔽广告 href = li.a.get('href') title = li.a.img.get('alt') #print(href,title) req = threadpool.WorkRequest(__task, [href]) pool.putRequest(req) pool.wait() print(url, "整页下载完成") return True except Exception as ex: print(ex) return False
def updateInstrument(): global errors errors={} pool=threadpool.ThreadPool(5) def callBack(req,out): if len(out)>0: errors[req.kwds['instrument']]=out errorlog=open('error_instrument.txt','w') for i in insts: print(i) req=threadpool.WorkRequest(od.update,kwds={'instrument':i},callback=callBack) pool.putRequest(req) pool.wait() errorlog.write(json.dumps(errors)) errorlog.close()
def usingThreadpool(limit,num_thread): urlset = open(config.SEED_FILE, "r") start = datetime.datetime.now() for url in urlset : try : if PAGESCache.get(url,None) == None : PAGESCache[url] = True else : continue hostname , filename = fetchPage.parse(url) req = threadpool.WorkRequest(fetchPage.downPage,args=[hostname,filename],kwds={},callback=callbackfunc) main.putRequest(req) except Exception: pass while True: try: main.poll() if config.total_url >= limit : break except threadpool.NoResultsPending: print "no pending results" break except Exception ,e: pass
def execute(self): log.debug("开始抓取夜不语小说...") load_engine = LoadEngine(__baseUrl__) chapters = YebuyuMuLuResolve(load_engine.execute()).execute() self.note.chapters = chapters has_exist = database.insert_note_if_not_exist(self.connection, self.note) if has_exist: chapters = self.find_error_and_update_chapters() requests = list() if chapters is not None and len(chapters) != 0: for chapter in chapters: requests.append( threadpool.WorkRequest(YebuyuEngine.__resolve_chapter, (self, chapter))) [g_pool.putRequest(req) for req in requests] g_pool.wait() log.debug("结束抓取夜不语小说" + self.note.simple_str()) return self.note
def vmotion(dc, vm_reg, host_name, ds_name, concurrency=10): host = dc.get_host_by_name(host_name) ds = dc.get_datastore_by_name(ds_name) if host is None: print 'Host {} not exist on DC.'.format(host_name) exit(1) if ds is None: print 'Datastore {} not exist on DC.'.format(ds_name) exit(1) vms = dc.get_vms_by_regex(utils.get_items(vm_reg)) vm_num = len(vms) import threadpool size = concurrency if concurrency < vm_num else vm_num pool = threadpool.ThreadPool(size) reqs = [ threadpool.WorkRequest(call_func, args=(vm, 'migrate', (host, ds))) for vm in vms ] for req in reqs: pool.putRequest(req) pool.wait() exit(0)