def scan(self): if self.source: threadpool = ThreadPool.ThreadPool(1) else: threadpool = ThreadPool.ThreadPool(5) for rule in self.rules: threadpool.addtask(rule.detect, (), (self.results, rule)) #if rule.detect(): # self.results.append(rule) threadpool.start() threadpool.wait() #threadpool.clear() #threadpool.stop() pass
def unzip_zip_file_async(zipfilename, unziptodir, end_cb=None): if not os.path.exists(unziptodir): os.makedirs(unziptodir) def unzip(zipfilename, unziptodir, end_cb=None): zfobj = zipfile.ZipFile(zipfilename, 'r') for name in zfobj.namelist(): name = name.replace('\\', '/') if name.endswith('/'): os.makedirs(os.path.join(unziptodir, name)) else: ext_filename = os.path.join(unziptodir, name) ext_dir = os.path.dirname(ext_filename) if not os.path.exists(ext_dir): os.makedirs(ext_dir) data = zfobj.read(name) outfile = open(ext_filename, 'wb') outfile.write(data) outfile.close() if None != end_cb: end_cb(zipfilename, zfobj.namelist()) t = ThreadPool.ThreadPool().Thread(target=unzip, args=(zipfilename, unziptodir, end_cb)) t.start() return t
def get_weibo_comment(weibo_content_id): url = URL_COMMIT_URL.replace('id=4184562986557218', 'id=' + weibo_content_id) content = request_url(url, 'get', '') try: result_json = json.loads(content) ok_flag = result_json['ok'] test_comment_pool = ThreadPool(10) if ok_flag == 1: comment_data_arr = result_json['data']['data'] get_comment_by_arr(weibo_content_id, comment_data_arr) page_count = result_json['data']['max'] for page_index in range(2, page_count + 1): url = url.replace('&page=' + str(page_index - 1), '&page=' + str(page_index)) content = request_url(url, 'get', '') try: result_json = json.loads(content) ok_flag = result_json['ok'] if ok_flag == 1: comment_data_arr = result_json['data']['data'] #get_comment_by_arr(weibo_content_id,comment_data_arr) test_comment_pool(get_comment_by_arr, ( weibo_content_id, comment_data_arr, ), callback=None) except Exception as e: continue test_comment_pool.close() except Exception as e: pass
def preprocessPercentileRatios(self): print "preprocessPercentileRatios start" distributionsFile = self.getDatasetSlidingSizesFile() if os.path.isfile(distributionsFile): #Teh distributions file is processed print "The distribution file exists" return self.initializePropertiesComputeStructures(False) print "computing the ratios" try: zpa = zipfile.ZipFile(distributionsFile,"w",zipfile.ZIP_DEFLATED) zpa.writestr("dummy.txt","dummy file") zpa.close() self.ziplock = threading.Lock() # Create a pool with three worker threads pool = ThreadPool.ThreadPool(5) sys.setcheckinterval(1000) for windowSize in self.slidingWindowSizes: if self.useWindowThreading: pool.queueTask(self.preprocessWindowSize, windowSize, None) else: self.preprocessWindowSize(windowSize) # When all tasks are finished, allow the threads to terminate if self.useWindowThreading: print "Joining the threads" pool.joinAll() except: os.unlink(distributionsFile) raise print "preprocessPercentileRatios end"
def __init__(self, store_location): #logging.debug("OFFLINE IMAGE CACHE STARTUP") self._store_location = store_location self._threadpool = ThreadPool.ThreadPool(5, "OfflineImageCache", daemon=True) self._cachers = {} # dict of urlcacher objects?
def __init__(self, crawlername, workerThreadNum, pollInterval=0.5, pollTimeout=None, downloadTimeout=5): self.threadPool = ThreadPool(workerThreadNum) self.crawlername = crawlername self.pollTimeout = pollTimeout self.crawlerThread = CrawlerThread(self.threadPool, pollTimeout) self.mp3Downloader = MP3Downloader(downloadTimeout)
def process_jobs(num_threads, job_queue): """ Process the jobs in parallel """ # Create the thread pool tp = ThreadPool.ThreadPool(num_threads) # Process the jobs job_queue = tp.process_jobs(job_queue) for i in range(len(job_queue)): job = job_queue[i] print(job.thread.thread_id, job.start_time)
def siper_weibo_comment(weibo_link_dict): comment_pool = ThreadPool(10) print 'weibo comment siper start ...' for weibo_content_id in weibo_link_dict: created_at = weibo_link_dict[weibo_content_id] print weibo_content_id, created_at comment_pool.run(get_weibo_comment, (weibo_content_id, ), callback=None) # 解析微博评论 #get_weibo_comment(weibo_content_id) print 'weibo comment siper end ...' comment_pool.close()
def __init__(self): self.countPosition = 0 self.threadPool = ThreadPool.ThreadPool(1000) self.diffTime = 1114 self.deviationTime = 200 self.timer = QBasicTimer() self.__noteMap = collections.defaultdict(list) # <startTime, note[]> self.__player = None self.__musicPlayerView = None self.sheetRecorder = None self.scoreRecorder = None self.sheet = None
def __init__(self): self.fairness_factor = 0.2 self.fairness_scores = {} self.share_scores = {} self.dag_planners = {} self.alpha = 0.5 self.available_bandwidth = 1200 # 10Gbps = 1.2 GBps = 1200 MBps self.cache_block_size = 1 # 1MBype self.cache = cache.Cache(10000) self.n_thp = 10 self.d3n_conn = thp.ThreadPool(self.n_thp)
def siper_data(user_id, start_date, end_date): main_pool = ThreadPool(5) start_date = datetime.strptime(start_date, '%Y-%m-%d') end_date = datetime.strptime(end_date, '%Y-%m-%d') # 采集微博用户信息 get_user_info(user_id) # 读取微博ID container_id = get_containerid(user_id) # 采集微博LINK时间以及ID weibo_link_dict = get_weibo_link(user_id, container_id, start_date, end_date) main_pool.run(siper_weibo_content, (weibo_link_dict, ), callback=None) main_pool.run(siper_weibo_comment, (weibo_link_dict, ), callback=None)
def siper_weibo_content(weibo_link_dict): content_pool = ThreadPool(10) print 'weibo content siper start ....' # 采集微博内容信息 for weibo_content_id in weibo_link_dict: created_at = weibo_link_dict[weibo_content_id] content_pool.run(get_weibo_content, (user_id, weibo_content_id, created_at), callback=None) # 采集微博内容信息 #get_weibo_content(user_id,weibo_content_id,created_at) print 'weibo content siper end ....' content_pool.close()
def __init__(self, sizeOfBuffer, numberOfCores, numberOfThreads, timeQuantum, contextSwitchTime): Buffer.Buffer.initBufferCount() self.buffer = Buffer.Buffer(sizeOfBuffer) self.numberOfCores = numberOfCores self.cores = [] for y in list(range(numberOfCores)): self.cores.append(Core.Core(y, 0)) self.threadPool = ThreadPool.ThreadPool(numberOfThreads) self.timeQuantum = timeQuantum self.contextSwitchTime = contextSwitchTime
def siper_weibo_comment(weibo_link_dict, user_id): comment_pool = ThreadPool(MAX_THREAD) print 'weibo comment siper start ...' for weibo_content_id in weibo_link_dict: created_at = weibo_link_dict[weibo_content_id] print weibo_content_id, created_at #get_weibo_comment_spider(weibo_content_id,user_id) comment_pool.run(get_weibo_comment_spider, ( weibo_content_id, user_id, ), callback=None) # 解析微博评论 #get_weibo_comment(weibo_content_id) print 'weibo comment siper end ...' comment_pool.close()
def __init__(self, parent): super().__init__(parent) self.fixedSprites = list( range(0, 8) ) # eight fixed top arrow, each arrow should be put in the number of the arrow self.arrowSprites = [] self.levelFeedbackSprites = [ None ] * 2 # level feedback of two players, [0] -> p1, [1] -> p2 self.setFocusPolicy(Qt.StrongFocus) self.threadPool = ThreadPool(20) self.spritePrototypeFactory = SpritePrototypeFactory() self.spritePrototypeFactory.prepare(MyGameView.GAME_WIDTH, MyGameView.GAME_HEIGHT) self.createAllTopFixedArrow() self.scoreRecordersMap = defaultdict(ScoreRecorder)
def __init__(self, view, home, share_path): PTVhtml.PTVhtml.__init__(self, view, home, share_path) self._htmlview = None self._document_lock = threading.Lock() self._image_cache = SimpleImageCache.SimpleImageCache() self._css = "" self._last_link_time = 0 self._view = view f = open(os.path.join(share_path, "gtkhtml.css")) for l in f.readlines(): self._css += l f.close() self._image_pool = ThreadPool.ThreadPool(5, "PlanetView") self._dl_total = 0 self._dl_count = 0
def __init__(self, app, media_dir, progress_callback=None, finished_callback=None): self.index = 0 #should this be lucene compatible? if utils.RUNNING_HILDON: max_downloads = 1 else: max_downloads = 5 self._style = BYDATE self.pool = ThreadPool.ThreadPool(max_downloads, "MediaManager") self.downloads = [] self.db = app.db self.time_appendix = 0 self.bt_settings = {'min_port': 6881, 'max_port': 6999, 'ul_limit': 0} self.id_time = 0 self.quitting = False self._net_connected = True self.pause_state = RUNNING if finished_callback: self.app_callback_finished = finished_callback else: self.app_callback_finished = self._basic_finished_callback if progress_callback: self.app_callback_progress = progress_callback else: self.app_callback_progress = self._basic_progress_callback home = self.db.home if media_dir[0] == '~': media_dir = os.getenv('HOME') + media_dir[1:] try: os.stat(media_dir) except: try: os.mkdir(media_dir) except: raise NoDir, "error creating " + media_dir self._media_dir = media_dir app.connect('online-status-changed', self.__online_status_changed) app.connect('new-database', self.__new_database_cb)
def getRemoteFileByUrlAsync(url, path, progress_cb=None, end_cb=None): file_size = 0 try: file_size = int(urlopen(url).info().get('Content-Length', -1)) except Exception as e: print(e) return None def getRemoteFunc(url, path, file_size, progress_cb=None, end_cb=None): try: if not os.path.exists(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) pbar = tqdm(total=file_size, initial=0, desc=path, unit='B', unit_scale=True) r = requests.get(url, stream=True, verify=False) curSize = 0.0 chunksize = 1024 * 1024 * 4 with open(path, "wb") as f: for chunk in r.iter_content(chunk_size=chunksize): f.write(chunk) curSize = curSize + len(chunk) if None != progress_cb: progress_cb(curSize, file_size) pbar.update(chunksize) f.close() pbar.close() if None != end_cb: end_cb(path, True) except Exception as e: print(e) if None != end_cb: end_cb(path, False) t = ThreadPool.ThreadPool().Thread(target=getRemoteFunc, args=(url, path, file_size, progress_cb, end_cb)) t.start() return t
def unzip_7z_file_async(zipfilename, unziptodir, end_cb=None): if not os.path.exists(unziptodir): os.makedirs(unziptodir) def unzip(zipfilename, unziptodir, end_cb=None): try: archive = py7zr.SevenZipFile(zipfilename, mode='r') names = archive.getnames() archive.extractall(path=unziptodir) archive.close() if None != end_cb: end_cb(zipfilename, names) except Exception as e: print(e) t = ThreadPool.ThreadPool().Thread(target=unzip, args=(zipfilename, unziptodir, end_cb)) t.start() return t
def spider_page(url, fid): content = request_url(url, 'get', '') soup = BeautifulSoup(content, 'lxml') tbody_list = soup.find('table', id='threadlisttableid').find_all('tbody') main_pool = ThreadPool(MAX_THREAD) for tbody in tbody_list: id = tbody.get('id') id = id.replace('separatorline', '').replace('stickthread', '').replace('normalthread', '').replace('_', '') if len(id) > 0 and id: content_url = 'https://bbs.ichunqiu.com/thread-' + str( id) + '-1-1.html' #ichunqiu_sipder(content_url,fid) main_pool.run(ichunqiu_sipder, ( content_url, fid, ), callback=None) main_pool.close()
def get_weibo_comment_spider(content_id, user_id): url = 'https://weibo.com/aj/v6/comment/big?id=' + content_id + '&filter=all&page=1' try: page_count = get_weibo_comment_ext(url, content_id, user_id) comment_pool = ThreadPool(MAX_THREAD) for i in range(2, page_count + 1): url = 'https://weibo.com/aj/v6/comment/big?id=' + content_id + '&filter=all&page=' + str( i) try: #get_weibo_comment_ext(url,content_id,user_id) comment_pool.run(get_weibo_comment_ext, (url, content_id, user_id), callback=None) except Exception as e1: print e1 traceback.print_exc() continue comment_pool.close() except Exception as e: traceback.print_exc()
def test(): import random import time import datetime def do_work(data): time.sleep(random.randint(1, 3)) res = str(datetime.datetime.now()) + "" + str(data) return res def print_result(request, result): print "---Result from request %s : %r" % (request.requestID, result) main = ThreadPool(3) for i in range(40): req = WorkRequest(do_work, args=[i], kwds={}, callback=print_result) main.putRequest(req) print "work request #%s added." % req.requestID print '-' * 20, main.workersize(), '-' * 20 counter = 0 while True: try: time.sleep(0.5) main.poll() if (counter == 5): print "Add 3 more workers threads" main.createWorkers(3) print '-' * 20, main.workersize(), '-' * 20 if (counter == 10): print "dismiss 2 workers threads" main.dismissWorkers(2) print '-' * 20, main.workersize(), '-' * 20 counter += 1 except NoResultsPending: print "no pending results" break main.stop() print "Stop"
def __init__(self,filepath): with open(filepath,'r',encoding='utf-8') as f:#解析配置文件 self.config=json.loads(f.read()) self.max_Task=self.config['max_Task']#最大任务数 self.thread_count=self.config['threadCount']#线程数 self.M_BLL=Mission_BLL.Mission()#任务表 self.SI_BLL=SpiderInfo_BLL.SpiderInfo()#配置表 self.HI_BLL=HoneyInfo_BLL.HoneyInfo()#数据库信息 self.EI_BLL=ExceptionInfo_BLL.ExceptionInfo()#异常信息 self.Exception_Model=Model.exceptionInfo("","")#异常模型 self.PRI=["high","common","low"]#优先级 self.T_Pool = ThreadPool(self.thread_count,self.max_Task)#任务线程池 #日志信息 self.loger=logging.getLogger() self.loger.setLevel(logging.DEBUG) hfile=logging.FileHandler("./Hive/LOG/ServerLogInfo.log") formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') hfile.setFormatter(formatter) self.loger.addHandler(hfile) self.loger.info("Server初始化!")
def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True, thread_num=10, RequestInterceptorClasses=[]): """Constructor. thread_num 线程池默认的线程数量 """ self.requestInterceptorClasses = RequestInterceptorClasses #初始化线程池 self.threadPool = ThreadPool.ThreadPool(thread_num) #设置urllib opener, HTTP server接收到请求时,需要通过urllib 转发出请求,这里全局设置urllib cj = http.cookiejar.CookieJar() #proxyHandler = urllib.request.ProxyHandler({'http','127.0.0.1:8888'}) 代理 opener = urllib.request.build_opener( urllib.request.HTTPCookieProcessor(cj), KIZHTTPRedirectHandler, KIZHttpErrorHandler) urllib.request.install_opener(opener) super().__init__(server_address, RequestHandlerClass, bind_and_activate)
Cek.close() Cekpoint.append(user+pass7) Except: Pass P = ThreadPool(30) P.map(main, id) Print "\033[1;95m♡──────────•◈•──────────♡\033[1;96mBlackMafia\033[1;95m♡──────────•◈•──────────♡" Print " \033[1;93m«---•◈•---Developed By love---•◈•---»" #Dev:love_hacker Print '\033[1;91mProcess Has Been Completed\033[1;92m....' Print"\033[1;91mTotal OK/\x1b[1;93mCP \033[1;91m: \033[1;91m"+str(len(oks))+"\033[1;97m/\033[1;95m"+str(len(cekpoint)) Print """
import tkinter as tk import tkinter.messagebox import socket import time import threading import queue import sys from Encryption import encryptPasswd, encodeId, decodeId, pad, readMessage, readRoomList, readUserList from Settings import HOST, PORT, COMMAND_CODE import ThreadPool client_thread_pool = ThreadPool.ThreadPool(3, True) MessageQueue = queue.Queue() RoomsList = list() # True for Rooms List HAVE BEEN Read, False for not RoomsListFlag = True UsersList = list() # True for Users List HAVE BEEN Read, False for not UsersListFlag = True ReturnCode = 0 # True for message HAVE BEEN Read, False for HAVE NOT BEEN Read ReturnCodeFlag = True ChatRooms = dict() def clientReceiveLogic(conn): global RoomsListFlag, RoomsList, ReturnCode, ReturnCodeFlag, loginPage conn = conn[0] while True: data = MessageQueue.get() Command = int.from_bytes(data[0:2], byteorder='big')
#python computeFVs.py videos vid_in vid_out if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("vid_path", help="Directory of the input videos", type=str) parser.add_argument("vid_in", help="list of input videos in .txt file", type=str) parser.add_argument("output_dir", help="output directory to save FVs (.fisher files)", type=str) parser.add_argument("gmm_list", help="File of saved list of GMMs", type=str) args = parser.parse_args() f = open(args.vid_in, 'r') input_videos = f.readlines() f.close() input_videos = [line.split()[0].split('/')[-1] for line in [video.rstrip() for video in input_videos]] ###Just to prevent overwriting already processed vids completed_vids = [filename.split('.')[0] for filename in os.listdir(args.output_dir) if filename.endswith('.npz')] overlap = [vid for vid in input_videos if vid.split('.')[0] in completed_vids] #Multi-threaded FV construction. numThreads = 10 pool = ThreadPool.ThreadPool(numThreads) for vid in input_videos: if vid not in overlap: pool.add_task(processVideo,vid,args.vid_path,args.output_dir,args.gmm_list) pool.wait_completion()
return asyncDecorator if __name__ == "__main__": from time import sleep from ThreadPool import * try: xrange except NameError: xrange = range class TestClass(): @Async() def testDecorated(self): print (345) testClass = TestClass() testClass.testDecorated() @Async(executor=ThreadPool(5)) def func(a, b): print ("func called") sleep(1) print ("func exit:" + str(a)) @Async() def funcWithoutExecutor(a): print (a) for x in xrange(1, 10): funcWithoutExecutor("noExecutor:" + str(x)) for x in xrange(1, 15): func(x, 2)
author = string_tag.find('a',class_='xi2').string break for date_div in soup.find_all('div',class_='cl',attrs={'style':'font-size: 12px; color: #888888;'}): tmp_str = str(date_div) date_arr = re.findall('\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}',tmp_str) if len(date_arr) > 0: content_date = date_arr[0] break content_id = MySQLdb.escape_string(content_id) title = MySQLdb.escape_string(title) author = MySQLdb.escape_string(author) content_date = MySQLdb.escape_string(content_date) connection = pool.connection() cursor = connection.cursor() sql = "INSERT IGNORE INTO ichunqiu_content(id,content_id,title,url,author,content_date,create_date,update_date) VALUES (DEFAULT,'"+content_id+"','"+title+"','"+url+"','"+author+"','"+content_date+"',NOW(),NOW())" #print sql cursor.execute(sql) connection.commit() cursor.close() connection.close() print 'content_id -->',content_id,' ok' if __name__ == '__main__': main_pool = ThreadPool(MAX_THREAD) for i in range(33742,34832+1): url = 'https://bbs.ichunqiu.com/thread-'+str(i)+'-1-1.html' #ichunqiu_sipder(url) main_pool.run(ichunqiu_sipder,(url,), callback=None) #break # print url main_pool.close()
from interface import implements from Task import * from ThreadPool import * from threading import Lock class ExampleTask(implements(Task)): def __init__(self, num, print_lock): self.num = num self.print_lock = print_lock def executeTask(self): self.print_lock.acquire() print("Executing task # " + str(self.num)) self.print_lock.release() if __name__ == "__main__": pool = ThreadPool(5) pool.initialize() print_lock = Lock() for i in range(50): task = ExampleTask(i, print_lock) pool.submit_task(task) pool.stop_threads()