def CheckLinksInLibrary(self): import thread_pool import time self.t_link_delay = int(control.setting('library.service.testlink.delay') or 0) if self.t_link_delay == 0: printuj_linka('[CheckLink]: START and STOP - Do not check library sources') return dirs,files = control.listDir(self.library_folder) self.progressDialogBG = control.progressDialogBG self.progressDialogBG.create('Sprawdzam \xc5\xbar\xc3\xb3d\xc5\x82a biblioteki cda.pl','Ilo\xc5\x9b\xc4\x87 pozycji: %d'%len(dirs)) printuj_linka('[CheckLink]: START Library Folders = %d'%len(dirs)) if not control.condVisibility('Window.IsVisible(infodialog)') and not control.condVisibility('Player.HasVideo'): try: control.infoDialog('Sprawdzam \xc5\xbar\xc3\xb3dla biblioteki cda.pl', time=3) self.infoDialog = True except: self.infoDialog = False pool = thread_pool.ThreadPool(self.fredy) self.res={'checked':0,'skipped':0,'removed':0,'updated':0} N = len(dirs) for idx,folder_ in enumerate(dirs): pool.add_task(self.check_remove, *(folder_,idx,N)) time.sleep(0.1) pool.wait_completion() printuj_linka('[CheckLink]: END Status checked:%d, skipped:%d, removed:%d, updated:%d'%(self.res['checked'],self.res['skipped'],self.res['removed'],self.res['updated'])) self.progressDialogBG.close() if self.infoDialog == True: control.infoDialog('Usuni\xc4\x99to :%d, Zaktualizowano :%d '%(self.res['removed'],self.res['updated']), time=30)
def main(): ''' Temporary test script for the auto discovery module. ''' config = get_config() dbname = config['dbname'] getIpConfig.cleartables(dbname); # create the thread pool and the task_queue. pool = thread_pool.ThreadPool(config['num_threads']) # Now add tasks to the pool for all the ips in the iprange that we have ip_ranges = config['ip_ranges'] for rng in ip_ranges: if getIpConfig.compare_ips(rng[0],rng[1]) > 0: raise Exception("invalid range specified : " + rng[0] + " to " + rng[1]) work_ip = rng[0] while getIpConfig.compare_ips(work_ip,rng[1]) <= 0: pool.add_task(getIpConfig.process,work_ip,ip_ranges,dbname) work_ip = next_ip.next_ip(work_ip) pool.wait() getIpConfig.post_process(dbname) switch_dbs(dbname)
def __init__(self, meta_configs, stanza_configs, job_factory): """ @meta_configs: a dict like object, implement dict.get/[] like interfaces to get the value for a key. meta_configs shall at least contain {"server_uri": uri, "checkpoint_dir": dir, "session_key": key} key/value pairs @stanza_configs: a list like object containing a list of dict like object. Each element shall implement dict.get/[] like interfaces to get the value for a key. Each element in the list shall at least contain {"duration": an_duration_int} key/value pair @job_factory: an object which creates jobs. Shall implement create_job(stanza_config) interface and return an callable object """ import thread_pool as tp import job_scheduler as js import timer_queue as tq self.settings = self._read_default_settings() self.meta_configs = meta_configs self.event_queue = Queue.Queue() self.wakeup_queue = Queue.Queue() self._set_event_queue(stanza_configs) pool_size = self._get_pool_size(stanza_configs) self.io_pool = tp.ThreadPool(pool_size) self.cpu_pool = self._create_process_pool() self.scheduler = js.JobScheduler(job_factory, stanza_configs) self.event_reporter = threading.Thread(target=self._output_events) self.event_reporter.daemon = True self.timer_queue = tq.TimerQueue() self._started = False
def _TestThreadPool(self, threads): """Tests pool with specified threads from end to end.""" pool = thread_pool.ThreadPool(threads) self.actual_call_count = 0 expected_call_count = 10000 self.data = xrange(expected_call_count) self.actual_result = 0 expected_result = sum(self.data) stats_lock = threading.Lock() def _Dummy(num): stats_lock.acquire() self.actual_call_count += 1 self.actual_result += num stats_lock.release() for data in xrange(expected_call_count): pool.AddTask(_Dummy, data) pool.WaitCompletion() self.assertEqual(self.actual_call_count, expected_call_count) self.assertEqual(self.actual_result, expected_result) pool.Shutdown() for thread in pool.threads: self.assertFalse(thread.is_alive())
def __init__(self, url): self.url = url self.sess = requests.session() self.hh = hackhttp.hackhttp(hackhttp.httpconpool()) self.tp = thread_pool.ThreadPool(500) self.headers = headers_dict = { 'X-Forwarder-For': '192.168.1.1', }
def __init__(self, device): """ Constructor. @type device: Device @param device: the device which owns this thread """ Thread.__init__(self, name="Device Thread %d" % device.device_id) self.device = device self.number_subthreads = 8 self.subthreads = thread_pool.ThreadPool(self.number_subthreads)
def run(): db = db_pool.get_db_pool(True) # 初始化数据库连接池 event_list = get_event_list(db) # 获取等待处理的发行列表 pool = thread_pool.ThreadPool(10) # 十条进程同时运算 for item in event_list: print(item) pool.put(pool_action, (db, item[0], item[1],), callback) while True: time.sleep(0.5) if len(pool.generate_list) - len(pool.free_list) == 0: print("Task finished! Closing...") pool.close() break else: # print("{} Threads still working.Wait.".format(len(pool.generate_list) - len(pool.free_list))) pass
def TestThreadPoolExceptionHandler(self): """Tests thread pool with exceptions.""" self.exception_raised = False def _ExceptionHandler(e): """Verify an exception is raised and that it's the correct one.""" self.assertTrue(isinstance(e, TypeError)) self.assertEqual(e[0], 'gsutil') self.exception_raised = True pool = thread_pool.ThreadPool(1, exception_handler=_ExceptionHandler) def _Dummy(): raise TypeError('gsutil') pool.AddTask(_Dummy) pool.WaitCompletion() pool.Shutdown() self.assertTrue(self.exception_raised)
def init(self): """初始化根节点和crawler,parser,save类,\ 分别用于抓取,分析和保存 """ self.logger.info('init') try: self.logger.info("read config file:%s" % (self.conf)) self.config = ConfigParser.ConfigParser() self.config.read(self.conf) self.url_list_file = self.config.get("spider", "url_list_file") self.output_dir = self.config.get("spider", "output_directory") self.max_depth = self.config.getint("spider", "max_depth") self.crawl_interval = self.config.getint("spider", "crawl_interval") self.crawl_timeout = self.config.getint("spider", "crawl_timeout") self.target_url = self.config.get("spider", "target_url") self.thread_count = self.config.getint("spider", "thread_count") except ConfigParser.Error as e: self.logger.warn(e) return -1 #初始化url_queue,保存已经抓取的URL列表和去除重复 self.url_queue = self.init_url_queue() if self.url_queue == -1: self.logger.warn('init url queue fail,return...') return -1 #初始化抓取类 self.logger.info('init crawler.') page_crawler = webpage_crawler.WebpageCrawler(self.logger) #初始化解析类 self.logger.info('init parser.') page_parser = webpage_parser.WebpageParser(self.logger, self.max_depth) #初始化保存类 self.logger.info('init saver.') page_saver = webpage_saver.WebpageSaver(self.logger, self.target_url, self.output_dir) #初始化线程池 self.logger.info('init thread_pool.') self.thread_pools = thread_pool.ThreadPool(self.logger, self.thread_count, page_crawler,\ page_parser, page_saver, self.url_queue) return 0
def __init__(self, configs, job_scheduler, event_writer): """ @configs: a list like object containing a list of dict like object. Each element shall implement dict.get/[] like interfaces to get the value for a key. @job_scheduler: schedulering the jobs. shall implement get_ready_jobs @event_writer: write_events """ import thread_pool as tp import timer_queue as tq self.settings = self._read_default_settings() self.configs = configs self.event_writer = event_writer self.wakeup_queue = Queue.Queue() pool_size = self._get_pool_size() self.io_pool = tp.ThreadPool(pool_size) self.cpu_pool = None self.scheduler = job_scheduler self.timer_queue = tq.TimerQueue() self._started = False self._stopped = False
def run(): db_p = db_pool.get_db_pool(True) pool = thread_pool.ThreadPool(30) e_list = get_elist_to_be_done(db_p) for item in e_list: pool.put(get_history_check, ( db_p, item[0], item[1], ), callback) # get_history_line(db_p, item[0], item[1]) while True: time.sleep(2) if len(pool.generate_list) - len(pool.free_list) == 0: print("Task finished! Closing...") pool.close() break else: print("{} Threads still working.Wait.".format( len(pool.generate_list) - len(pool.free_list))) pass
def GetNewMovies(self): from cdapl import searchCDA import thread_pool import time self.check_setting = 'false' url = 'https://www.cda.pl/video/show/cale_filmy_or_caly_film_or_lektor_or_pl_or_dubbing_or_napisy_or_fps_or_odc/p%d?duration=dlugie§ion=&quality=720p§ion=&s=date§ion=' if not control.condVisibility('Window.IsVisible(infodialog)') and not control.condVisibility('Player.HasVideo'): try: control.infoDialog('Szukam nowych film\xc3\xb3w ...', time=3) self.infoDialog = True except: self.infoDialog = False self.progressDialogBG = control.progressDialogBG self.progressDialogBG.create('cda.pl','Szukam nowych film\xc3\xb3w ...') items=[] control.setSetting('library.service.last.run', datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) for stronkaX in range(int(control.setting('library.service.pages') or 1)): stronkaX +=1 printuj_linka('[AddMovie]: Searching cda.pl, page [%d]'%stronkaX) nowy_link,next=searchCDA(url%(stronkaX),False,False) items.extend(nowy_link) self.progressDialogBG.update(0,message='Znalaz\xc5\x82em pozycji [%d] '%(len(items))) self.progressDialogBG.update(0,message='Indentyfikuje %d film\xc3\xb3w w %d w\xc4\x85tkach ...'%(len(items),self.fredy)) items = [x for x in items if x.get('code','')!= ''] printuj_linka('[AddMovie]: Found Total %d videos ... '%(len(items))) pool = thread_pool.ThreadPool(self.fredy) self.out=[] N=len(items) for idx,one in enumerate(items): pool.add_task(self.dod_Movies, *(one,idx,N)) time.sleep(0.1) pool.wait_completion() printuj_linka('[AddMovie]: (After Threading) Found Total %d Movies ... '%(len(self.out))) self.out.reverse() self.progressDialogBG.close() self.add2( self.out ) control.setSetting('library.service.last.run', datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
# url to search news bing_news_base = "http://www.bing.com/news/search?q=" # url to get current trends bing_hint_request_url = "http://api.bing.com/qsonhs.aspx?" \ "FORM=ASAPIW&mkt=en-US&type=cb&cb=sa_inst.apiCB&cp=0&q=&o=l+a+hc+p" bing_hint_request_params = { "FORM": "ASAPIW", "mkt": "en-US", "type": "cb", "cb": "sa_inst.apiCB", "cp": "0", "q": "", "o": "l a hc p" } m = hashlib.md5() output_dir = "./Output/" keyword_queue = set() # store hashcode of title covered_keyword = set() # link_list = [] lock = threading.Lock() # worker_captain = worker.WorkerCaptain() pool = thread_pool.ThreadPool(max_crawler) regex = re.compile('[%s]+' % re.escape(string.punctuation))
import thread_pool class Student(): def run(self, *args, **kwargs): print(args) print('1234') if __name__ == '__main__': student = Student() #student.run([student]) pool = thread_pool.ThreadPool(3) requests = thread_pool.makeRequests(student.run, [[]]) pool.putRequest(requests[0]) pool.wait()
# if 0 != newpid: # self.poll_connects(listen_fd) for p in _processes: p.join() def poll_connects(self, listen_fd): try: epoll_fd = select.epoll() epoll_fd.register( listen_fd.fileno(), select.EPOLLIN | select.EPOLLET | select.EPOLLERR | select.EPOLLHUP) except select.error, e: print e, vars.backtrace() vars.logger.error(e) tp = thread_pool.ThreadPool(4) tp.start() params = {} def clear_fd(fd): try: _param = params[fd] epoll_fd.unregister(fd) _param["connection"].close() except Exception, e: print e, vars.backtrace() pass if fd in params: del params[fd] # try:
def tearDown(self): thread_pool.thread_pool = thread_pool.ThreadPool()
def __init__(self): self.work_queue = threadsafequeue.thread_safe_queue() self.thread_pool = thread_pool.ThreadPool(self.work_queue) self.socket = socket.socket()