def run(self): socket_server = self.server_socket socket_server.listen() task_pool = ThreadPool() while True: client_socket, _ = socket_server.accept() task_pool.add_task((client_socket, self.job))
def from_file(m163,option): """ download objects (songs, albums...) from an input file. """ urls = [] with open(option.inFile) as f: urls = f.readlines() global total, done, xiami_obj total = len(urls) print border LOG.info(msgTxt.fmt_links_in_file % total) print border pool = ThreadPool(config.THREAD_POOL_SIZE) for link in [u for u in urls if u]: link = link.rstrip('\n') #if it is a xiami link, init xiami object if re.match(pat_xm, link): __init_xiami_obj(option) pool.add_task(from_url_xm, xiami_obj,link, verbose=False) elif re.match(pat_163, link): pool.add_task(from_url_163, m163,link, verbose=False) else: LOG.warning(msgTxt.fmt_skip_unknown_url % link) pool.wait_completion()
def pickle_all_companies(): tpool = ThreadPool(50) companies = Company.objects.all() for c in companies: tpool.add_task(pickle_company, c.symbol) tpool.wait_completion() return None
def main(latest_block: int, url: str, threads: int): # Create a folder to store the invalids folder = f"./invalids/{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}" os.mkdir(folder) iterations = latest_block + 1 for iteration in tqdm(range(1, iterations, threads)): pool = ThreadPool(threads) for block_number in range(iteration, iteration + threads): pool.add_task(worker, url, block_number, folder) pool.wait_completion()
def main(): start_time = time.time() parser = add_parser() args = parser.parse_args() app_dir = args.path + "\\" + args.app_dir global encoding encoding = args.encoding #line feed line_feed = "\n" # use console to show information global console console = Console() console.show("Target Path: " + args.path) console.show("Webapp Directory: " + app_dir) console.show("Testing Website: " + args.website) console.show("Output File: " + args.output) # start fetching console.show("Start fetching url and its parameters in " + args.path) global url_data url_data = UrlData() get_url_list(args.path, app_dir, args.website) url_amount = url_data.len() #fetch complete console.show("Fetched " + str(url_amount) + " url(s).") if args.get_status != 1 or args.website == "": url_data.export(args.output) #exit sys.exit() console.show("Start testing url status with " \ + str(args.thread_num) + " thread(s).") #init thread pool pool = ThreadPool(args.thread_num) for url in url_data.get_urls(): pool.add_task(url_request, url) console.show_progress(pool.get_progress(), url_amount) while pool.get_progress() != url_amount: console.show_progress(pool.get_progress(), url_amount) #pool.destroy() finish_time = time.time() elapsed_time = int(finish_time - start_time) #export url_data.export(args.output) console.show("Task finished in " + str(elapsed_time) + " seconds.")
def prime_cache(self): """Ensures that the webpage cache is filled in the quickest time possible by making many requests in parallel""" print "Getting data for parts from suppliers' websites" pool = ThreadPool(NUM_THREADS) for srcode, pg in self.iteritems(): print srcode pool.add_task(pg.get_price) pool.wait_completion()
def download_all(links, save_folder): if not os.path.exists(save_folder): os.makedirs(save_folder) if USING_THREAD: pool = ThreadPool(NUMBER_OF_THREADS) for link in links: with _thread_print_lock: logger.debug("download from link: \n %s \n" % link) pool.add_task(download_file, link, save_folder) #pool.add_task(test_func) time.sleep(5) pool.wait_completion() else: for link in links: logger.debug("download from link: \n %s \n" % link) download_file(link, save_folder)
def job(): """ Updater job for periodic repetitions. """ print('[+] PUT UPDATER INTO THREADPOOL AT [%s]' % (now())) pool = ThreadPool(POOL_COUNT) pool.add_task(populate, None) pool.wait_completion() print('[+] COMPLETE POPULATE AT [%s]' % (now())) pool.add_task(update, None) pool.wait_completion() print('[+] COMPLETE UPDATE AT [%s]' % (now())) del pool cache.rpush( 'incomingQueue', 'StartUpdateVulnerabilityDataBase')
def from_file(xm_obj, infile): """ download objects (songs, albums...) from an input file. """ urls = [] with open(infile) as f: urls = f.readlines() global total, done total = len(urls) print border LOG.info(u' 文件包含链接总数: %d' % total) print border pool = ThreadPool(config.THREAD_POOL_SIZE) for link in [u for u in urls if u]: pool.add_task(from_url, xm_obj,link.rstrip('\n'), verbose=False) pool.wait_completion()
def from_file(xm_obj, infile): """ download objects (songs, albums...) from an input file. """ urls = [] with open(infile) as f: urls = f.readlines() global total, done total = len(urls) print border LOG.info(u' 文件包含链接总数: %d' % total) print border pool = ThreadPool(config.THREAD_POOL_SIZE) for link in [u for u in urls if u]: pool.add_task(from_url, xm_obj, link.rstrip('\n'), verbose=False) pool.wait_completion()
class Spider(object): def __init__(self,seed,depth,pool_size=10): self.seed = seed self.depth = depth self.all_url_list = [seed] self.finished_url_list = [] self.failure_url_list = [] self.pool = ThreadPool(pool_size) def crawl(self): base_deep_size = 0 while base_deep_size <= self.depth: for url in self.all_url_list: if url not in self.finished_url_list: self.pool.add_task(self.download,url) self.pool.close() self.depth-=1 def download(self,url): try: data = urllib2.urlopen(url) page = data.read() self.finished_url_list.append(url) links = self.get_urls(page) return page,links except Exception as e: print 'open url:%s raise exception(%s)'%(url,e) return None def get_urls(self,page): soup = BeautifulSoup(page,fromEncoding="gb18030") if soup.title: print soup.title.string links = [] for item in soup.findAll('a'): link=item.get('href') if link and link.startswith('http://') and link not in self.finished_url_list: links.append(link) print links return links def get_next_url(self): pass
def run(): logging.info('start subscribe server.....') ##创建抓取网页的线程池 grab_pool = ThreadPool(GRAB_NUM) for i in range(GRAB_NUM): grab_pool.add_task(do_grab,None,id = i+1) ##创建解析网页的线程池 paser_pool = ThreadPool(PASER_NUM) for i in range(PASER_NUM): paser_pool.add_task(do_paser,None,id = i+1) ##创建发送邮件的线程池 send_pool = ThreadPool(MAIL_SENDER_NUM) for i in range(MAIL_SENDER_NUM): send_pool.add_task(do_send,None,id = i+1) # Join and destroy all threads grab_pool.destroy() paser_pool.destroy() send_pool.destroy()
def from_file(xm_obj,m163, infile): """ download objects (songs, albums...) from an input file. """ urls = [] with open(infile) as f: urls = f.readlines() global total, done total = len(urls) print border LOG.info(u' 文件包含链接总数: %d' % total) print border pool = ThreadPool(config.THREAD_POOL_SIZE) for link in [u for u in urls if u]: link = link.rstrip('\n') if re.match(pat_xm, link): pool.add_task(from_url_xm, xm_obj,link, verbose=False) elif re.match(pat_163, link): pool.add_task(from_url_163, m163,link, verbose=False) else: LOG.warning(u' 略过不能识别的url [%s].' % link) pool.wait_completion()
def make_all_pairs(use_celery=False, skip_update=False, skip_pickle=False, skip_worker_update=False): """ This will check all of the pairs, either threaded or via celery (i.e. local v cloud) """ logger.info(colored('Collecting companies', 'white', attrs=['bold'])) companies = Company.objects.all() tpool = ThreadPool(50) if not skip_update: logger.info(colored('Updating prices', 'white', attrs=['bold'])) for c in companies: tpool.add_task(c.update_prices) tpool.wait_completion() logger.info(colored('Prices updated', 'white', attrs=['bold'])) symbols = [c.symbol for c in companies] if not skip_pickle: logger.info(colored('Pickling companies', 'white', attrs=['bold'])) pickle_all_companies() if not skip_worker_update: logger.info(colored('Updating workers', 'white', attrs=['bold'])) update_workers() if use_celery: for s1, s2 in itertools.combinations(symbols, 2): make_pair.delay(s1, s2) else: for s1, s2 in itertools.combinations(symbols, 2): tpool.add_task(make_pair, s1, s2) tpool.wait_completion() return
def run(self, symnames, init_symbols=None, N=4): if isinstance(symnames, (str, unicode)): symnames = [ symnames ] self.finalize() toporder = self._topsort_subgraph(symnames) sources = ( v for v in toporder if len(self.depends[v]) == 0 ) remain_to_submit = SynchronizedCounter(len(toporder)) finished_deps = defaultdict(SynchronizedCounter) p = ThreadPool(N) if init_symbols is None: syms = SymbolTable() else: syms = init_symbols parentlock = RLock() done_submitting = Condition() # If the child thread notifies before the parent thread reaches the # wait statement, then the parent will never receive the notification # and will block forever. To fix this, the child will decrement this # counter to zero, and the parent will check this before waiting. done_submitting_helper = SynchronizedCounter(1) # The callback runs within the thread. Don't know how to fix. def make_apply_callback(gf): def finished(new_syms): parentlock.acquire() self.results[gf] = new_syms parentlock.release() parentlock.acquire() # print "%s finished! printing state"%(gf.name) # print "finished_deps", finished_deps # print >> sys.stderr, "%s completed. new_syms = %s"%(gf.name, new_syms) # print "self.depends", self.depends parentlock.release() # Update the functions which we precede for next_gf in self.preceded_by[gf]: finished_deps_next_gf = finished_deps[next_gf].inc() if finished_deps_next_gf == len(self.depends[next_gf]): # All dependencies satisfied; we can run! # This may take a bit of time, but we want to do # all data manipulation in this process. print >> sys.stderr, "Dependencies for %s satisfied. Queueing."%next_gf.name symtable = SymbolTable(parents=[self.results[r] for r in self.depends[next_gf]]) # Queue doesn't need to be locked p.add_task(next_gf, args=(symtable,), callback=make_apply_callback(next_gf)) if remain_to_submit.dec() == 0: print >> sys.stderr, "All jobs have been submitted. Waiting for parent thread to be ready to receive done_submitting" done_submitting.acquire() done_submitting.notify() done_submitting.release() done_submitting_helper.dec() return finished for s in sources: remain_to_submit.dec() p.add_task(s, args=(SymbolTable(),), callback=make_apply_callback(s)) if done_submitting_helper.get() > 0: done_submitting.acquire() print >> sys.stderr, "PARENT THREAD: Awaiting condition variable" done_submitting.wait() done_submitting.release() print >> sys.stderr, "PARENT THREAD: Joining the thread pool" p.wait_completion() ret = dict((sym, self.results[self.supplier[sym]][sym]) for sym in symnames) return ret
class Database: """Asynchronous database interface. The `driver' argument specifies which database to use. Possible values are: MySQLdb - for MySQL psycopg2 - for Postgres """ def __init__(self, driver=None, database=None, user=None, password=None, host='localhost', ioloop=tornado.ioloop.IOLoop.instance(), num_threads=10, tx_connection_pool_size=5, queue_timeout=1): if not (driver): raise ValueError("Missing 'driver' argument") self._driver = driver self._database = database self._user = user self._password = password self._host = host self._threadpool = ThreadPool( per_thread_init_func=self.create_connection, per_thread_close_func=self.close_connection, num_threads=num_threads, queue_timeout=queue_timeout) self._ioloop = ioloop # Connection pool for transactions self._connection_pool = [] for i in xrange(tx_connection_pool_size): conn = self.create_connection() self._connection_pool.append(conn) self._waiting_on_connection = deque() def create_connection(self): """This method is executed in a worker thread. Initializes the per-thread state. In this case we create one database connection per-thread. """ if self._driver == "psycopg2": try: import psycopg2 conn = psycopg2.connect(database=self._database, user=self._user, password=self._password, host=self._host) except Exception as ex: raise ex elif self._driver == "MySQLdb": try: import MySQLdb conn = MySQLdb.connect(db=self._database, user=self._user, passwd=self._password, host=self._host, port=3306) except Exception as ex: raise ex else: raise ValueError("Unknown driver %s" % self._driver) return conn def close_connection(self, conn): conn.close() def stop(self): self._threadpool.stop() for conn in self._connection_pool: conn.close() @async def beginTransaction(self, callback): """Begins a transaction. Picks up a transaction from the pool and passes it to the callback. If none is available, adds the callback to `_waiting_on_connection'. """ if self._connection_pool: conn = self._connection_pool.pop() callback(conn) else: self._waiting_on_connection.append(callback) @async def commitTransaction(self, connection, callback): self._threadpool.add_task( partial(self._commitTransaction, connection, callback)) def _commitTransaction(self, conn, callback, thread_state=None): """Invoked in a worker thread. """ conn.commit() self._ioloop.add_callback( partial(self._releaseConnectionInvokeCallback, conn, callback)) @async def rollbackTransaction(self, connection, callback): self._threadpool.add_task( partial(self._rollbackTransaction, connection, callback)) def _rollbackTransaction(self, conn, callback, thread_state=None): """Invoked in a worker thread. """ conn.rollback() self._ioloop.add_callback( partial(self._releaseConnectionInvokeCallback, conn, callback)) def _releaseConnectionInvokeCallback(self, conn, callback): """Release the connection back in the connection pool and invoke the callback. Invokes any waiting callbacks before releasing the connection into the pool. """ # First invoke the callback to let the program know we're done # with the transaction. callback(conn) # Now check to see if we have any pending clients. If so pass # them the newly released connection. if self._waiting_on_connection: callback = self._waiting_on_connection.popleft() callback(conn) else: self._connection_pool.append(conn) @async def runQuery(self, query, args=None, conn=None, callback=None): """Send a SELECT query to the database. The callback is invoked with all the rows in the result. """ self._threadpool.add_task(partial(self._query, query, args, conn), callback) def _query(self, query, args, conn=None, thread_state=None): """This method is called in a worker thread. Execute the query and return the result so it can be passed as argument to the callback. """ if not conn: conn = thread_state cursor = conn.cursor() cursor.execute(query, args) rows = cursor.fetchall() cursor.close() return rows @async def runOperation(self, stmt, args=None, conn=None, callback=None): """Execute a SQL statement other than a SELECT. The statement is committed immediately. The number of rows affected by the statement is passed as argument to the callback. """ self._threadpool.add_task(partial(self._execute, stmt, args, conn), callback) def _execute(self, stmt, args, conn=None, thread_state=None): """This method is called in a worker thread. Executes the statement. """ # Check if stmt is a tuple. This can happen when we use map() # with adisp to execute multiple statements in parallel. if isinstance(stmt, tuple): args = stmt[1] stmt = stmt[0] if not conn: conn = thread_state should_commit = True else: should_commit = False cursor = conn.cursor() cursor.execute(stmt, args) if should_commit: conn.commit() rowcount = cursor.rowcount cursor.close() return rowcount
def functional_test(): r = redis.StrictRedis(host='localhost', port=8323) # set/get print r.set("key", 'b'*56000) print len(r.get("key")) # incr print r.set("incr_key",0) print r.get("incr_key") print r.incr('incr_key') print r.get("incr_key") def press_test(): r = redis.StrictRedis(host='localhost', port=8323) for i in range(10000): key = 'foo_%d'%i r.set(key, 'b'*i) if i%1000==0: print key, "->", len(r.get(key)) if __name__=="__main__": #functional_test() # Create thread pool with nums threads pool = ThreadPool(32) # Add a task into pool for n in range(10): pool.add_task(functional_test) pool.add_task(press_test) # Join and destroy all threads pool.destroy()
def start(baseUrl,seedUrl): # clean reffer in reffer.txt f = open("reffer.txt","w") f.close() #seed = Request(base='http://192.168.42.131/dvwa/index.php',url='http://192.168.42.131/dvwa/index.php',method='get') seed = request.Request(base=baseUrl,url=seedUrl,timeout=config.conf['connTimeout'],query={},method='get') #seed = request.Request(base='http://192.168.42.132/dvwa/',url='http://192.168.42.132/dvwa/',query={},method='get') colors.blue( '种子URL: %s\n'%seed._url) logfileName = create_logfile(seed._url) cookie = getCookie(seed._url) # begin crawler tup = urlparse.urlparse(seed._url) netloc = tup.netloc # seed url count = 0 q = Queue.Queue() bf = bloomFilter.BloomFilter(0.001,100000) # readreffer from reffer.txt ''' reffer = readReffer() reqSet = [] reqSet.append(seed) reqSet.extend(reffer) for i in reqSet: q.put(i) bf.insert(i._url) ''' q.put(seed) bf.insert(seed._url) nums = config.conf['MaxThread'] pool = ThreadPool(nums) begin = time.time() while(not q.empty()): req = q.get() req._cookies = cookie reqs = crawler.crawl(req,tree) if req._query != {} and is_tree_full(req._url,tree): #if req._query != {}: count += 1 print 'URL: ',req._BFUrl,' ', req._source pool.add_task(startCheck,req,logfileName) for x in reqs: if not bf.exist(x._BFUrl): bf.insert(x._BFUrl) q.put(x) pool.destroy() end = time.time() f = open(logfileName,'r') colors.blue('\n扫描结果:\n\n') x = f.read() colors.green(x) colors.blue('\n扫描结果已保存在 "%s"\n\n'%(os.getcwd()+'/'+logfileName)+' 中') cost = end - begin print "耗时:%f秒"%cost print "进行测试的URL数量:",count f.close() f = open(logfileName,'a') f.write(advice()) f.close() os.system('ps -ef | grep -v grep | grep proxy.py | awk \'{print $2}\'|xargs kill -9') '''
def lagouScrapy(self): tp = ThreadPool(18) for i in range(200): tp.add_task(self.lagou, i) i += 1 tp.destroy()
class Tasks(Codes): def __init__(self): self.operate = Operate() self._api = OpenApi() self._http = HttpClient.getInstance() self._pool = ThreadPool(5) # 初始化5个线程 print("Task Class 初始化完毕") def getAllAdmin(self): print("所有管理员: %s", variable.Admins) return variable.Admins def getAllGroup(self): print("所有关注群: %s", variable.Groups) return variable.Groups def addAdmin(self, qq): return self.operate.addAdmin(qq) def delAdmin(self, qq): return self.operate.delAdmin(qq) def isAdmin(self, qq): return self.operate.isAdmin(qq) def addGroup(self, qq): return self.operate.addGroup(qq) def delGroup(self, qq): return self.operate.delGroup(qq) def inGroup(self, qq): # print("inGroup: %s", qq) return self.operate.inGroup(qq) def addAsk(self, question, answer): return self.operate.addAsk(question, answer) def delAsk(self, Id): return self.operate.delAsk(Id) def getAsk(self, content): return self.operate.getAsk(content) def end(self): self._pool.destroy() def uin_to_qq(self, uin): if uin in variable.UsersQQ: return variable.UsersQQ.get(uin) print("获取qq %s %s %s", uin, variable.Vfwebqq, variable.Referer) html = self._http.get(variable.Get_friend_uin2.format(uin, self.bytesToStr(variable.Vfwebqq)), referer = variable.Referer) print("uin_to_qq: %s", html) try: result = json.loads(self.bytesToStr(html)) if result.get("retcode") != 0: return "" qq = result.get("result").get("account") if qq: variable.UsersQQ[uin] = str(qq) return str(qq) except Exception as e: print(e) return "" def sendMsg(self, *args, **kwargs): print("回复消息") url = kwargs.get("url") data = kwargs.get("data") # print(data) referer = kwargs.get("referer") result = self._http.post(url = url, data = data, referer = referer) print("回复结果: %s", result) def otherMsg(self, content, to, url, uin): if content: html = self._http.get(url = variable.RobotUrl.format(quote(content), uin)) html = html.replace("\\n", "").replace("\n", "") html = self._api.parse(html) html = self._api.getResult() if html: print("智能回复: ", html) data = {'r' : variable.Msg_Data.format(to, uin, html, variable.Clientid, variable.Msgid, variable.Psessionid)} print(data) self._pool.add_task(callback = self.sendMsg, url = url, data = data, referer = variable.Referer) def analyze(self, qq, uin, content, iseq = None): print("开始解析消息") if iseq: print("消息来自群") to = "group_uin" url = variable.Send_qun_msg2 else: print("消息来自好友") to = "to" url = variable.Send_buddy_msg2 # 是管理员 if self.isAdmin(qq) and content in ("开启机器人", "关闭机器人", "退出"): # 解析管理员命令 _msg = "" print("是管理员消息") if content == "开启机器人": variable.State = True print("机器人已开启") _msg = "机器人已开启" elif content == "关闭机器人": variable.State = False print("机器人已关闭") _msg = "机器人已关闭" elif content == "退出": variable.State = False variable.Exit = True print("机器人已退出") _msg = "机器人已退出" if _msg: data = {'r' : variable.Msg_Data.format(to, uin, _msg, variable.Clientid, variable.Msgid, variable.Psessionid)} self._pool.add_task(callback = self.sendMsg, url = url, data = data, referer = variable.Referer) return # 给我发送的私人消息(是否是命令) result = variable.Command.findall(content) if result and to == "to": ver, msg = result[0] _msg = "" if ver == variable.AddAdmin: # 添加管理员 print("添加管理员") if self.addAdmin(msg): _msg = "添加管理员: " + msg + " 成功" else: _msg = "添加管理员: " + msg + " 失败" elif ver == variable.DelAdmin: # 删除管理员 print("删除管理员") if self.delAdmin(msg): _msg = "删除管理员: " + msg + " 成功" else: _msg = "删除管理员: " + msg + " 失败" elif ver == variable.AddAttention: # 添加关注群号 print("添加关注群") if self.addGroup(msg): _msg = "添加关注群: " + msg + " 成功" else: _msg = "添加关注群: " + msg + " 失败" elif ver == variable.DelAttention: # 删除关注群号 print("删除关注群号") if self.delGroup(msg): _msg = "删除关注群: " + msg + " 成功" else: _msg = "删除关注群: " + msg + " 失败" if _msg: data = {'r' : variable.Msg_Data.format(to, uin, _msg, variable.Clientid, variable.Msgid, variable.Psessionid)} self._pool.add_task(callback = self.sendMsg, url = url, data = data, referer = variable.Referer) return if content.startswith("#") and len(content) > 2 and self.inGroup(iseq): # 开头以#开头并且长度大于2并且是关注群发送的消息 i = 0 content = content[1:].strip() for w in content: if w in variable.Filter: print("发现过滤词: ", w) i += 1 if i == 0: self._pool.add_task(callback = self.otherMsg, content = content, to = to, url = url, uin = uin) return else: data = {'r' : variable.Msg_Data.format(to, uin, "你想干什么!f**k", variable.Clientid, variable.Msgid, variable.Psessionid)} self._pool.add_task(callback = self.sendMsg, url = url, data = data, referer = variable.Referer) return if to == "to" and len(content) > 2: # 私人消息 self._pool.add_task(callback = self.otherMsg, content = content.strip(), to = to, url = url, uin = uin) return def delwith(self, fuin, suin, iseq, content): ''' #fuin 消息发送者 #suin 群消息发送人qq #iseq 群号码 #content 消息内容 ''' print("%s %s %s %s", fuin, suin, iseq, content) # 如果是群文件消息 if content.startswith("<?xml"): print("发现共享文件") # 发送者qq if iseq and suin: qq = self.uin_to_qq(suin) else: qq = self.uin_to_qq(fuin) print("qq: %s", qq) self.analyze(qq, fuin, content, iseq)
for i in range(1, 10): try: imageUrls = get_image_url(restUrl) break except Exception, e: print 'get restUrl error times' + str(i) + ': %s' % (e,) logging.error('get restUrl error times' + str(i) + ': %s' % (e,)) time.sleep(10) if imageUrls is None or len(imageUrls) == 0: print 'get imageUrls error %s' % restUrl logging.error('get imageUrls error %s' % restUrl) continue # logging.debug("progress: %d of %d, %s , %d images", progress, total, reviewName, len(urls)) # 进度 count = 0 for imageUrl in imageUrls: # download_image(imageUrl, LOCAL_DIR + reviewName, # imageUrl[imageUrl.rfind("/") + 1:] + ".jpg") pool.add_task(download_image, imageUrl, LOCAL_DIR + reviewName, imageUrl[imageUrl.rfind("/") + 1:], SLEEP_SECONDS) # 多线程下载图片 count += 1 # logging.debug("task added: %d", count) # logging.debug("finished : %s", reviewName) # print "finished : %s" % ( reviewName) #logging.info("finished : %s" % ( reviewName)) pool.destroy()
def main_fresh(dbOrNot): """ Monitor URLs using fresh data. """ # set value for oldUrlObjDic dict. f = open("./urgentCriterion_new") while 1: string = f.readline().strip() if not string: break arr = string.split(",") #URL Object Format: URL(length, md5) oldUrlObjDic[arr[0]] = URL(int(arr[1]), arr[2]) f.close() f = open("./urgentAccessErrorURLs") while 1: string= f.readline().strip() if not string: break aeURLs.append(string) f.close() #lxw_tp #threadingNum = threading.Semaphore(THREADS_NUM) tp = ThreadPool(THREADS_NUM) threads = [] urlCount = 0 # monitor each url in .urls file f = open("./.urgentURLS") while 1: url = f.readline().strip() if not url: break #lxw_tp #Multiple Thread: Deal with "one url by one single thread". #mt = MyThread(monitor, (url,), threadingNum) tp.add_task(monitor, url) #mt.start() #threads.append(mt) urlCount += 1 f.close() #lxw_tp tp.destroy() #for thread in threads: # thread.start() """ while 1: over = True for thread in threads: if thread.isAlive(): if not thread.isTimedOut(): # not "Timed Out". over = False else: urgentMyUtils.writeLog("lxw_Timed Out", thread.getURL(), "") if over: break """ if aeCount > 0: allContent = "本次共监测网站{0}个, 其中有{1}个网站访问异常, 详细信息如下:\n\n{2}".format(urlCount, aeCount, aeContent) urgentMyUtils.sendEmail(aeSubject, allContent) if uwCount >0: allContent = "本次共监测网站{0}个, 其中有{1}个网站监测到有更新, 详细信息如下:\n\n{2}".format(urlCount, uwCount, uwContent) urgentMyUtils.sendEmail(uwSubject, allContent) #Update Criterion file. f = open("./urgentCriterion_new", "w") for url in newUrlObjDic.keys(): f.write("{0},{1},{2}\n".format(url, newUrlObjDic[url].length, newUrlObjDic[url].getMD5Str())) f.close() dbOrNot = False if dbOrNot: #update criterion in database urgentMyUtils.updateCriterion(newUrlObjDic) #Update accessErrorURLs file. f = open("./urgentAccessErrorURLs", "w") for url in aeURLs: f.write(url + "\n") f.close()
class Server(object): """ A server to host MPI, will delegate all of its calls to the active simulation kernel. """ # Don't forward some of the internally provided functions, but simply raise an AttributeError noforward = frozenset(["__str__", "__getstate__", "__setstate__", "__repr__"]) def __init__(self, name, totalSize): """ Constructor :param name: the name of the server, used for addressing (in MPI terms, this is the rank) :param totalSize: the total size of the network in which the model lives """ self.name = name self.kernel = None self.size = totalSize self.proxies = [MPIRedirect(i) for i in range(totalSize)] from MPIRedirect import LocalRedirect self.proxies[name] = LocalRedirect(self) self.queuedMessages = [] self.queuedTime = None if totalSize > 1: self.threadpool = ThreadPool(2) self.bootMPI() def getProxy(self, rank): """ Get a proxy to a specified rank. This rank is allowed to be the local server, in which case a local shortcut is created. :param rank: the rank to return a proxy to, should be an int :returns: proxy to the server, either of type MPIRedirect or LocalRedirect """ return self.proxies[rank] def checkLoadCheckpoint(self, name, gvt): """ Reconstruct the server from a checkpoint. :param name: name of the checkpoint :param gvt: the GVT to restore to :returns: bool -- whether or not the checkpoint was successfully loaded """ rank = self.name #assert debug("Accessing file " + str("%s_%s_%s.pdc" % (name, gvt, rank))) try: infile = open("%s_%s_%s.pdc" % (name, gvt, rank), 'r') pickle.load(infile) return True except KeyboardInterrupt: # If the user interrupts, still reraise raise except Exception as e: # Something went wrong print("Error found: " + str(e)) return False def loadCheckpoint(self, name, gvt): """ Reconstruct the server from a checkpoint. :param name: name of the checkpoint :param gvt: the GVT to restore to """ rank = self.name #assert debug("Accessing file " + str("%s_%s_%s.pdc" % (name, gvt, rank))) infile = open("%s_%s_%s.pdc" % (name, gvt, rank), 'r') self.kernel = pickle.load(infile) self.kernel.server = self from MPIRedirect import LocalRedirect self.proxies[self.name] = LocalRedirect(self) infile.close() #assert debug("Closing file") self.kernel.loadCheckpoint() def setPickledData(self, pickled_data): """ Set the pickled representation of the model. For use on the controller itself, as this doesn't need to unpickle the model. :param pickled_data: the pickled model """ self.kernel.pickledModel = pickled_data def prepare(self, scheduler): """ Prepare the server to receive the complete model over MPI :param scheduler: the scheduler to use """ data = middleware.COMM_WORLD.bcast(None, root=0) if data is not None: self.saveAndProcessModel(data, scheduler) middleware.COMM_WORLD.barrier() def saveAndProcessModel(self, pickledModel, scheduler): """ Receive the model and set it on the server, but also saves it for further reinitialisation. :param pickledModel: pickled representation of the model :param scheduler: the scheduler to use """ self.sendModel(pickle.loads(pickledModel), scheduler) self.kernel.pickledModel = pickledModel def getName(self): """ Returns the name of the server Is practically useless, since the server is previously addressed using its name. This does have a use as a ping function though. """ # Actually more of a ping function... return self.name # All calls to this server are likely to be forwarded to the currently # active simulation kernel, so provide an easy forwarder def __getattr__(self, name): """ Remote calls happen on the server object, though it is different from the simulation kernel itself. Therefore, forward the actual function call to the correct kernel. :param name: the name of the method to call :returns: requested attribute """ # For accesses that are actually meant for the currently running kernel if name in Server.noforward: raise AttributeError() return getattr(self.kernel, name) def processMPI(self, data, comm, remote): """ Process an incomming MPI message and reply to it if necessary :param data: the data that was received :param comm: the MPI COMM object :param remote: the location from where the message was received """ # Receiving a new request resendTag = data[0] function = data[1] args = data[2] kwargs = data[3] result = getattr(self, function)(*args, **kwargs) if resendTag is not None: if result is None: result = 0 comm.send(result, dest=remote, tag=resendTag) def listenMPI(self): """ Listen for incomming MPI messages and process them as soon as they are received """ comm = middleware.COMM_WORLD status = middleware.MPI.Status() while 1: #assert debug("[" + str(comm.Get_rank()) + "]Listening to remote " + str(middleware.MPI.ANY_SOURCE) + " -- " + str(middleware.MPI.ANY_TAG)) # First check if a message is present, otherwise we would have to do busy polling data = comm.recv(source=middleware.MPI.ANY_SOURCE, tag=middleware.MPI.ANY_TAG, status=status) tag = status.Get_tag() #assert debug("Got data from " + str(status.Get_source()) + " (" + str(status.Get_tag()) + "): " + str(data)) if tag == 0: # Flush all waiters, as we will never receive an answer when we close the receiver... self.finishWaitingPool() break elif tag == 1: # NOTE Go back to listening ASAP, so do the processing on another thread if data[1] == "receive" or data[1] == "receiveAntiMessages": self.threadpool.add_task(Server.processMPI, self, list(data), comm, status.Get_source()) else: # Normal 'control' commands are immediately executed, as they would otherwise have the potential to deadlock the node threading.Thread(target=Server.processMPI, args=[self, list(data), comm, status.Get_source()]).start() else: # Receiving an answer to a previous request try: event = MPIRedirect.waiting[tag] MPIRedirect.waiting[tag] = data event.set() except KeyError: # Probably processed elsewhere already, just skip pass except AttributeError: # Key was already set elsewhere pass def finishWaitingPool(self): """ Stop the complete MPI request queue from blocking, used when stopping simulation is necessary while requests are still outstanding. """ for i in MPIRedirect.waiting: try: i.set() except AttributeError: # It was not a lock... pass except KeyError: # It was deleted in the meantime pass def bootMPI(self): """ Boot the MPI receivers when necessary, on an other thread to prevent blocking """ if self.size > 1: listener = threading.Thread(target=Server.listenMPI, args=[self]) # Make sure that this is a daemon on the controller, as otherwise this thread will prevent the atexit from stopping # Though on every other node this should NOT be a daemon, as this is the only part still running if middleware.COMM_WORLD.Get_rank() == 0: listener.daemon = True listener.start() def sendModel(self, data, scheduler): """ Receive a complete model and set it. :param data: a tuple containing the model, the model_ids dictionary, scheduler name, and a flag for whether or not the model was flattened to allow pickling :param scheduler: the scheduler to use """ model, model_ids, flattened = data if self.name == 0: self.kernel = Controller(self.name, model, self) else: self.kernel = BaseSimulator(self.name, model, self) self.kernel.sendModel(model, model_ids, scheduler, flattened) def finish(self): """ Stop the currently running simulation """ sim = self.kernel with sim.simlock: # Shut down all threads on the topmost simulator sim.finished = True sim.shouldrun.set() self.finishWaitingPool() # Wait until they are done sim.simFinish.wait() def queueMessage(self, time, model_id, action): """ Queue a delayed action from being sent, to make it possible to batch them. Will raise an exception if previous messages form a different time were not yet flushed! This flushing is not done automatically, as otherwise the data would be received at a further timestep which causes problems with the GVT algorithm. :param time: the time at which the action happens :param model_id: the model_id that executed the action :param action: the action to execute (as a string) """ if self.queuedTime is None: self.queuedTime = time elif time != self.queuedTime: raise DEVSException("Queued message at wrong time! Probably forgot a flush") self.queuedMessages.append([model_id, action]) def flushQueuedMessages(self): """ Flush all queued messages to the controller. This will block until all of them are queued. It is required to flush all messages right after all of them happened and this should happen within the critical section! """ if self.queuedTime is not None: self.getProxy(0).massDelayedActions(self.queuedTime, self.queuedMessages) self.queuedMessages = [] self.queuedTime = None
class Server(object): """ A server to host MPI, will delegate all of its calls to the active simulation kernel. """ # Don't forward some of the internally provided functions, but simply raise an AttributeError noforward = frozenset( ["__str__", "__getstate__", "__setstate__", "__repr__"]) def __init__(self, name, totalSize): """ Constructor :param name: the name of the server, used for addressing (in MPI terms, this is the rank) :param totalSize: the total size of the network in which the model lives """ self.name = name self.kernel = None self.size = totalSize self.proxies = [MPIRedirect(i) for i in range(totalSize)] from MPIRedirect import LocalRedirect self.proxies[name] = LocalRedirect(self) self.queuedMessages = [] self.queuedTime = None if totalSize > 1: self.threadpool = ThreadPool(2) self.bootMPI() def getProxy(self, rank): """ Get a proxy to a specified rank. This rank is allowed to be the local server, in which case a local shortcut is created. :param rank: the rank to return a proxy to, should be an int :returns: proxy to the server, either of type MPIRedirect or LocalRedirect """ return self.proxies[rank] def checkLoadCheckpoint(self, name, gvt): """ Reconstruct the server from a checkpoint. :param name: name of the checkpoint :param gvt: the GVT to restore to :returns: bool -- whether or not the checkpoint was successfully loaded """ rank = self.name #assert debug("Accessing file " + str("%s_%s_%s.pdc" % (name, gvt, rank))) try: infile = open("%s_%s_%s.pdc" % (name, gvt, rank), 'r') pickle.load(infile) return True except KeyboardInterrupt: # If the user interrupts, still reraise raise except Exception as e: # Something went wrong print("Error found: " + str(e)) return False def loadCheckpoint(self, name, gvt): """ Reconstruct the server from a checkpoint. :param name: name of the checkpoint :param gvt: the GVT to restore to """ rank = self.name #assert debug("Accessing file " + str("%s_%s_%s.pdc" % (name, gvt, rank))) infile = open("%s_%s_%s.pdc" % (name, gvt, rank), 'r') self.kernel = pickle.load(infile) self.kernel.server = self from MPIRedirect import LocalRedirect self.proxies[self.name] = LocalRedirect(self) infile.close() #assert debug("Closing file") self.kernel.loadCheckpoint() def setPickledData(self, pickled_data): """ Set the pickled representation of the model. For use on the controller itself, as this doesn't need to unpickle the model. :param pickled_data: the pickled model """ self.kernel.pickledModel = pickled_data def prepare(self, scheduler): """ Prepare the server to receive the complete model over MPI :param scheduler: the scheduler to use """ data = middleware.COMM_WORLD.bcast(None, root=0) if data is not None: self.saveAndProcessModel(data, scheduler) middleware.COMM_WORLD.barrier() def saveAndProcessModel(self, pickledModel, scheduler): """ Receive the model and set it on the server, but also saves it for further reinitialisation. :param pickledModel: pickled representation of the model :param scheduler: the scheduler to use """ self.sendModel(pickle.loads(pickledModel), scheduler) self.kernel.pickledModel = pickledModel def getName(self): """ Returns the name of the server Is practically useless, since the server is previously addressed using its name. This does have a use as a ping function though. """ # Actually more of a ping function... return self.name # All calls to this server are likely to be forwarded to the currently # active simulation kernel, so provide an easy forwarder def __getattr__(self, name): """ Remote calls happen on the server object, though it is different from the simulation kernel itself. Therefore, forward the actual function call to the correct kernel. :param name: the name of the method to call :returns: requested attribute """ # For accesses that are actually meant for the currently running kernel if name in Server.noforward: raise AttributeError() return getattr(self.kernel, name) def processMPI(self, data, comm, remote): """ Process an incomming MPI message and reply to it if necessary :param data: the data that was received :param comm: the MPI COMM object :param remote: the location from where the message was received """ # Receiving a new request resendTag = data[0] function = data[1] args = data[2] kwargs = data[3] result = getattr(self, function)(*args, **kwargs) if resendTag is not None: if result is None: result = 0 comm.send(result, dest=remote, tag=resendTag) def listenMPI(self): """ Listen for incomming MPI messages and process them as soon as they are received """ comm = middleware.COMM_WORLD status = middleware.MPI.Status() while 1: #assert debug("[" + str(comm.Get_rank()) + "]Listening to remote " + str(middleware.MPI.ANY_SOURCE) + " -- " + str(middleware.MPI.ANY_TAG)) # First check if a message is present, otherwise we would have to do busy polling data = comm.recv(source=middleware.MPI.ANY_SOURCE, tag=middleware.MPI.ANY_TAG, status=status) tag = status.Get_tag() #assert debug("Got data from " + str(status.Get_source()) + " (" + str(status.Get_tag()) + "): " + str(data)) if tag == 0: # Flush all waiters, as we will never receive an answer when we close the receiver... self.finishWaitingPool() break elif tag == 1: # NOTE Go back to listening ASAP, so do the processing on another thread if data[1] == "receive" or data[1] == "receiveAntiMessages": self.threadpool.add_task(Server.processMPI, self, list(data), comm, status.Get_source()) else: # Normal 'control' commands are immediately executed, as they would otherwise have the potential to deadlock the node threading.Thread( target=Server.processMPI, args=[self, list(data), comm, status.Get_source()]).start() else: # Receiving an answer to a previous request try: event = MPIRedirect.waiting[tag] MPIRedirect.waiting[tag] = data event.set() except KeyError: # Probably processed elsewhere already, just skip pass except AttributeError: # Key was already set elsewhere pass def finishWaitingPool(self): """ Stop the complete MPI request queue from blocking, used when stopping simulation is necessary while requests are still outstanding. """ for i in MPIRedirect.waiting: try: i.set() except AttributeError: # It was not a lock... pass except KeyError: # It was deleted in the meantime pass def bootMPI(self): """ Boot the MPI receivers when necessary, on an other thread to prevent blocking """ if self.size > 1: listener = threading.Thread(target=Server.listenMPI, args=[self]) # Make sure that this is a daemon on the controller, as otherwise this thread will prevent the atexit from stopping # Though on every other node this should NOT be a daemon, as this is the only part still running if middleware.COMM_WORLD.Get_rank() == 0: listener.daemon = True listener.start() def sendModel(self, data, scheduler): """ Receive a complete model and set it. :param data: a tuple containing the model, the model_ids dictionary, scheduler name, and a flag for whether or not the model was flattened to allow pickling :param scheduler: the scheduler to use """ model, model_ids, flattened = data if self.name == 0: self.kernel = Controller(self.name, model, self) else: self.kernel = BaseSimulator(self.name, model, self) self.kernel.sendModel(model, model_ids, scheduler, flattened) def finish(self): """ Stop the currently running simulation """ sim = self.kernel with sim.simlock: # Shut down all threads on the topmost simulator sim.finished = True sim.shouldrun.set() self.finishWaitingPool() # Wait until they are done sim.simFinish.wait() def queueMessage(self, time, model_id, action): """ Queue a delayed action from being sent, to make it possible to batch them. Will raise an exception if previous messages form a different time were not yet flushed! This flushing is not done automatically, as otherwise the data would be received at a further timestep which causes problems with the GVT algorithm. :param time: the time at which the action happens :param model_id: the model_id that executed the action :param action: the action to execute (as a string) """ if self.queuedTime is None: self.queuedTime = time elif time != self.queuedTime: raise DEVSException( "Queued message at wrong time! Probably forgot a flush") self.queuedMessages.append([model_id, action]) def flushQueuedMessages(self): """ Flush all queued messages to the controller. This will block until all of them are queued. It is required to flush all messages right after all of them happened and this should happen within the critical section! """ if self.queuedTime is not None: self.getProxy(0).massDelayedActions(self.queuedTime, self.queuedMessages) self.queuedMessages = [] self.queuedTime = None
class server: """docstring for server.""" def __init__(self, host, port): self.max_threads = 8 self.host = host self.port = port self.chatrooms = {} # room_name -> room self.chatroom_ids = {} # id -> name self.client_ids = {} self.setup_socket() self.tp = ThreadPool(self.max_threads) self.accept_connections() def setup_socket(self): ''' Sets up a tcp socket that listens on the host and port given ''' self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.s.bind((self.host, int(self.port))) self.s.listen(50) def accept_connections(self): while True: client, address = self.s.accept() print("in accept, client: " + str(client)) print("in accept, address: " + str(address)) self.client_thread(client, address) print("loops in accept connection") def client_thread(self, client, addr): kargs = {"client": client, "address": addr} self.tp.add_task(self.handle, **kargs) # t = threading.Thread(target=self.handle, kwargs=kargs).start() def handle(self, **kwargs): client = kwargs["client"] # client ip address = kwargs["address"] data = client.recv(1024) data = data.decode("utf-8") print(data) if data[:5] == "HELO ": text = data[6:] self.handle_helo(client, text) elif data[:len("KILL_SERVICE")] == "KILL_SERVICE": self.handle_kill() elif data[:4] == "join": message = data.split("\n") # these need to be changed the remove the labels, now it takes just the label message[0] = message[0][14:] # chatroom name message[1] = message[1][10:] # port message[2] = message[2][5:] # ip message[3] = message[3][13:] # client name print("message recieved: ") for i in message: i = i.strip() print(i) self.handle_join_room(message[0], message[3], client, address) elif data[:len("DISCONNECT: ")] == "DISCONNECT: ": self.handle_disconnect() else: self.handle_other() def handle_helo(self, client, text): """ send back required information """ ip = socket.gethostbyname(socket.gethostname()) reply = "HELO " + text + "IP:" + ip + "\nPort:" + str(self.port) +\ "\nStudentID:13325878\n" client.send(reply.encode('utf-8')) def handle_kill(self): """ shuts down service """ os._exit(0) def handle_other(self): """ called when we receive an unhandled message """ pass def handle_join_room(self, room_name, client_name, client_ip=0, client_port=0): if room_name not in self.chatrooms: room = Chatroom(room_name) self.chatrooms[room_name] = room self.chatroom_ids[room.get_id()] = room_name room.subscribe(client_name, client_ip) else: # create and join room room = self.chatrooms[room_name] room.subscribe(client_name, client_ip) room.subscribe(client_name, client_ip, client_port) l = room.get_publish_list() client_ip.send( self.success_response(room_name, self.host, self.port, room.get_id(), id(client_name)).encode()) self.client_ids[id(client_name)] = client_name for i in l: # print("i in l loop: " + str(i) + "\n\n") i[1].sendall((str(client_name) + " has joined the room").encode()) print("sent") # this could be wrong def success_response(self, room_name0, host, port, room_id, client_id): return "JOINED_CHATROOM: " + room_name0 + "\n" +\ "SERVER_IP: " + str(host) + "\n" +\ "PORT: " + str(port) + "\n" +\ "ROOM_REF: " + str(room_id) + "\n" +\ "JOIN_ID: " + str(client_id) def handle_leave_chatroom(self, host, port, room_id, client_name): room = self.chatrooms[self.chatroom_ids[room_id]] room.unsubscribe(client_name) host.send(self.leave_message(room_id, client_name)) l = room.get_publish_list() for i in l: i[0].sendall(self.left_message(room_id, client_name).encode()) def leave_message(self, room_id, client_name): cid = self.client_ids[client_name] return "LEAVE_CHATROOM: " + str(room_id) + "\n" +\ "JOIN_ID: " + str(cid) + "\n" +\ "CLIENT_NAME: " + client_name def left_message(self, room_id, client_name): cid = self.chatroom_ids[client_name] return "LEFT_CHATROOM: " + str(room_id) + "\n" +\ "JOIN_ID: " + str(cid) def handle_disconnect(self): self.s.close() def handle_chat(self, room_id, client_name, msg): mmessage = "CHAT: " + str(room_id) + "\n" +\ "CLIENT_NAME: " + client_name + "\n" +\ "MESSAGE: " + msg room = self.chatrooms[self.chatroom_ids[room_id]] l = room.get_publish_list() for i in l: i[0].sendall(mmessage.encode())
class Database: """Asynchronous database interface. The `driver' argument specifies which database to use. Possible values are: MySQLdb - for MySQL psycopg2 - for Postgres """ def __init__(self, driver=None, database=None, user=None, password=None, host='localhost', ioloop=tornado.ioloop.IOLoop.instance(), num_threads=10, tx_connection_pool_size=5, queue_timeout=1, thread_idle_life=60*60): if not(driver): raise ValueError("Missing 'driver' argument") self._driver = driver self._database = database self._user = user self._password = password self._host = host self._threadpool = ThreadPool( per_thread_init_func=self.create_connection, per_thread_close_func=self.close_connection, num_threads=num_threads, queue_timeout=queue_timeout, thread_idle_life=thread_idle_life) self._ioloop = ioloop # Connection pool for transactions self._connection_pool = [] for i in xrange(tx_connection_pool_size): conn = self.create_connection() self._connection_pool.append(conn) self._waiting_on_connection = deque() def create_connection(self): """This method is executed in a worker thread. Initializes the per-thread state. In this case we create one database connection per-thread. """ # if self._driver == "psycopg2": # try: # import psycopg2 # conn = psycopg2.connect(database=self._database, # user=self._user, # password=self._password, # host=self._host) # except Exception as ex: # raise ex if self._driver == "MySQLdb": try: import MySQLdb conn = MySQLdb.connect(db=self._database, user=self._user, passwd=self._password, host=self._host, port=3306) except Exception as ex: raise ex else: raise ValueError("Unknown driver %s" % self._driver) return conn def close_connection(self, conn): conn.close() def stop(self): self._threadpool.stop() for conn in self._connection_pool: conn.close() @async def beginTransaction(self, callback): """Begins a transaction. Picks up a transaction from the pool and passes it to the callback. If none is available, adds the callback to `_waiting_on_connection'. """ if self._connection_pool: conn = self._connection_pool.pop() callback(conn) else: self._waiting_on_connection.append(callback) @async def commitTransaction(self, connection, callback): self._threadpool.add_task( partial(self._commitTransaction, connection, callback)) def _commitTransaction(self, conn, callback, thread_state=None): """Invoked in a worker thread. """ conn.commit() self._ioloop.add_callback( partial(self._releaseConnectionInvokeCallback, conn, callback)) @async def rollbackTransaction(self, connection, callback): self._threadpool.add_task( partial(self._rollbackTransaction, connection, callback)) def _rollbackTransaction(self, conn, callback, thread_state=None): """Invoked in a worker thread. """ conn.rollback() self._ioloop.add_callback( partial(self._releaseConnectionInvokeCallback, conn, callback)) def _releaseConnectionInvokeCallback(self, conn, callback): """Release the connection back in the connection pool and invoke the callback. Invokes any waiting callbacks before releasing the connection into the pool. """ # First invoke the callback to let the program know we're done # with the transaction. callback(conn) # Now check to see if we have any pending clients. If so pass # them the newly released connection. if self._waiting_on_connection: callback = self._waiting_on_connection.popleft() callback(conn) else: self._connection_pool.append(conn) @async def runQuery(self, query, args=None, conn=None, callback=None): """Send a SELECT query to the database. The callback is invoked with all the rows in the result. """ self._threadpool.add_task( partial(self._query, query, args, conn), callback) def _query(self, query, args, conn=None, thread_state=None): """This method is called in a worker thread. Execute the query and return the result so it can be passed as argument to the callback. """ if not conn: conn = thread_state should_commit = True else: should_commit = False cursor = conn.cursor() try: cursor.execute(query, args) except: raise else: rows = cursor.fetchall() return rows finally: if should_commit: conn.commit() cursor.close() @async def runOperation(self, stmt, args=None, conn=None, callback=None): """Execute a SQL statement other than a SELECT. The statement is committed immediately. The number of rows affected by the statement is passed as argument to the callback. """ self._threadpool.add_task( partial(self._execute, stmt, args, conn), callback) def _execute(self, stmt, args, conn=None, thread_state=None): """This method is called in a worker thread. Executes the statement. """ # Check if stmt is a tuple. This can happen when we use map() # with adisp to execute multiple statements in parallel. if isinstance(stmt, tuple): args = stmt[1] stmt = stmt[0] if not conn: conn = thread_state should_commit = True else: should_commit = False cursor = conn.cursor() try: cursor.execute(stmt, args) except: raise else: rowcount = cursor.rowcount return rowcount finally: if should_commit: conn.commit() cursor.close() @async def runOperationMany(self, stmt, args=None, conn=None, callback=None): """Execute a SQL statement other than a SELECT. The statement is committed immediately. The number of rows affected by the statement is passed as argument to the callback. """ self._threadpool.add_task( partial(self._executeMany, stmt, args, conn), callback) def _executeMany(self, stmt, args, conn=None, thread_state=None): """This method is called in a worker thread. Executes the statement. """ # Check if stmt is a tuple. This can happen when we use map() # with adisp to execute multiple statements in parallel. if isinstance(stmt, tuple): args = stmt[1] stmt = stmt[0] if not conn: conn = thread_state should_commit = True else: should_commit = False cursor = conn.cursor() try: cursor.executemany(stmt, args) except: raise else: rowcount = cursor.rowcount return rowcount finally: if should_commit: conn.commit() cursor.close()
class Server(object): #cython-remove noforward = frozenset(["__str__", "__getstate__", "__setstate__", "__repr__", "simstack"]) local = frozenset(["setCoupledModel", "finish", "endNestedSim", "nestedSim", "continuePrevSim", "startNestedSim", "announceSim", "announceEnd", "setKernels", "notifyDone"]) def __init__(self, name, daemon = None): self.daemon = daemon self.name = str(name) self.simstack = [None] import logging setLogger(str(name), ('localhost', 514), logging.DEBUG) MPIRedirect.local = self if middleware.USE_MPI: self.threadpool = ThreadPool(5) def getName(self): # Actually more of a ping function... return self.name # All calls to this server are likely to be forwarded to the currently # active simulation kernel, so provide an easy forwarder def __getattr__(self, name): # For accesses that are actually meant for the currently running kernel if name in Server.noforward: raise AttributeError() if self.simstack[-1] is None: return getattr(self.simstack[-2], name) else: return getattr(self.simstack[-1], name) def setNextLP(self, location, size): self.nextLP = getProxy(location) self.nextLPid = location self.size = size for i in self.simstack: i.setNextLP(location, size) def getNextLP(self): return self.nextLPid def nestedSim(self): self.simstack[-1].nestedlock.acquire() # Now the current simulation is halted, ready to add a new simulation def endNestedSim(self): sim = self.simstack.pop() # Clear simulator del sim def continuePrevSim(self): self.simstack[-1].nestedlock.release() def getLayer(self): return len(self.simstack) def startNestedSim(self): self.simstack.append(None) def announceSim(self, requestname): # A new simulation must start, halt ALL simulators # Wait for the GVT to stop running #NOTE mandatory that this runs on the controller # Try to initiate a nested simulation at this level if not self.simstack[-1].nestingLock.acquire(False): return False for i in range(self.size): if str(i) == str(requestname): # Don't wait for ourselves, as we are still simulating while inside the nested model continue getProxy(i).nestedSim() for i in range(self.size): # All cores are locked, initiate a new round getProxy(i).startNestedSim() return True def announceEnd(self, requestname): # Prevent GVT algorithm from running in this unstable state for i in range(self.size): getProxy(i).endNestedSim() for i in range(self.size): if str(i) == str(requestname): # Don't continue with ourselves, since we are actually still running continue getProxy(i).continuePrevSim() # Unlock GVT lock self.simstack[-1].nestingLock.release() def processMPI(self, data, comm, remote): # Receiving a new request resendTag = data[0] function = data[1] kernel = data[2] args = data[3] kwargs = data[4] # Run this simulation at the topmost simulation kernel if (function in Server.local): func = getattr(self, function) else: func = getattr(self.simstack[kernel], function) result = func(*args, **kwargs) if result is None: result = 0 if resendTag is not None: assert debug("Sending data to " + str(remote) + " -- " + str(resendTag) + ": " + str(result)) comm.send(result, dest=remote, tag=resendTag) def listenMPI(self): """ import cProfile cProfile.runctx("self.profile_listenMPI()", locals(), globals()) def profile_listenMPI(self): """ comm = COMM_WORLD #NOTE could preallocate the memory for the incomming call, though possible threading problems lasttime = time.time() status = MPI.Status() while True: assert debug("[" + str(comm.Get_rank()) + "]Listening to remote " + str(MPI.ANY_SOURCE) + " -- " + str(MPI.ANY_TAG)) # First check if a message is present, otherwise we would have to do busy polling """ while not comm.Iprobe(source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG, status=status): # Yield to other threads # 0 to mean that we should yield elsewhere, UNLESS no other process is waiting time.sleep(0) """ data = comm.recv(source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG, status=status) tag = status.Get_tag() assert debug("Got data from " + str(status.Get_source()) + " (" + str(status.Get_tag()) + "): " + str(data)) if tag == 0: # Flush all waiters, as we will never receive an answer when we close the receiver... for i in MPIRedirect.waiting: try: i.set() except AttributeError: # It was not a lock... pass except KeyError: # It was deleted in the meantime pass break elif tag == 1: # NOTE Go back to listening ASAP, so do the processing on another thread if data[1] == "getTargets": # This should not be queued as it might be called recursively and we want to prevent this threading.Thread(target=Server.processMPI, args=[self, list(data), comm, status.Get_source()]).start() else: self.threadpool.add_task(Server.processMPI, self, list(data), comm, status.Get_source()) else: # Receiving an answer to a previous request try: event = MPIRedirect.waiting[tag] MPIRedirect.waiting[tag] = data event.set() except KeyError: # Probably processed elsewhere already, just skip pass except AttributeError: # Key was already set elsewhere pass def bootMPI(self): if COMM_WORLD.Get_size() > 1: listener = threading.Thread(target=Server.listenMPI, args=[self]) listener.start() def setCoupledModel(self, modelCode, parent_location, parent_id, imports = None): if isinstance(modelCode, str): # Put this here for Python3 model = None if imports is not None: exec(imports) model = eval(modelCode) else: model = modelCode if not isinstance(model, CoupledDEVS): raise DEVSException("Only coupled models are allowed to be distributed") model.internal = False model.location = self.name model.setRootSim(self, self.name) if parent_id is not None: model.parent = RemoteCDEVS(parent_location) else: model.parent = None if self.name == "0": sim = Controller(self.name, model) else: sim = BaseSimulator(self.name, model) self.simstack[-1] = sim # Should still fill in all model_ids sim.setID() def notifyDone(self, name): for i in self.locations: proxy = getProxy(i) makeOneway(proxy, "notifyDone") proxy.notifyKernelDone(name) def finish(self): sim = self.simstack[-1] sim.simlock.acquire() sim.nestedlock.acquire() # Shut down all threads on the topmost simulator sim.finished = True sim.inmsg.set() sim.shouldrun.set() if middleware.USE_MPI: # It is possible that we are still waiting on a reply # since it was agreed to stop, all messages should be useless for i in MPIRedirect.waiting: try: i.set() except AttributeError: # It was not a lock... pass except KeyError: # It was deleted in the meantime pass # Wait until they are done sim.queueFinish.wait() sim.simFinish.wait() if middleware.USE_PYRO: if len(self.simstack) == 1: try: self.daemon.shutdown() except: pass # Release the simlock, since we are possibly working in one of the layers # which will try to grab the simlock sim.nestedlock.release() sim.simlock.release() def setKernels(self, kernels): self.locations = kernels
def run(self): pool = ThreadPool(3) pool.add_task(self.__listen, [self.__requests]) pool.add_task(self.__respond, [self.__responses]) pool.add_task(self.__execute, [self.react, self.__requests, self.__responses])
# An algorithm node, usually with 2 threads each (messaging, main loop) def start_node(index): amqp_url = 'amqp://*****:*****@localhost:5672/%2F' node_identifier = "MulticastNode" + str(index) consumer = AlgorithmNode( node_identifier, amqp_url=amqp_url, on_message_callback_debug=on_message_callback_debug) consumer.run() def start_async_node(index): amqp_url = 'amqp://*****:*****@localhost:5672/%2F' node_identifier = "MulticastNode" + str(index) consumer = AsyncNode(node_identifier, amqp_url=amqp_url, on_message_receive_debug=on_message_callback_debug) consumer.__run() if __name__ == '__main__': # Kickstart the node workers # feel free to introduce random delays like f.e. time.sleep(100) between each start LOGGER.info("Booting algorithm simulator for 2 workers") pool = ThreadPool(1) for i in range(len(pool.workers)): pool.add_task(start_async_node, i) pool.wait_completion()
# set/get print r.set("key", 'b' * 56000) print len(r.get("key")) # incr print r.set("incr_key", 0) print r.get("incr_key") print r.incr('incr_key') print r.get("incr_key") def press_test(): r = redis.StrictRedis(host='localhost', port=8323) for i in range(10000): key = 'foo_%d' % i r.set(key, 'b' * i) if i % 1000 == 0: print key, "->", len(r.get(key)) if __name__ == "__main__": #functional_test() # Create thread pool with nums threads pool = ThreadPool(32) # Add a task into pool for n in range(10): pool.add_task(functional_test) pool.add_task(press_test) # Join and destroy all threads pool.destroy()
class Mailer (object): MAX_TRIES = 3 DELAY_AFTER_FAILURES = 2 def __init__(self, smtp_server, smtp_port, num_threads, queue_timeout, default_sender, ioloop=tornado.ioloop.IOLoop.current()): self.threadpool = ThreadPool( poolname='Mailer', thread_global_data=self, thread_quit_hook=self._quit_smtp, num_threads=num_threads, queue_timeout=queue_timeout) self.smtp_server = smtp_server self.smtp_port = smtp_port self.default_sender = default_sender self.ioloop = ioloop def _create_smtp(self): """This method is executed in a worker thread. Initializes the per-thread state. In this case we create one smtp per-thread. """ smtp = smtplib.SMTP(self.smtp_server, self.smtp_port) return smtp @staticmethod def _quit_smtp(global_data, local_data): smtp = local_data.smtp if smtp is not None: smtp.quit() def send(self, receivers, subject, body, sender=None, reply_to=None, callback=None): self.threadpool.add_task( partial(self._send, sender or self.default_sender, receivers, subject, body, reply_to), callback ) def _send(self, sender, receivers, subject, body, reply_to=None, global_data=None, local_data=None): try: for i in range(self.MAX_TRIES, 0, -1): log_email.debug('sending: try=%d, to=%s, subj=%s', self.MAX_TRIES - i + 1, receivers, subject) try: smtp = local_data.smtp if hasattr(local_data, 'smtp') else None if smtp is None: smtp = global_data._create_smtp() local_data.smtp = smtp msg = MIMEMultipart("alternative") msg["Subject"] = subject if reply_to is not None: msg['reply-to'] = reply_to part1 = MIMEText(body, "plain", "utf-8") msg.attach(part1) smtp.sendmail(sender, receivers, msg.as_string().encode('ascii') ) log_email.debug('mail sent succesfully') return True except smtplib.SMTPException as e: if i == 1: raise e # global_data.quit_smtp() local_data.smtp = None time.sleep(self.DELAY_AFTER_FAILURES) except: etype, evalue, tb = sys.exc_info() log_email.error('can\'t send mail: subject=%s, cause=%s/%s', subject, etype, evalue) log_email.debug('email body: %s', body) log_email.debug('full stacktrace:\n%s', loglib.TracebackFormatter(tb)) return False
class Scheduler(object): """ job scheduler """ def __init__(self, total_resources, job_tree): """" constructor """ self._total_resources = total_resources if not isinstance(self._total_resources, int): raise "total_resources is NOT an int" if self._total_resources < 1: raise "no resources available" self._resources_available = total_resources self._job_tree = job_tree if not isinstance(self._job_tree, list): raise "job_tree is NOT a list" # thread pool for producer/consumer self._tp_prod_cons = ThreadPool(2) # semaphores self._free_res_sem = None self._used_res_sem = None # queue for exchange self._job_buffer = Queue() # thread pool fo the resource management self._tp_resources = ThreadPool(total_resources) # thread pool for the synchronous management of messages self._msg_manager = ThreadPool(1) # descriptions self._results = ["NOT_TESTED", "SCHEDULED", "RUNNING", "OK", "FAILED", "SKIPPED"] def run(self): """ start the scheduler """ # semaphore managed by the producer self._free_res_sem = Semaphore(self._total_resources) # semaphore managed by the consumer self._used_res_sem = Semaphore(0) # threads creation for producer/consumer self._tp_prod_cons.add_task(func=self.__producer) self._tp_prod_cons.add_task(func=self.__consumer) self._tp_prod_cons.wait_completion() self._job_buffer.join() def __print_msg_private(self, msg): print "%s --> %s" % (dt.datetime.now(), msg) def __print_msg(self, msg): self._msg_manager.add_task(self.__print_msg_private, msg) def __job_run(self, job): start_time = dt.datetime.now() five_sec = timedelta(seconds=1) * 5 self.__print_msg(" Job: %s -- Start time: %s" % (job.name, start_time)) while True: sleep(1) if dt.datetime.now() >= start_time + five_sec: break self.__print_msg(" Job: %s -- End time: %s" % (job.name, dt.datetime.now())) #------------------- self._job_buffer.task_done() for _ in range(job.resources): self._used_res_sem.acquire() self._resources_available = self._resources_available + 1 self._free_res_sem.release() # ---------------- # get the result of the test on this job # FOR TEST: put always if job.name == "Test_010": job.status = JobState.TEST_FAIL else: job.status = JobState.TEST_OK self.__print_msg(" Job: %s executed with result -> %s." % \ (job.name, self._results[job.status])) self.__print_msg(" Job: %s resources released -> %d." % \ (job.name, job.resources)) self.__print_msg(" Job: Resources available now: %d." % \ (self._resources_available)) # if the test fails the put all the job children in the TEST_SKIPPED state if job.status == JobState.TEST_FAIL: self.__skip_children(job) def __get_next_job(self, job): next_job = None # if the parent job has not finished the test then skip if job.parent: if job.parent.status != JobState.TEST_OK: return next_job if job.status == JobState.NOT_TESTED and \ job.resources <= self._resources_available: next_job = job if next_job is None and job.children: next_job = self.__get_next_job(job.children[0]) if next_job is None and job.next: next_job = self.__get_next_job(job.next) return next_job def __job_not_scheduled(self, job): job_not_scheduled = (job.status == JobState.NOT_TESTED) if not job_not_scheduled and job.children: job_not_scheduled = self.__job_not_scheduled(job.children[0]) if not job_not_scheduled and job.next: job_not_scheduled = self.__job_not_scheduled(job.next) return job_not_scheduled def __producer(self): """ get the next job and send it to the buffer """ stop_producer = False while True: # if there is at least one resource available if self._resources_available > 0: # search the next job next_job = self.__get_next_job(self._job_tree[0]) if next_job: self.__print_msg("Producer: scheduled the job -> %s resources needed: %d" % \ (next_job.name, next_job.resources)) next_job.status = JobState.SCHEDULED for _ in range(next_job.resources): self._free_res_sem.acquire() self._resources_available = self._resources_available - 1 self._used_res_sem.release() self.__print_msg("Producer: Resources still available: %d." % \ (self._resources_available)) # put the job in the buffer self._job_buffer.put(next_job) else: # are there jobs not tested yet? If not stop then stop Producer if not self.__job_not_scheduled(self._job_tree[0]): stop_producer = True if stop_producer: self.__print_msg("Producer: finished.") break sleep(1) def __job_not_executed(self, job): job_not_executed = (job.status < JobState.TEST_OK) if not job_not_executed and job.children: job_not_executed = self.__job_not_executed(job.children[0]) if not job_not_executed and job.next: job_not_executed = self.__job_not_executed(job.next) return job_not_executed def __skip_children(self, parent_job): for child_job in parent_job.children: child_job.status = JobState.TEST_SKIPPED self.__print_msg(" Job: %s marked as %s." % \ (child_job.name, self._results[child_job.status])) self.__skip_children(child_job) def __consumer(self): """ get the next scheduled job from the buffer and execute it """ stop_consumer = False while True: if not self._job_buffer.empty(): # get the job from the buffer job = self._job_buffer.get() job.status = JobState.RUNNING self.__print_msg("Consumer: the job -> %s is sent to execution." % (job.name)) # add the job to the thread pool self._tp_resources.add_task(self.__job_run, job) else: # are all the jobs tested? If yes then stop Consumer if not self.__job_not_executed(self._job_tree[0]): stop_consumer = True if stop_consumer: self.__print_msg("Consumer: finished.") break sleep(1) self._tp_resources.wait_completion() self._msg_manager.wait_completion()
#!/usr/bin/env python ## coding:utf-8 from threadpool import ThreadPool import time def do_work(*args, **kwds): num = kwds['id'] my_num = 0 while my_num < 10: print 'num:%d' % num print 'do some thing:%d' % my_num my_num = my_num + 1 time.sleep(num) # Create thread pool with nums threads pool = ThreadPool(5) # Add a task into pool pool.add_task(do_work, None, id=1) pool.add_task(do_work, None, id=2) pool.add_task(do_work, None, id=3) # Join and destroy all threads pool.destroy()