def Get_values(hosts,Redis_Key,keys,history,preset,operation): RC.delete(Redis_Key) hosts = [str(host[0]) for host in hosts] method = 'history.get' RC.set('%s_time' % Redis_Key, time.strftime('%Y-%m-%d %H:%M',time.localtime())) def Run(host): try: values = {} for key in keys: value = zabbix_api.GET_value(host, keys[key], method, history) if value != None: if '.' in value: value = float(value) if operation == 'lt': if int(value) < preset: values[key] = value RC.hset(Redis_Key, host, values) if operation == 'gt': if int(value) > preset: values[key] = value RC.hset(Redis_Key, host, values) except Exception as e: logging.error(e) pool = ThreadPool(4) pool.map_async(Run,hosts) pool.close() pool.join()
def test_replication_after_partition(both_https_cluster): node1.query("truncate table test_table") node2.query("truncate table test_table") manager = PartitionManager() def close(num): manager.partition_instances(node1, node2, port=9010) time.sleep(1) manager.heal_all() def insert_data_and_check(num): node1.query( "insert into test_table values('2019-10-15', {}, 888)".format(num)) time.sleep(0.5) closing_pool = Pool(1) inserting_pool = Pool(5) cres = closing_pool.map_async(close, [random.randint(1, 3) for _ in range(10)]) ires = inserting_pool.map_async(insert_data_and_check, list(range(100))) cres.wait() ires.wait() assert_eq_with_retry(node1, "SELECT count() FROM test_table", '100') assert_eq_with_retry(node2, "SELECT count() FROM test_table", '100')
def getvasprun(self, files): m = self.m maindir = pwd() if m.engine == "vasp": calculator = self.getVaspRun_vasp elif m.engine == "lammps": calculator = self.getVaspRun_lammps self.jm = jobManager() for file in files: print(file) dir = "dirs/dir_" + file mkdir(dir) mv(file, dir + '/POSCAR') cd(dir) calculator() cd(maindir) self.jm.run() if m.th: mkdir(m.pbsname) self.thcode(files, m.pbsname) cp("dirs", m.pbsname) passthru("tar zcf %s.tar.gz %s" % (m.pbsname, m.pbsname)) print('start check') self.jm.check() if m.engine == "lammps1": from multiprocessing.dummy import Pool pool = Pool() pool.map_async(lammpsvasprun, files) pool.close() pool.join()
def main(): pool = ThreadPool(1000) # 一定要指定线程数,不然就坑了自己(自己想为啥) print("准备开抢") pool.map_async(shopping, range(1000)) pool.close() pool.join() print(f"剩余库存{count}")
def test_concurrent_writes(self): sender_pool = ThreadPool(100) receiver_pool = ThreadPool(100) message = can.Message( arbitration_id=0x123, extended_id=True, timestamp=121334.365, data=[254, 255, 1, 2] ) workload = 1000 * [message] def sender(msg): self.bus1.send(msg) def receiver(_): result = self.bus2.recv(timeout=2.0) self.assertIsNotNone(result) self.assertEqual(result, message) sender_pool.map_async(sender, workload) receiver_pool.map_async(receiver, len(workload) * [None]) sender_pool.close() sender_pool.join() receiver_pool.close() receiver_pool.join()
def main(): p = ThreadPool() p.map_async(test, list(range(5))) p.close() p.join() print(num) # 应该是500000,发生了数据混乱,结果少了很多
def get_past_data(self, thread=10): self.c1, self.c2, self.conn1, self.conn2 = self.db_init() print("正在获取基金的历史数据") # 获取所有的基金的code codes = self.get_fund_earning_perday() self.c1.execute(self.create_info_table) code = self.code_split(codes, 100) for i in code: # 多线程爬取 print("正在以:", thread, "个线程爬取历史数据") pool = ThreadPool(thread) pool.map_async(self.detail, i) # 开始数据库插入进程 #p = Thread(target=write2sql) #p.start() pool.close() pool.join() #p.join() self.write2sql() self.sql1 = [] self.sql2 = [] # 关闭数据库连接 self.close_db()
def __processing(processArray): # init global vars global cur global curPercent global countServer global countOpen global once global PLOCK global TLOCK # split the host list in number of threads threadHostArray = array_split(processArray, this.threads) # init threading pool tPool = ThreadPool(this.threads) try: # start threading pool tPool.map_async(__threading, threadHostArray).get(9999999) except KeyboardInterrupt: PLOCK.aquire() if once.value == 0: once.value += 1 print('\n\n[!] killing all python processes') # use pkill to make sure every thread is killed os.system('pkill python') sys.exit(0) # close threading pool tPool.close() tPool.join()
def test_exception_message(started_cluster): assert node1.query("select number from nums order by number") == "0\n1\n" def node_busy(_): for i in range(10): node1.query("select sleep(2)", user="******", ignore_error=True) busy_pool = Pool(3) busy_pool.map_async(node_busy, range(3)) time.sleep(1) # wait a little until polling starts with pytest.raises(Exception) as exc_info: for i in range(3): assert (node1.query( "select number from remote('node1', 'default', 'nums')", user="******", ) == "0\n1\n") exc_info.match("Too many simultaneous queries for all users") for i in range(3): assert (node1.query( "select number from remote('node1', 'default', 'nums')", user="******", settings={"max_concurrent_queries_for_all_users": 0}, ) == "0\n1\n")
def mysql_scheduler(): t = time.strftime('%Y-%m-%d', time.localtime()) MYSQL = Mysql.MYSQL(USER, PASSWORD, HOST, PORT, DB) def Run_sql(val): id, IP, PORT, DB = val[:4] CMD = val[5] val = Mysql.Query_sql(IP, PORT, DB, CMD) if val: val = str(val).replace("'", '') else: val = 'None' cmd = "update sql_scheduler set status = '已执行' ,results = '%s' where id = '%s';" % ( val, id) loging.write(cmd, log_path=log_path) MYSQL.Run(cmd) MYSQL.Close() try: cmd = "select * from sql_scheduler where status = '未执行' and time = '%s';" % t values = MYSQL.Run(cmd) MYSQL.Close() if values: # 并发执行 POOLS = Third_pool(10) POOLS.map_async(Run_sql, values) POOLS.close() POOLS.join() except Exception as e: loging.write(e, log_path=log_path)
def main(): p = ThreadPool() p.map_async(test, list(range(5))) p.close() p.join() print(f"{current_process().name}")
def handle_download_picture(self): '''这是下载图片的主函数''' print("进入图片下载程序...") picture_list = [] conn = sqlite3.connect('iwatu.db') cursor = conn.cursor() sql = 'select dlink,title,referer from dlink where log=0' picture_list = cursor.execute(sql).fetchall() if cursor: cursor.close() if conn: conn.commit() conn.close() if len(picture_list) != 0: print("下载任务进行中,请稍后...") pool = ThreadPool(self.ThreadPool) ''' for each_tuple in picture_list: print("添加新任务到子进程(父Pid:%s):%s" % (os.getpid(),each_tuple[0])) #根据sql的查询内容来看,这个list应该分别是0放dlink,1放title,2放referer p.apply_async(self.multi_handle_download_picture, args=(each_tuple,)) ''' pool.map_async(self.multi_handle_download_picture, picture_list) pool.close() pool.join() print("All Done!") else: print("没有搜索到需要下载的图片连接记录")
def handle_subjectpage(self, single_subjectpage=None): '''这是主题页处理函数,负责把每一个图片的地址分析出来''' print("进入主题页处理程序...") targeturl = [] #先判断参数那里有没有任务传进来,如果有,先加上 if single_subjectpage != None: targeturl.append(single_subjectpage) #下面把数据库里未处理过的主题也一并放进来去处理 conn = sqlite3.connect('iwatu.db') cursor = conn.cursor() result = cursor.execute( 'select * from urls where log=0 and competence=1 and domain=?', (self.domain, )).fetchall() old_exists_list = [] for each_tuple in result: old_exists_list.append(each_tuple[0]) targeturl.extend( old_exists_list ) #extend会自动把来源按照“一定的系统内部规律”打散再加入:列表会按照“,”打散成字符串;字符串则会打散成字母 if cursor: cursor.close() if conn: conn.commit() conn.close() if len(targeturl) > 0: #创建进程池正式开始处理啦 print("分析任务进行中,请稍后...") pool = ThreadPool(self.ThreadPool) pool.map_async(self.multi_handle_subjectpage, targeturl) pool.close() pool.join() print("All Done!") else: print("没有主题页需要处理")
def handle_pagelist(self, targeturl, x=None, y=0): '''这是列表页处理函数,负责把主题页的地址分析出来然后再提交给主题页处理函数去处理''' print("进入列表页处理程序...") list_urls = [] #这是用来放当前要处理的连接的容器 #判断任务是单一列表页还是多个列表页 if targeturl.find('*') != -1: if x == None: x = 1 print("你指定了任务为多页面模式,但是页数未指定,程序采用默认值:'1'") urlhandler = targeturl.split('*') #以*为标记截断原始URL starturl = urlhandler[0] + str(x) + urlhandler[1] #初始化任务的第一个列表页地址 list_urls.append(starturl) #并且放入页面链接的list while x <= y: #用循环把用户指定的连接创建出来并且一一放入list里面 x = x + 1 mission_url = urlhandler[0] + str(x) + urlhandler[1] list_urls.append(mission_url) #循环生成链接地址并且放入list里面 else: list_urls.extend(targeturl) #如果找不到'*'那么就是单一页面的任务而已,直接放入去就可以了 if len(list_urls) > 0: print("分析任务进行中,请稍后...") pool = ThreadPool(self.ThreadPool) pool.map_async(self.multi_handle_pagelist, list_urls) pool.close() pool.join() #调用方法要加‘()’啊你妹! print("All Done!") else: print("没有列表页需要处理")
def __start(ipList, domainList, threads): this.ipList = ipList this.domainList = domainList this.logFile = open("log/load"+str(time.time())+".log","w") this.end = len(this.ipList) this.cur = 0 this.packets = {} fileName = 'conf/mdb.txt' with open(fileName) as lines: this.creds = [] for line in lines: line = line.replace('\n', '') this.creds.append(line) pool = ThreadPool(int(threads)) try: if this.mode == "dns": pool.map_async(__threadDNS, this.ipList).get(9999999) else: pool.map_async(__threadOther, this.ipList).get(9999999) except KeyboardInterrupt: print "\n[!] we killed killed all python processes for you" os.system('pkill python') sys.exit(0) pool.close() pool.join() this.logFile.close()
def thread_start(self, thread_run, args): if len(args) > 0: self.preAction(args) pool = Pool(self.cpu) pool.map_async(thread_run, args) pool.close() pool.join()
def test_concurrent_writes(self): sender_pool = ThreadPool(100) receiver_pool = ThreadPool(100) message = can.Message( arbitration_id=0x123, channel=self.CHANNEL_1, is_extended_id=True, timestamp=121334.365, data=[254, 255, 1, 2] ) workload = 1000 * [message] def sender(msg): self.bus1.send(msg) def receiver(_): return self.bus2.recv(timeout=2.0) sender_pool.map_async(sender, workload) for msg in receiver_pool.map(receiver, len(workload) * [None]): self.assertIsNotNone(msg) self.assertEqual(message.arbitration_id, msg.arbitration_id) self.assertTrue(message.equals(msg, timestamp_delta=None)) sender_pool.close() sender_pool.join() receiver_pool.close() receiver_pool.join()
def getvasprun(self, files): m = self.m maindir = pwd() if m.engine == "vasp": calculator = self.getVaspRun_vasp elif m.engine == "lammps": calculator = self.getVaspRun_lammps self.jm = jobManager() for file in files: print(file) dir = "dirs/dir_" + file mkdir(dir) mv(file, dir + '/POSCAR') cd(dir) calculator() cd(maindir) self.jm.run() if m.th: mkdir(m.pbsname) self.thcode(files, m.pbsname) cp("dirs", m.pbsname) passthru("tar zcf %s.tar.gz %s" % (m.pbsname, m.pbsname)) print('start check') self.jm.check() if m.engine == "lammps1": from multiprocessing.dummy import Pool pool = Pool() pool.map_async(lammpsvasprun, files) pool.close() pool.join()
def main(): p = ThreadPool() p.map_async(test, list(range(5))) p.close() p.join() print(num)
def mutil_brute(self, _data_list: list): pool = Pool(20) pool.map_async(ServiceBrute.brute_main, _data_list, callback=self.CallBackFunc) pool.close() pool.join()
def Get_values(hosts, Redis_Key, keys, history, preset, operation): RC.delete(Redis_Key) hosts = [str(host[0]) for host in hosts] method = 'history.get' RC.set('%s_time' % Redis_Key, time.strftime('%Y-%m-%d %H:%M', time.localtime())) def Run(host): try: values = {} for key in keys: value = zabbix_api.GET_value(host, keys[key], method, history) if value != None: if '.' in value: value = float(value) if operation == 'lt': if int(value) < preset: values[key] = value RC.hset(Redis_Key, host, values) if operation == 'gt': if int(value) > preset: values[key] = value RC.hset(Redis_Key, host, values) except Exception as e: logging.error(e) pool = ThreadPool(4) pool.map_async(Run, hosts) pool.close() pool.join()
def test_concurrent_writes(self): sender_pool = ThreadPool(100) receiver_pool = ThreadPool(100) message = can.Message(arbitration_id=0x123, extended_id=True, timestamp=121334.365, data=[254, 255, 1, 2]) workload = 1000 * [message] def sender(msg): self.bus1.send(msg) def receiver(_): result = self.bus2.recv(timeout=2.0) self.assertIsNotNone(result) self.assertEqual(result, message) sender_pool.map_async(sender, workload) receiver_pool.map_async(receiver, len(workload) * [None]) sender_pool.close() sender_pool.join() receiver_pool.close() receiver_pool.join()
def test_concurrent_queries(started_cluster): conn = get_postgres_conn(started_cluster, started_cluster.postgres_ip, True) cursor = conn.cursor() node1.query(''' CREATE TABLE test_table (key UInt32, value UInt32) ENGINE = PostgreSQL('postgres1:5432', 'clickhouse', 'test_table', 'postgres', 'mysecretpassword')''' ) cursor.execute('CREATE TABLE test_table (key integer, value integer)') prev_count = node1.count_in_log('New connection to postgres1:5432') def node_select(_): for i in range(20): result = node1.query("SELECT * FROM test_table", user='******') busy_pool = Pool(20) p = busy_pool.map_async(node_select, range(20)) p.wait() count = node1.count_in_log('New connection to postgres1:5432') print(count, prev_count) # 16 is default size for connection pool assert (int(count) <= int(prev_count) + 16) def node_insert(_): for i in range(5): result = node1.query( "INSERT INTO test_table SELECT number, number FROM numbers(1000)", user='******') busy_pool = Pool(5) p = busy_pool.map_async(node_insert, range(5)) p.wait() result = node1.query("SELECT count() FROM test_table", user='******') print(result) assert (int(result) == 5 * 5 * 1000) def node_insert_select(_): for i in range(5): result = node1.query( "INSERT INTO test_table SELECT number, number FROM numbers(1000)", user='******') result = node1.query("SELECT * FROM test_table LIMIT 100", user='******') busy_pool = Pool(5) p = busy_pool.map_async(node_insert_select, range(5)) p.wait() result = node1.query("SELECT count() FROM test_table", user='******') print(result) assert (int(result) == 5 * 5 * 1000 * 2) node1.query('DROP TABLE test_table;') cursor.execute('DROP TABLE test_table;') count = node1.count_in_log('New connection to postgres1:5432') print(count, prev_count) assert (int(count) <= int(prev_count) + 16)
def _start(self): try: print '-' * 60 print u'{}[-] : {}{} '.format(self.O, socket.gethostbyname(self.target), self.W) print '-' * 60 # 线程数 pool = ThreadPool(processes=100) # get传递超时时间,用于捕捉ctrl+c pool.map_async(self.run, self.ports).get(0xffff) pool.close() pool.join() fo = open("./file/result/Port_scan.txt","w") for i in list_res: fo.write(i) fo.write('\n') fo.close() print '-' * 60 print u'{}[-] : {} .{}'.format(self.O, time() - self.time, self.W) except Exception as e: print e except KeyboardInterrupt: print self.R + u'\n[-] ...' sys.exit(1)
def parallel_run(func, params, count=0, thread=False, monitor_func=None): from multiprocessing import cpu_count from multiprocessing import Pool if count == 0: count = cpu_count() if count >= 4: count = count * 2 if thread: from multiprocessing.dummy import Pool as ThreadPool pool = ThreadPool(count) else: pool = Pool(count) sync = True if monitor_func is not None: sync = False ret = [] if sync: ret = pool.map(func, params) else: def over_callback(_rets): params.clear() for _ret in _rets: ret.append(_ret) pool.map_async(func, params, callback=over_callback) import time time.sleep(0.5) if monitor_func is not None: monitor_func(params) pool.close() pool.join() return ret
def test_concurrent_queries(started_cluster): conn = get_postgres_conn(started_cluster.postgres_ip, started_cluster.postgres_port, database=False) cursor = conn.cursor() database_name = 'concurrent_test' cursor.execute(f'DROP DATABASE IF EXISTS {database_name}') cursor.execute(f'CREATE DATABASE {database_name}') conn = get_postgres_conn(started_cluster.postgres_ip, started_cluster.postgres_port, database=True, database_name=database_name) cursor = conn.cursor() cursor.execute('CREATE TABLE test_table (key integer, value integer)') node1.query(f''' CREATE TABLE test.test_table (key UInt32, value UInt32) ENGINE = PostgreSQL(postgres1, database='{database_name}', table='test_table') ''') node1.query(f''' CREATE TABLE test.stat (numbackends UInt32, datname String) ENGINE = PostgreSQL(postgres1, database='{database_name}', table='pg_stat_database') ''') def node_select(_): for i in range(20): result = node1.query("SELECT * FROM test.test_table", user='******') def node_insert(_): for i in range(20): result = node1.query("INSERT INTO test.test_table SELECT number, number FROM numbers(1000)", user='******') def node_insert_select(_): for i in range(20): result = node1.query("INSERT INTO test.test_table SELECT number, number FROM numbers(1000)", user='******') result = node1.query("SELECT * FROM test.test_table LIMIT 100", user='******') busy_pool = Pool(30) p = busy_pool.map_async(node_select, range(30)) p.wait() count = int(node1.query(f"SELECT numbackends FROM test.stat WHERE datname = '{database_name}'")) print(count) assert(count <= 18) busy_pool = Pool(30) p = busy_pool.map_async(node_insert, range(30)) p.wait() count = int(node1.query(f"SELECT numbackends FROM test.stat WHERE datname = '{database_name}'")) print(count) assert(count <= 18) busy_pool = Pool(30) p = busy_pool.map_async(node_insert_select, range(30)) p.wait() count = int(node1.query(f"SELECT numbackends FROM test.stat WHERE datname = '{database_name}'")) print(count) assert(count <= 18) node1.query('DROP TABLE test.test_table;') node1.query('DROP TABLE test.stat;')
class MyPool: def __init__(self, n=5): self.queue = Queue() self.pool = Pool(n) self.n = n def start(self): self.pool.map_async(self.run, range(self.n)) def stop(self): self.pool.close() self.pool.join() def run(self, i): while True: try: req = self.queue.get(timeout=1) self.process(req, i) self.queue.task_done() except Empty: pass except: traceback.print_exc() self.queue.task_done() @property def count(self): return self.queue.unfinished_tasks def put(self, req): self.queue.put(req) def process(self, req, i): pass
def update_advalue(update=False): ADValueTable.metadata.create_all(db) KDJValueTable.metadata.create_all(db) # idlist = sc.get_sza_index(index=0) # idlist.extend(sc.get_sha_index(index=0)) idlist = sc.get_china_stock_list(index=0) # idlist = ['600777'] # lock = threading.Lock() # for num in idlist: # while threading.activeCount() >= 10: # time.sleep(10) # th = ADValueThread(lock, num, update) # th.setDaemon(True) # th.start() # for t in threading.enumerate(): # try: # t.join() # except Exception as ex: # logger.debug('%s' % str(ex)) # continue lock = multiprocessing.Lock() tp = ThreadPool(10) # lock = multiprocessing.Manager().Lock() # tp = multiprocessing.Pool() tp.map_async(advalueprocess, [(lock, num, update) for num in idlist]).get(36000) tp.close() tp.join()
def run_steps(self, steps=1): self.steps = steps pool = ThreadPool() pool.map_async(self._run, self.planners) pool.close() pool.join()
def run_steps(self, steps=1): self.steps = steps pool = ThreadPool() pool.map_async(self._run, self.planners) pool.close() pool.join()
def test4(): for n in range(100000): def test5(i): n += i tpool = TheaderPool(processes=1) tpool.map_async(test5,range(100000)) tpool.close() tpool.join()
def main(): p = ThreadPool() # 两个生产者 p.map_async(Shop, range(2)) # 五个消费者 p.map_async(User, range(5)) p.close() p.join()
def BuilTable(): """ Construit la table. Utilise du multithreading """ print("Construction de la table en cours") global alphabet global table701 global temp global alphabet global doc temp = [[]] * 701 table701 = [deepcopy(temp) for i in range(0, 359)] f = open("word2.txt") temp = f.readlines() f.close() pool = ThreadPool(16) a = time.time() doc = [] pool.map(constrDoc, temp) pool.close() pool.join() print(time.time() - a) print("Mots recuperes du fichier") pool = ThreadPool(16) alphabet = [] pool.map(constrAlphabet, doc) pool.close() pool.join() a = time.time() alphabet = uniq(alphabet) print(time.time() - a) print("Alphabet construit") a = time.time() pool1 = ThreadPool(16) pool1.map_async(constrTable, doc) pool1.close() pool1.join() print(time.time() - a) print("Table construite")
def execute_moves(self, moves): def act(args): args[0](*args[1:]) pool = ThreadPool(len(moves)) pool.map_async(act, moves) pool.close() pool.join()
def main(): from multiprocessing.dummy import Pool os.chdir(os.path.join(os.path.dirname(__file__), os.pardir)) pool = Pool() tasklist = parse_emakefile('Emakefile') pool.map_async(do_make, tasklist).get(0xfff) pool.close() pool.join() print '±àÒë³É¹¦£¡'
def execute_moves(self, moves): def act(args): args[0](*args[1:]) pool = ThreadPool(len(moves)) pool.map_async(act, moves) pool.close() pool.join()
def test4(): for n in range(1000000): def test5(i): n += i tpool = TheaderPool(processes=1) tpool.map_async(test5, range(1000000)) tpool.close() tpool.join()
def get_taobao_seller(keywords): # 爬取指定数量的店铺信息 def get_seller_from_num(nums): url = "https://shopsearch.taobao.com/search?data-key=s&data-value={0}&" \ "ajax=true&_ksTS=1481770098290_1972&callback=jsonp602&app=shopsearch&q={1}&js=1&isb=0".format( nums, keywords) # url = "https://shopsearch.taobao.com/search?data-key=s&data-value={0}&ajax=true&callback=jsonp602&app=shopsearch&q={1}".format(nums,keywords) content = requests.get(url) wbdata = content.text[11:-2] data = json.loads(wbdata) shop_list = data['mods']['shoplist']['data']['shopItems'] for s in shop_list: name = s['title'] # 店铺名 nick = s['nick'] # 卖家昵称 nid = s['nid'] # 店铺ID provcity = s['provcity'] # 店铺区域 shopUrl = s['shopUrl'] # 店铺链接 totalsold = s['totalsold'] # 店铺宝贝数量 procnt = s['procnt'] # 店铺销量 startFee = s['startFee'] # 未知 mainAuction = s['mainAuction'] # 店铺关键词 userRateUrl = s['userRateUrl'] # 用户评分链接 dsr = json.loads(s['dsrInfo']['dsrStr']) goodratePercent = dsr['sgr'] # 店铺好评率 srn = dsr['srn'] # 店铺等级 category = dsr['ind'] # 店铺分类 mas = dsr['mas'] # 描述相符 sas = dsr['sas'] # 服务态度 cas = dsr['cas'] # 物流速度 data = { 'name': name, 'nick': nick, 'nid': nid, 'provcity': provcity, 'shopUrl': shopUrl, 'totalsold': totalsold, 'procnt': procnt, 'startFee': startFee, 'goodratePercent': goodratePercent, # 'mainAuction':mainAuction, 'userRateUrl': userRateUrl, 'srn': srn, 'category': category, 'mas': mas, 'sas': sas, 'cas': cas } print(data) print("插入数据成功") pool = TheaderPool(processes=1) pool.map_async(get_seller_from_num, range(0, 10020, 20)) pool.close() pool.join()
def run(self): try: pool = ThreadPool(processes=self.thread) pool.map_async(self.start, self.ips).get(0xffff) pool.close() pool.join() except Exception as e: pass except KeyboardInterrupt: print u'\n{}[-] 用户终止扫描...{}'.format(R, W) sys.exit(1) finally: print '-'*55 print u'{}[+] 扫描完成耗时 {} 秒.{}'.format(O, time.time()-self.time, W)
def main(self): self.s = self.login() self.keyword = input('请输入要搜的图的关键字:') #raw_input(u'请输入...')输入中文? print(self.keyword) rootPath = 'E:\ProgramCode\Python\pixivStar\\'+self.keyword if not os.path.exists(rootPath): os.mkdir(rootPath) #TODO 自定义路径 #TODO arg参数,收藏数 #get_search(keyword) searchUrl = self.index_url + '/search.php?s_mode=s_tag&word='+self.keyword+'&order=date_d&p='+str(1) respo = self.s.get(searchUrl, headers=self.header) content1 = respo.content.decode('utf-8') '''with open('contentExample.txt', 'w', encoding='utf-8') as f: print(content1,f)''' descriPat = re.compile('<meta name="description" content="(.*?)">', re.S) descri = re.findall(descriPat, content1)[0] print(descri) amountPat = re.compile('count-badge">(.*?)\u4ef6</span>', re.S) amount = int(re.findall(amountPat, content1)[0]) // 20 print('页数:'+str(amount)) '''self.thumbHeader = {'Host':'i2.pixiv.net', #host i1 ~ i4 'If-Modified-Since':'Thu, 11 Aug 2016 01:15:38 GMT', 'Referer':searchUrl, 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36}'} signCheckPat = re.compile('signup-sina-button"(.*?)</button>') sign = re.findall(signCheckPat, content1) print(len(sign))''' '''signCheckPat2 = re.compile('bookmarks"(.*?)</li>') sign2 = re.findall(signCheckPat2, content1) print(len(sign2))''' #TODO 两种搜索模式 #'s_mode=' 's_tag' #tag #'s_tc' #标题和介绍 pool = ThreadPool(8) urls = [] for page in range(1, amount): urls.append(self.index_url + '/search.php?word='+self.keyword+'&order=date_d&p='+str(page)) #self.get_search(self.index_url + 'search.php?s_mode=s_tag&word='+self.keyword+'&order=date_d&p='+str(page)) pool.map_async(self.get_search, urls) pool.close() pool.join() print('--mission complete--') #sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8') '''fileName = 'E:\ProgramCode\Python\searchPage.txt'
def multiRunuser(): pool = ThreadPool(cpu_count() * 8) global ip_list global results results = pool.map_async(runuser, ip_list) pool.close() pool.join()
class BaseLogger: def __init__(self, outfile ='baselogger.log', sockets=[]): """ A basic logging system. :param outfile: The file the logging output will be directed to. :param sockets: A list of external sockets who wish to be notified """ self.outfile = 'baselogger.out' self.fh = open(outfile, 'a') self.sockets = sockets self.level_text = ['INFO', 'WARNING', 'ERROR'] self.pool = Pool(2 if self.sockets else 1) def Log(self, message, level=0): """ Fires off a job to write log information to file and the listening sockets. """ timestamp = strftime("%Y-%m-%d %H:%M:%S") format_message = "# {0} [{1}]\t: {2}".format(timestamp, self.level_text[level], message) if level == 1: color = Color.YELLOW elif level == 2: color = Color.RED else: color = Color.GREEN sys.stdout.write(color + format_message + Color.END + "\n") self.pool.map_async(_log_fs, [(self.fh, format_message)] ) self.pool.map_async(_log_socket, [(s, format_message) for s in self.sockets]) def destroy(self): """ Clean up resources. Closes all open files and sockets. """ self.Log("Destroying logging system.", level=1) self.pool.close() # Continue until all operations have finished self.fh.close() for s in self.sockets: s.close() self.pool.close()
def main(): #TODO clean Logs parser = argparse.ArgumentParser(prog='scan.py', description='Pentesting Scan') parser.add_argument('-t', dest='TARGET', required=True, help='192.168.1.0/24 or 192.168.1.0-128') #required parser.add_argument('-d', dest='DIR', default='results', help='specifiy output-directory')#optional args = parser.parse_args() global logdir logdir = args.DIR checkEnv(args.DIR) global targets targets = netscan(args.TARGET) global db db=dict() portscanPool = Pool(4) portscanResults = portscanPool.map_async(portscan, targets) portscanPool.close() portscanPool.join() ipServiceList= [] for ip in db.keys(): if len(db[ip][1]) == 0 and len(db[ip][0]) == 0: print "[INFO ] no open ports in %s, delete key and remove dir" % ip del db[ip] removeEmptyEnv(ip) else: for s in range(len(db[ip][1])): ipServiceList.append([ip, s]) sericePool = Pool(4) serviceResults = sericePool.map_async(checkService, ipServiceList) sericePool.close() sericePool.join() print "[ END ] "+"="*35+" [ END ]"
class IoTaskThreadPool(object): ''' thread pool for io operations ''' def __init__(self, poolsize): self.ioPool = ThreadPool(poolsize) def execTasks(self, ioFunc, ioParams): if not ioParams or len(ioParams) == 0: return [] return self.ioPool.map(ioFunc, ioParams) def execTasksAsync(self, ioFunc, ioParams): if not ioParams or len(ioParams) == 0: return [] self.ioPool.map_async(ioFunc, ioParams) def close(self): self.ioPool.close() def join(self): self.ioPool.join()
def update_signal(update=False): SignalTable.metadata.create_all(db) id_list = sc.get_sha_index(index=0) id_list.extend(sc.get_sza_index(index=0)) # lock = threading.Lock() # for num in id_list: # while threading.activeCount() >= 10: # time.sleep(10) # th = SignalThread(lock, num, db, update) # th.setDaemon(True) # th.start() # for t in threading.enumerate(): # try: # t.join() # except Exception as ex: # logger.debug('%s' % str(ex)) # continue lock = multiprocessing.Lock() tp = ThreadPool(10) tp.map_async(signalprocess, [(lock, num, update) for num in id_list]).get(36000) tp.close() tp.join()
def _start_nodes_parallel(self, nodes, max_thread_pool_size): """ Start the nodes using a pool of multiprocessing threads for speed-up. Return set of nodes that were actually started. """ # Create one thread for each node to start thread_pool_size = min(len(nodes), max_thread_pool_size) thread_pool = Pool(processes=thread_pool_size) log.debug("Created pool of %d threads", thread_pool_size) # pressing Ctrl+C flips this flag, which in turn stops the main loop # down below keep_running = True def sigint_handler(signal, frame): """ Makes sure the cluster is saved, before the sigint results in exiting during node startup. """ log.error( "Interrupted: will save cluster state and exit" " after all nodes have started.") keep_running = False # intercept Ctrl+C with sighandler(signal.SIGINT, sigint_handler): result = thread_pool.map_async(self._start_node, nodes) while not result.ready(): result.wait(1) # check if Ctrl+C was pressed if not keep_running: log.error("Aborting upon user interruption ...") # FIXME: `.close()` will keep the pool running until all # nodes have been started; should we use `.terminate()` # instead to interrupt node creation as soon as possible? thread_pool.close() thread_pool.join() self.repository.save_or_update(self) # FIXME: should raise an exception instead! sys.exit(1) # keep only nodes that were successfully started return set(node for node, ok in itertools.izip(nodes, result.get()) if ok)
class Collector(Singleton): def __init__(self): self.redis = RedisOperation() self.redis.connect() self.pool = ThreadPool(10) self.pid = os.getpid() self.pname = multiprocessing.current_process().name def dispatch_work(self, task_data): group = task_data.get("group") urls = task_data.get("urls") if 0 == len(urls): LOG.warn("group(%s)'s task list is empty.'" % group) return LOG.info("pname(%s) pid(%s) receive and do task: %r." % (self.pname, self.pid, task_data)) map_obj = self.pool.map_async(self.pull_work, urls) ret_list = map_obj.get() LOG.info("pname(%s) pid(%s) finished group(%s) pull task." %(self.pname, self.pid, group)) self.finish_task(group, ret_list) def pull_work(self, task_url): pw = PullWorker(task_url) data = pw.run() return data def finish_task(self, group, ret_list): """ 1. compute metric 2. send metric 3. redis del group task for task state manage """ task_data = self.redis.get_redis_cache(group) LOG.info("pname(%s) pid(%s) start push metric." %(self.pname, self.pid)) pm = PushMetric(ret_list, task_data) pm.run() LOG.info("pname(%s) pid(%s) push metric finished." %(self.pname, self.pid)) self.redis.delete(group) LOG.info("pname(%s) pid(%s) task state manage del " "task(%s) finished." %(self.pname, self.pid, group))
def last_news(self, game_list): """ Get the last news for every game in your owned games list. Sort by ascending date. Takes your owned games list as a parameter. """ # Building appid list to feed Pool appid = [game['appid'] for game in game_list['response']['games']] # Keeping name and icon_url from owned game list metadata = [] for game in game_list['response']['games']: icon_url = ("http://media.steampowered.com/steamcommunity/" "public/images/apps/" + str(game['appid']) + "/" + str(game['img_logo_url']) + ".jpg") metadata.append({'name' : game['name'], 'appid' : game['appid'], 'icon_url' : icon_url}) # Multiprocessing news = [] pool = ThreadPool(8) results = pool.map_async(self.get_news, appid, callback=news.extend) results.wait() pool.close() pool.join() # Building output data = [] for x,y in zip(news, metadata): if x['appnews']['appid'] == y['appid']: name = y['name'] icon_url = y['icon_url'] content = x['appnews']['newsitems'] for z in content: news_date = datetime.datetime.fromtimestamp(int(z['date'])) if self.now - news_date < datetime.timedelta(weeks=4): data.append({'title': z['title'], 'date': news_date.strftime('%Y-%m-%d'), 'name': name, 'timestamp': z['date'], 'content': z['contents'], 'url': z['url'], 'icon_url': icon_url}) return sorted(data, key=itemgetter('timestamp'))
def geoconv_bd(geolist): from multiprocessing.dummy import Pool as ThreadPool max_geo = 50 pool = ThreadPool(5) grp = [geolist[i:i+max_geo] for i in xrange(0,len(geolist),max_geo)] result = pool.map_async(_conv, grp) pool.close() pool.join() ret = [] for elem in result.get(): ret.extend(elem) print '%d geo converted.' % (len(ret)) return ret
def fetch_and_extract(gsm_list): start = time.time() tmp_pathData = os.path.join(os.path.expanduser("~%s" % username), "AppData", "Local", "Temp", "GSM_extractData") # Max imap concurrent connections for Office365 : 20 thread_pool = ThreadPool(max_connections, initialize_connections) zip_files = thread_pool.map_async(fetch_zip_file, gsm_list).get() # Close connections while not connection_queue.empty(): cn = connection_queue.get() cn.close() cn.logout() # Extracting csv files from .zip print("Extracting files ...") process_pool = ProcessPool() process_pool.map(unzip, zip_files) print("Downloads and extraction successfully completed in {0}".format(timedelta(seconds=time.time() - start)))
def parallel_read(file_names, thread_count): logger.debug('start active_threads: %s', threading.activeCount()) params = [(index, file_name) for index, file_name in enumerate(file_names)] pool = ThreadPool(thread_count) logger.debug('create active_threads: %s', threading.activeCount()) try: async_results = pool.map_async( file_handling.read_file_wrapper, params, chunksize=1) results = async_results.get(10) except multiprocessing.TimeoutError: logger.error('missing results') return [] except: logger.exception('underlying thread error') return [] finally: pool.close() pool.join() logger.debug('end active_threads: %s', threading.activeCount()) return results
def _fetch_metadata(self): iprogress = cycle(ICHARS) def progress(): sys.stderr.write(CLEAR + 'Fetching metadata %s' % next(iprogress)) sys.stderr.flush() pool = Pool(POOL_SIZE) todo = [self.start_date - timedelta(days=i) for i in range(self.days)] metadata = pool.map_async(self.get_day_metadata, todo) while not metadata.ready(): progress() sleep(.1) sys.stderr.write(CLEAR + 'Fetching metadata [ * ]\n') sys.stderr.flush() return metadata.get()
def start(self): global type_value, threads_value, cid, aid pool = ThreadPool(int(threads_value.get())) if (type_value.get() == 'Подписчиков'): print('followers start') aid = getaccountid(link_value.get()) pool.map_async(follow_n, loginlist) if (type_value.get() == 'Лайков'): print('likes start') cid = getphotoid(link_value.get()) pool.map_async(like_n, loginlist) if (type_value.get() == 'Комментариев'): print('comments start') cid = getphotoid(link_value.get()) pool.map_async(comment_n, loginlist)
def start(self): self.blacklist.load(self.datadir) print 'Will fetch %d photo(s) per day for the last %d days' % \ (self.per_day, self.days) to_fetch = self.get_photos() # XXX optim listdir to_fetch = [photo for photo in to_fetch if not os.path.exists(photo.path)] print '%d photos to fetch' % len(to_fetch) def progress(): count = len([_ for _ in to_fetch if _.done]) dd = '%%%dd' % len(str(len(to_fetch))) template = 'Fetching %s/%s photos' % (dd, dd) sys.stderr.write('\r' + template % (count, len(to_fetch))) sys.stderr.flush() pool = Pool(POOL_SIZE) res = pool.map_async(self.store_photo, to_fetch) while not res.ready(): progress() sleep(.1) progress() print >> sys.stderr, '' self.blacklist.save() if not res.successful(): res.get() self.remove_expired()
'https://wiki.python.org/moin/LocalUserGroups', 'http://www.python.org/psf/', 'http://docs.python.org/devguide/', 'http://www.python.org/community/awards/' ] #高io同步用线程池 from multiprocessing.dummy import Pool as ThreadPool pool = ThreadPool() results = pool.map(urllib2.urlopen, urls) pool.close() pool.join() def test(url): res = urllib2.urlopen(url) return res.read() #高io同步用线程池 from multiprocessing.dummy import Pool as ThreadPool import multiprocessing pool = ThreadPool(multiprocessing.cpu_count()) results = pool.map_async(test, urls) pool.close() pool.join() print results.wait(timeout=10) print results.successful() print results.ready() for item in results.get(timeout=10): print item
class TradeApi(object): """交易API""" HUOBI = 'huobi' HADAX = 'hadax' SYNC_MODE = 'sync' ASYNC_MODE = 'async' #---------------------------------------------------------------------- def __init__(self): """Constructor""" self.accessKey = '' self.secretKey = '' self.mode = self.ASYNC_MODE self.active = False # API工作状态 self.reqid = 0 # 请求编号 self.queue = Queue() # 请求队列 self.pool = None # 线程池 #---------------------------------------------------------------------- def init(self, host, accessKey, secretKey, mode=None): """初始化""" if host == self.HUOBI: self.hostname = HUOBI_API_HOST else: self.hostname = HADAX_API_HOST self.hosturl = 'https://%s' %self.hostname self.accessKey = accessKey self.secretKey = secretKey if mode: self.mode = mode self.proxies = {} return True #---------------------------------------------------------------------- def start(self, n=10): """启动""" self.active = True if self.mode == self.ASYNC_MODE: self.pool = Pool(n) self.pool.map_async(self.run, range(n)) #---------------------------------------------------------------------- def close(self): """停止""" self.active = False self.pool.close() self.pool.join() #---------------------------------------------------------------------- def httpGet(self, url, params): """HTTP GET""" headers = copy(DEFAULT_GET_HEADERS) postdata = urllib.urlencode(params) try: response = requests.get(url, postdata, headers=headers, timeout=TIMEOUT) if response.status_code == 200: return True, response.json() else: return False, u'GET请求失败,状态代码:%s' %response.status_code except Exception as e: return False, u'GET请求触发异常,原因:%s' %e #---------------------------------------------------------------------- def httpPost(self, url, params, add_to_headers=None): """HTTP POST""" headers = copy(DEFAULT_POST_HEADERS) postdata = json.dumps(params) try: response = requests.post(url, postdata, headers=headers, timeout=TIMEOUT) if response.status_code == 200: return True, response.json() else: return False, u'POST请求失败,返回信息:%s' %response.json() except Exception as e: return False, u'POST请求触发异常,原因:%s' %e #---------------------------------------------------------------------- def generateSignParams(self): """生成签名参数""" timestamp = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') d = { 'AccessKeyId': self.accessKey, 'SignatureMethod': 'HmacSHA256', 'SignatureVersion': '2', 'Timestamp': timestamp } return d #---------------------------------------------------------------------- def apiGet(self, path, params): """API GET""" method = 'GET' params.update(self.generateSignParams()) params['Signature'] = createSign(params, method, self.hostname, path, self.secretKey) url = self.hosturl + path return self.httpGet(url, params) #---------------------------------------------------------------------- def apiPost(self, path, params): """API POST""" method = 'POST' signParams = self.generateSignParams() signParams['Signature'] = createSign(signParams, method, self.hostname, path, self.secretKey) url = self.hosturl + path + '?' + urllib.urlencode(signParams) return self.httpPost(url, params) #---------------------------------------------------------------------- def addReq(self, path, params, func, callback): """添加请求""" # 异步模式 if self.mode == self.ASYNC_MODE: self.reqid += 1 req = (path, params, func, callback, self.reqid) self.queue.put(req) return self.reqid # 同步模式 else: return func(path, params) #---------------------------------------------------------------------- def processReq(self, req): """处理请求""" path, params, func, callback, reqid = req result, data = func(path, params) if result: if data['status'] == 'ok': callback(data['data'], reqid) else: msg = u'错误代码:%s,错误信息:%s' %(data['err-code'], data['err-msg']) self.onError(msg, reqid) else: self.onError(data, reqid) # 失败的请求重新放回队列,等待下次处理 self.queue.put(req) #---------------------------------------------------------------------- def run(self, n): """连续运行""" while self.active: try: req = self.queue.get(timeout=1) self.processReq(req) except Empty: pass #---------------------------------------------------------------------- def getSymbols(self): """查询合约代码""" if self.hostname == HUOBI_API_HOST: path = '/v1/common/symbols' else: path = '/v1/hadax/common/symbols' params = {} func = self.apiGet callback = self.onGetSymbols return self.addReq(path, params, func, callback) #---------------------------------------------------------------------- def getCurrencys(self): """查询支持货币""" if self.hostname == HUOBI_API_HOST: path = '/v1/common/currencys' else: path = '/v1/hadax/common/currencys' params = {} func = self.apiGet callback = self.onGetCurrencys return self.addReq(path, params, func, callback) #---------------------------------------------------------------------- def getTimestamp(self): """查询系统时间""" path = '/v1/common/timestamp' params = {} func = self.apiGet callback = self.onGetTimestamp return self.addReq(path, params, func, callback) #---------------------------------------------------------------------- def getAccounts(self): """查询账户""" path = '/v1/account/accounts' params = {} func = self.apiGet callback = self.onGetAccounts return self.addReq(path, params, func, callback) #---------------------------------------------------------------------- def getAccountBalance(self, accountid): """查询余额""" if self.hostname == HUOBI_API_HOST: path = '/v1/account/accounts/%s/balance' %accountid else: path = '/v1/hadax/account/accounts/%s/balance' %accountid params = {} func = self.apiGet callback = self.onGetAccountBalance return self.addReq(path, params, func, callback) #---------------------------------------------------------------------- def getOrders(self, symbol, states, types=None, startDate=None, endDate=None, from_=None, direct=None, size=None): """查询委托""" path = '/v1/order/orders' params = { 'symbol': symbol, 'states': states } if types: params['types'] = types if startDate: params['start-date'] = startDate if endDate: params['end-date'] = endDate if from_: params['from'] = from_ if direct: params['direct'] = direct if size: params['size'] = size func = self.apiGet callback = self.onGetOrders return self.addReq(path, params, func, callback) #---------------------------------------------------------------------- def getMatchResults(self, symbol, types=None, startDate=None, endDate=None, from_=None, direct=None, size=None): """查询委托""" path = '/v1/order/matchresults' params = { 'symbol': symbol } if types: params['types'] = types if startDate: params['start-date'] = startDate if endDate: params['end-date'] = endDate if from_: params['from'] = from_ if direct: params['direct'] = direct if size: params['size'] = size func = self.apiGet callback = self.onGetMatchResults return self.addReq(path, params, func, callback) #---------------------------------------------------------------------- def getOrder(self, orderid): """查询某一委托""" path = '/v1/order/orders/%s' %orderid params = {} func = self.apiGet callback = self.onGetOrder return self.addReq(path, params, func, callback) #---------------------------------------------------------------------- def getMatchResult(self, orderid): """查询某一委托""" path = '/v1/order/orders/%s/matchresults' %orderid params = {} func = self.apiGet callback = self.onGetMatchResult return self.addReq(path, params, func, callback) #---------------------------------------------------------------------- def placeOrder(self, accountid, amount, symbol, type_, price=None, source=None): """下单""" if self.hostname == HUOBI_API_HOST: path = '/v1/order/orders/place' else: path = '/v1/hadax/order/orders/place' params = { 'account-id': accountid, 'amount': amount, 'symbol': symbol, 'type': type_ } if price: params['price'] = price if source: params['source'] = source func = self.apiPost callback = self.onPlaceOrder return self.addReq(path, params, func, callback) #---------------------------------------------------------------------- def cancelOrder(self, orderid): """撤单""" path = '/v1/order/orders/%s/submitcancel' %orderid params = {} func = self.apiPost callback = self.onCancelOrder return self.addReq(path, params, func, callback) #---------------------------------------------------------------------- def batchCancel(self, orderids): """批量撤单""" path = '/v1/order/orders/batchcancel' params = { 'order-ids': orderids } func = self.apiPost callback = self.onBatchCancel return self.addReq(path, params, func, callback) #---------------------------------------------------------------------- def onError(self, msg, reqid): """错误回调""" print msg, reqid #---------------------------------------------------------------------- def onGetSymbols(self, data, reqid): """查询代码回调""" #print reqid, data for d in data: print d #---------------------------------------------------------------------- def onGetCurrencys(self, data, reqid): """查询货币回调""" print reqid, data #---------------------------------------------------------------------- def onGetTimestamp(self, data, reqid): """查询时间回调""" print reqid, data #---------------------------------------------------------------------- def onGetAccounts(self, data, reqid): """查询账户回调""" print reqid, data #---------------------------------------------------------------------- def onGetAccountBalance(self, data, reqid): """查询余额回调""" print reqid, data for d in data['data']['list']: print d #---------------------------------------------------------------------- def onGetOrders(self, data, reqid): """查询委托回调""" print reqid, data #---------------------------------------------------------------------- def onGetMatchResults(self, data, reqid): """查询成交回调""" print reqid, data #---------------------------------------------------------------------- def onGetOrder(self, data, reqid): """查询单一委托回调""" print reqid, data #---------------------------------------------------------------------- def onGetMatchResult(self, data, reqid): """查询单一成交回调""" print reqid, data #---------------------------------------------------------------------- def onPlaceOrder(self, data, reqid): """委托回调""" print reqid, data #---------------------------------------------------------------------- def onCancelOrder(self, data, reqid): """撤单回调""" print reqid, data #---------------------------------------------------------------------- def onBatchCancel(self, data, reqid): """批量撤单回调""" print reqid, data
class LbankRestApi(object): """""" #---------------------------------------------------------------------- def __init__(self): """Constructor""" self.apiKey = '' self.secretKey = '' self.active = False # API工作状态 self.reqID = 0 # 请求编号 self.queue = Queue() # 请求队列 self.pool = None # 线程池 self.sessionDict = {} # 连接池 #---------------------------------------------------------------------- def init(self, apiKey, secretKey): """初始化""" self.apiKey = apiKey self.secretKey = secretKey #---------------------------------------------------------------------- def start(self, n=10): """""" if self.active: return self.active = True self.pool = Pool(n) self.pool.map_async(self.run, range(n)) #---------------------------------------------------------------------- def close(self): """退出""" self.active = False if self.pool: self.pool.close() self.pool.join() #---------------------------------------------------------------------- def processReq(self, req, i): """处理请求""" # 读取方法和参数 method, path, params, callback, reqID = req url = REST_HOST + path # 在参数中增加必须的字段 params['api_key'] = self.apiKey params['sign'] = self.generateSignature(params) # 发送请求 payload = urllib.urlencode(params) try: # 使用会话重用技术,请求延时降低80% session = self.sessionDict[i] resp = session.request(method, url, params=payload) #resp = requests.request(method, url, params=payload) code = resp.status_code d = resp.json() if code == 200: callback(d, reqID) else: self.onError(code, str(d)) except Exception as e: self.onError(type(e), e.message) #---------------------------------------------------------------------- def run(self, i): """连续运行""" self.sessionDict[i] = requests.Session() while self.active: try: req = self.queue.get(block=True, timeout=1) # 获取请求的阻塞为一秒 self.processReq(req, i) except Empty: pass #---------------------------------------------------------------------- def addReq(self, method, path, params, callback): """发送请求""" # 请求编号加1 self.reqID += 1 # 生成请求字典并放入队列中 req = (method, path, params, callback, self.reqID) self.queue.put(req) # 返回请求编号 return self.reqID #---------------------------------------------------------------------- def generateSignature(self, params): """生成签名""" params = sorted(params.iteritems(), key=lambda d:d[0], reverse=False) params.append(('secret_key', self.secretKey)) message = urllib.urlencode(params) m = hashlib.md5() m.update(message) m.digest() sig = m.hexdigest() return sig #---------------------------------------------------------------------- def onError(self, code, msg): """错误推送""" print(code, msg) #---------------------------------------------------------------------- def onData(self, data, reqID): """""" print(data, reqID)
multiProcessCount = 2 hostList=[] hostOsTimeList = [] NewOSFilterInterval = 1 #days # add the host waiting for check in hostList hostList.append('10.1.4.100') hostList.append('10.1.4.23') # print hostList cli = "stat /lost+found/ | grep Modify | awk -F ' ' {'print $2,$3,$4'};" cli += "exit $?" ## auto logout username='******' password='******' pool = Pool(processes=multiProcessCount) res=pool.map_async(pssh,((host,username,password,cli) for host in hostList)) result=res.get() import time import datetime import string for output in result: if output[1] and output[1] != '' : timeArr=output[1].split('\n')[1].split(' ') realTimeStruct = time.strptime(timeArr[0]+' '+timeArr[1].split('.')[0],'%Y-%m-%d %H:%M:%S') realTime = datetime.datetime(*realTimeStruct[:6]) osInstallTime_UTC = None utcDelta=string.atoi(timeArr[2][1:]) if '+' in timeArr[2]: osInstallTime_UTC = realTime + datetime.timedelta(hours=-1*(utcDelta/100)) elif '-' in timeArr[2]:
def query_Id_AuId(id1, auId2, json1, json2): #sys.stderr.write('query_AuId_Id ' + str(id1) + ' ' + str(auId2) + '\n') print 'query_Id_AuId', id1, auId2 ans = [] #now = time.time() #json1 = getPaperJson(id1, 'RId,F.FId,J.JId,C.CId,AA.AuId,AA.AfId') #print 'time use: ', time.time() - now #now = time.time() #url = 'https://oxfordhk.azure-api.net/academic/v1.0/evaluate?expr=Composite(AA.AuId=%d)&count=20000&attributes=Id,F.FId,J.JId,C.CId,AA.AuId&orderby=D:asc&subscription-key=f7cc29509a8443c5b3a5e56b0e38b5a6'%auId2 #json2 = json.loads((urllib.urlopen(url)).read())['entities'] # Prepare for Id-AA.AuId-AA.AfId-AuId AFIdSet2 = set() for paper in json2: if paper.has_key('AA'): for author in paper['AA']: if author['AuId'] == auId2 and author.has_key('AfId'): AFIdSet2.add(author['AfId']) if len(AFIdSet2) > 0: authorPaperListResult = [] pool = Pool(20) if json1.has_key('AA'): for author in json1['AA']: url = 'https://oxfordhk.azure-api.net/academic/v1.0/evaluate?expr=Composite(AA.AuId=%d)&count=3000&attributes=AA.AuId,AA.AfId&orderby=D:desc&subscription-key=f7cc29509a8443c5b3a5e56b0e38b5a6'%author['AuId'] authorPaperListResult.append(pool.apply_async(lambda url: json.loads((urllib.urlopen(url)).read())['entities'], (url,))) paperIdList = map(lambda x:x['Id'], json2) paperIdList.sort() # =========== 1-hop =========== # Id-AuId if json1.has_key('AA') and (auId2 in map(lambda x:x['AuId'], json1['AA'])): answer(ans, [id1, auId2]) # =========== 2-hop =========== # Id-Id-AuId if json1.has_key('RId'): RIdList = json1['RId'] RIdList.sort() for RId in join(paperIdList, RIdList): answer(ans, [id1, RId, auId2]) # =========== 3-hop =========== # Id-F.FId-Id-AuId if json1.has_key('F'): FIdList1 = map(lambda x:x['FId'], json1['F']) FIdList1.sort() for paper in json2: if paper.has_key('F'): FIdListTmp = map(lambda x:x['FId'], paper['F']) FIdListTmp.sort() for FId in join(FIdListTmp, FIdList1): answer(ans, [id1, FId, paper['Id'], auId2]) # Id-C.CId-Id-AuId if json1.has_key('C'): CId1 = json1['C']['CId'] for paper in json2: if paper.has_key('C') and paper['C']['CId'] == CId1: answer(ans, [id1, CId1, paper['Id'], auId2]) # Id-J.JId-Id-AuId if json1.has_key('J'): JId1 = json1['J']['JId'] for paper in json2: if paper.has_key('J') and paper['J']['JId'] == JId1: answer(ans, [id1, JId1, paper['Id'], auId2]) # Id-AA.AuId-Id-AuId if json1.has_key('AA'): AuIdList1 = map(lambda x:x['AuId'], json1['AA']) AuIdList1.sort() for paper in json2: if paper.has_key('AA'): AuIdListTmp = map(lambda x:x['AuId'], paper['AA']) AuIdListTmp.sort() for AuId in join(AuIdListTmp, AuIdList1): answer(ans, [id1, AuId, paper['Id'], auId2]) # Id-Id-Id-AuId if json1.has_key('RId'): pool = Pool(20) citePaperInfoResults = pool.map_async(lambda x:getPaperJson(x, 'RId'), RIdList) pool.close() #pool.join() citePaperInfos = citePaperInfoResults.get() for citePaperInfo in citePaperInfos: if citePaperInfo.has_key('RId'): RIdListTmp = citePaperInfo['RId'] RIdListTmp.sort() for comRId in join(RIdListTmp, paperIdList): answer(ans, [id1, RId, comRId, auId2]) # Id-AA.AuId-AA.AfId-AuId if len(AFIdSet2) > 0: uniqueSet = set() idx = 0 if json1.has_key('AA'): for author in json1['AA']: auId1 = author['AuId'] authorPaperList = authorPaperListResult[idx].get() idx += 1 for paper in authorPaperList: if paper.has_key('AA'): for author2 in paper['AA']: if author2['AuId'] == auId1 and author2.has_key('AfId') and (author2['AfId'] in AFIdSet2) and not((auId1, author2['AfId']) in uniqueSet): answer(ans, [id1, auId1, author2['AfId'], auId2]) uniqueSet.add((auId1, author2['AfId'])) #AFIdSet2 = set() #for paper in json2: # if paper.has_key('AA'): # for author in paper['AA']: # if author['AuId'] == auId2 and author.has_key('AfId'): # AFIdSet2.add(author['AfId']) #if json1.has_key('AA'): # for author in json1['AA']: # if author.has_key('AfId'): # if author['AfId'] in AFIdSet2: # answer(ans, [id1, author['AuId'], author['AfId'], auId2]) #print 'time use2: ', time.time() - now return ans