def execute(self, item_generator, item_processor): # Create a queue to communicate with the worker threads try: queue = Queue() threads = [] # Create 8 worker threads for x in range(self.no_of_threads): worker = Neo4jUploadWorker(queue, item_processor) # Setting daemon to True will let the main thread exit even though the workers are blocking worker.daemon = True worker.start() threads.append(worker) # Put the tasks into the queue as a tuple for item in item_generator.generate: queue.put(item) # Causes the main thread to wait for the queue to finish processing all the tasks queue.join() # stop workers for i in range(self.no_of_threads): queue.put(None) for t in threads: t.join() finally: print('In the finally') item_generator.close() for i in range(self.no_of_threads): queue.put(None) for t in threads: t.join()
def main_shutdown_callback(q: mp.Queue, *, cbtype: "ShutdownCallback"): async def main(): # Inform the test caller that the main coro is ready q.put_nowait("ok") await asyncio.sleep(10.0) # Inform the test caller that the fut was unblocked successfully. q.put_nowait(True) if cbtype is CallbackType.FUNCTION: def shutdown_callback(loop): q.put_nowait(True) elif cbtype is CallbackType.ASYNC_DEF_FUNCTION: async def shutdown_callback(loop): q.put_nowait(True) elif cbtype is CallbackType.COROUTINE_OBJECT: async def shutdown_callback_fn(loop): q.put_nowait(True) shutdown_callback = shutdown_callback_fn(None) else: raise TypeError('Unexpected cbtype') run(main(), shutdown_callback=shutdown_callback) q.put(None) q.join()
def main_sigterm_enduring_indirect_cancel(q: mp.Queue): async def corofn(): q.put_nowait("ok") await asyncio.sleep(0.2) q.put_nowait(True) def direct_cancel(): """Reach inside, find the one task that is marked "do not cancel" for shutdown, and then cancel it directly. This should raise CancelledError at the location of the caller for `shutdown_waits_for()`.""" tasks = all_tasks() for t in tasks: # pragma: no cover if t._coro in _DO_NOT_CANCEL_COROS: t.cancel() return async def main(): loop = asyncio.get_event_loop() coro = corofn() loop.call_later(0.1, direct_cancel) try: await shutdown_waits_for(coro) except asyncio.CancelledError: q.put_nowait("got cancellation as expected") else: q.put_nowait("no cancellation raised") run(main()) q.put(None) q.join()
def ip(): num_threads = 10 q = Queue() def pingme(i, queue): while True: ip = queue.get() ret = subprocess.call('ping -c 1 %s' % i, shell=True, stdout=open('e:\\git\\ip.txt', 'w'), stderr=subprocess.STDOUT) if ret == 0: print('%s-%s is up!' % i) elif ret == 1: print('%s is down...' % i) queue.task_done() for i in range(num_threads): t = Thread(target=pingme, args=(i, q)) # 多线程调用 t.setDaemon(True) # 设置守护线程 t.start() info = Monitor.query.all() for i in info: q.put(i.IpAddr) # 上传列表 q.join() print('完成') return 'ok'
def main_shutdown_callback_error_and_main_error(q: mp.Queue): import logging log_messages = [] def filt(record): log_messages.append(record.getMessage()) return record logging.getLogger("aiorun").addFilter(filt) async def main(): await asyncio.sleep(1e-3) raise Exception("main") def shutdown_callback(loop): raise Exception("blah") try: run(main(), stop_on_unhandled_errors=True, shutdown_callback=shutdown_callback) except Exception as e: q.put_nowait(e) else: q.put_nowait("exception was not raised") finally: q.put_nowait(log_messages) q.put_nowait(None) q.join()
def test_looker(): print('Прослушивание почты...') params = config() spreadsheetId = params.get('spreadsheetid', None) GOOGLE_CREDENTIALS_FILE = params.get('credential_file', None) GOOGLE_CREDENTIALS_DATA = get_obj_json_from_file(GOOGLE_CREDENTIALS_FILE) print("type=", type(GOOGLE_CREDENTIALS_DATA)) google_sheets_creadential_json = GOOGLE_CREDENTIALS_DATA # while True: # init_looker_multythread(spreadsheetId=spreadsheetId, # google_sheets_creadential_json=GOOGLE_CREDENTIALS_DATA, # imap_server='imap.yandex.ru', # email='*****@*****.**', # passwd='kpwoltwjqpsboxde', # folder='test2') imap_server = 'imap.yandex.ru' email = '*****@*****.**' passwd = 'alpglyneshancphh' folder = 'test2' queue = Queue() IMAP4_server = imaplib.IMAP4_SSL(imap_server) IMAP4_server.login(email, passwd) look = SilentLooker(queue, folder, IMAP4_server) # Запускаем потом и очередь for i in range(5): t = SSWriter(queue, spreadsheetId, google_sheets_creadential_json) t.setDaemon(True) t.start() look.run() # Ждем завершения работы очереди queue.join()
def execute(self, item_generator, item_processor): # Create a queue to communicate with the worker threads try: queue = Queue() threads = [] # Create 8 worker threads for x in range(self.no_of_threads): worker = Neo4jUploadWorker(queue, item_processor) # Setting daemon to True will let the main thread exit even though the workers are blocking worker.daemon = True worker.start() threads.append(worker) # Put the tasks into the queue as a tuple for item in item_generator.generate: queue.put(item) # Causes the main thread to wait for the queue to finish processing all the tasks queue.join() # stop workers for i in range(self.no_of_threads): queue.put(None) for t in threads: t.join() finally: print("In the finally") item_generator.close() for i in range(self.no_of_threads): queue.put(None) for t in threads: t.join()
def vulnerabilities_by_date(self, year): url = F"https://www.cvedetails.com/vulnerability-list/year-{year}/vulnerabilities.html" res = requests.get(url, headers=self.headers) html = etree.HTML(res.content) #获得漏洞页面的链接漏洞总数: total_vuln = html.xpath('//*[@id="pagingb"]/b/text()') link = html.xpath('//*[@id="pagingb"]/a/@href') page_link = ["https://www.cvedetails.com" + i for i in link] #创建表格 cur = self.conn.cursor() #cve_id, cve_type, cve_score, cve_authority, cve_vendor, cve_produce, cve_produce_version sql = F"CREATE TABLE IF NOT EXISTS cve{year}(cve_id TEXT PRIMARY KEY, type TEXT,score TEXT, authority TEXT, vendor TEXT, produce TEXT, produce_version TEXT)" cur.execute(sql) #设置两个队列 url_queue = Queue(maxsize=self.thread_num * 3) cve_info_queue = Queue(maxsize=self.thread_num * 3) #生成cve详情url producer_thread = Process(target=self.producer, args=(url_queue, page_link)) producer_thread.daemon = True producer_thread.start() #处理cve详情页面 for index in range(self.thread_num): consumer_thread = Process(target=self.cve_data, args=( url_queue, cve_info_queue, )) consumer_thread.daemon = True consumer_thread.start() #将cve信息存储到表格之中 excel_thread = Process(target=self.write_sql, args=( cve_info_queue, cur, year, )) excel_thread.daemon = True excel_thread.start() #控制线程进度,确定能够生产完毕 producer_thread.join() url_queue.join() # print(url_queue.qsize()) cve_info_queue.join() # print(cve_info_queue.qsize()) self.conn.commit() # self.conn.close() print(F"{year}年cve信息全部写入成功")
def main_sig_pause(q: mp.Queue): async def main(): try: q.put_nowait("ok") await asyncio.sleep(5.0) except asyncio.CancelledError: print("in cancellation handler") await asyncio.sleep(0.1) run(main()) q.put("done") q.put(None) q.join()
class ThreadPool: """Pool of threads consuming tasks from a queue""" def __init__(self, num_threads): self.tasks = Queue(num_threads) for _ in range(num_threads): Worker(self.tasks) def add_task(self, func, *args, **kargs): """Add a task to the queue""" self.tasks.put((func, args, kargs)) def wait_completion(self): """Wait for completion of all the tasks in the queue""" self.tasks.join()
def main(links, output_dir): # Create a queue to communicate with the worker threads queue = Queue() # Create 5 worker threads for x in range(5): worker = Worker(queue, output_dir) # Setting daemon to True will let the main thread exit even though the workers are blocking worker.setDaemon(True) worker.start() # Put the tasks into the queue as a tuple for link in links: queue.put(link) # Causes the main thread to wait for the queue to finish processing all the tasks queue.join()
def main_shutdown_callback_error(q: mp.Queue): async def main(): await asyncio.sleep(1e-3) def shutdown_callback(loop): raise Exception("blah") try: run(main(), shutdown_callback=shutdown_callback) except Exception as e: q.put_nowait(e) else: q.put_nowait("exception was not raised") finally: q.put_nowait(None) q.join()
def main(): manager = Manager() d = manager.dict() que=Queue(10000) p = Process(recv_items(que)) p.daemon = True p.start() pool = Pool(100) for i in range(1,100): pool.apply_async(handle,args=(que,i,d)) p.join() que.join()
def main(q: mp.Queue, **kwargs): async def main(): q.put("ok") await asyncio.sleep(5.0) if kwargs.pop("use_exe", None): exe = ThreadPoolExecutor() else: exe = None loop = None if kwargs.pop("user_supplied_loop", None) and not kwargs.get("use_uvloop"): loop = newloop() try: run(main(), executor=exe, loop=loop, **kwargs) finally: q.put(None) q.join()
def main_shutdown_callback(q: mp.Queue): fut = None async def _main(): nonlocal fut q.put_nowait("ok") fut = asyncio.Future() await fut q.put_nowait(True) async def main(): await shutdown_waits_for(_main()) def shutdown_callback(loop): fut.set_result(None) run(main(), shutdown_callback=shutdown_callback) q.put(None) q.join()
def main_sigterm_enduring_create_task(q: mp.Queue, spawn_method): async def corofn(): q.put_nowait("ok") await asyncio.sleep(0.05) q.put_nowait(True) async def main(): loop = asyncio.get_event_loop() if spawn_method == "create_task": loop.create_task(shutdown_waits_for(corofn())) elif spawn_method == "ensure_future": asyncio.ensure_future(shutdown_waits_for(corofn())) elif spawn_method == "await": await shutdown_waits_for(corofn()) elif spawn_method == "bare": shutdown_waits_for(corofn()) run(main()) q.put(None) q.join()
def main(sysargv=[]): TLIM = 50 #the maximum number of concurrent threads MLIM = 1000.0 #the memory limit (in GB) if len(sysargv) > 0: TLIM = int(sysargv[0]) if len(sysargv) > 1: MLIM = float(sysargv[1]) #imprecise safety precautions TLIM = TLIM - 1 MLIM = int(MLIM * (2**20) * float(TLIM) / (TLIM + 1)) #in kB t_arr = [] #array of threads q = Queue() #queue of commands lock = Lock() ################################################# ##GENERATE FILELIST HERE filelist = [] ################################################# #put each command on the queue for i, f in enumerate(filelist): #i is the counter, f is the filename ################################################# ##GENERATE CMD HERE cmd = "echo \"stuff\"" ################################################# q.put(cmd) #create TLIM workers to process the queue for i in range(TLIM): t_arr.append(threading.Thread(target=worker, args=( q, lock, ))) t_arr[i].daemon = True t_arr[i].start() q.join()
def main(): q = Queue(120000) database = load_database(DATABASE) producer = Thread(target=generate, args=(q, "weakpass_2a")) producer.start() consumers = [] for i in range(4): name = 'Consumer-{}'.format(i) consumer = Process(target=check, args=(name, q, database)) consumer.start() consumers.append(consumer) producer.join() for consumer in consumers: consumer.join() q.join()
def init_looker_multythread(spreadsheetId, google_sheets_creadential_json, imap_server, email, passwd, folder): IMAP4_server = imaplib.IMAP4_SSL(imap_server) IMAP4_server.login(email, passwd) emails_info = get_data_email_message(folder, IMAP4_server) """ Запускаем программу """ queue = Queue() # Запускаем очередь for i in range(5): t = SSWriter(queue, spreadsheetId, google_sheets_creadential_json) t.setDaemon(True) t.start() # Добавляем в очередь письмо for email in emails_info: queue.put(email) # Ждем завершения работы очереди queue.join()
def runClient(self): try: send_msg = Queue() recive_msg = Queue() sending = Process(target=self.send, args=(send_msg)) reciveing = Process(target=self.receive, arg=(recive_msg)) sending.start() reciveing.start() while True: reciveing.join() command = recive_msg.get() # Arduino Parts Start_point = time.time() self.arduino.Value_to_T_data(self.position.BF_desire, self.position.LR_desire) self.arduinoSerial_write() self.arduinoSerial_read() if self.arduino.R_data != "": print(self.arduino.R_data) send_msg.put(self.arduino.R_data) send_msg.join() End_point = time.time() time.sleep(self.arduino.Loop_time - (End_point - Start_point)) reciveing.run() send_msg.run() except KeyboardInterrupt: print("Emergency stop!!!!") self.sock.close() reciveing.close() send_msg.close() finally: print("OFF") self.sock.close() reciveing.close() send_msg.close()
class PlasmaShmQueue: def __init__(self, maxsize: int = 0): r"""Use pyarrow in-memory plasma store to implement shared memory queue. Compared to native `multiprocess.Queue`, `PlasmaShmQueue` avoid pickle/unpickle and communication overhead, leading to better performance in multi-process application. :type maxsize: int :param maxsize: maximum size of the queue, `None` means no limit. (default: ``None``) """ self.socket_name = MGE_PLASMA_STORE_MANAGER.socket_name # TODO: how to catch the exception happened in `plasma.connect`? self.client = None # Used to store the header for the data.(ObjectIDs) self.queue = Queue(maxsize) # type: Queue def put(self, data, block=True, timeout=None): if self.client is None: self.client = plasma.connect(self.socket_name) try: object_id = self.client.put(data) except plasma.PlasmaStoreFull: raise RuntimeError("plasma store out of memory!") try: self.queue.put(object_id, block, timeout) except queue.Full: self.client.delete([object_id]) raise queue.Full def get(self, block=True, timeout=None): if self.client is None: self.client = plasma.connect(self.socket_name) object_id = self.queue.get(block, timeout) if not self.client.contains(object_id): raise RuntimeError( "ObjectID: {} not found in plasma store".format(object_id) ) data = self.client.get(object_id) self.client.delete([object_id]) return data def qsize(self): return self.queue.qsize() def empty(self): return self.queue.empty() def join(self): self.queue.join() def disconnect_client(self): if self.client is not None: self.client.disconnect() def close(self): self.queue.close() self.disconnect_client() def cancel_join_thread(self): self.queue.cancel_join_thread()
c1=Process(target=consumer,args=('alex',q)) p1.start() c1.start() \由消费者发通知,说它确实收到了数据。使用JoinableQueue方法 #JoinableQueue([maxsize]):这就像是一个Queue对象,但队列允许项目的使用者通知生成者项目已经被成功处理。通知进程是使用共享的信号和条件变量来实现的。 #参数介绍: maxsize是队列中允许最大项数,省略则无大小限制。 #方法介绍: JoinableQueue的实例p除了与Queue对象相同的方法之外还具有: q.task_done(): 使用者使用此方法发出信号,表示q.get()的返回项目已经被处理。如果调用此方法的次数大于从队列中删除项目的数量,将引发ValueError异常 q.join(): 生产者调用此方法进行阻塞,直到队列中所有的项目均被处理。阻塞将持续到队列中的每个项目均调用q.task_done()方法为止 from multiprocessing import Process,Queue,JoinableQueue import time import random def producer(name,food,q): for i in range(3): res='%s%s' %(food,i) time.sleep(random.randint(1,3)) q.put(res) print('%s 生产了 %s' %(name,res)) def consumer(name,q): while True: res=q.get()
def exampleJob(worker): time.sleep(0.5) with print_lock : print(threading.current_thread().name,worker) def threader(): while True : worker = q.get() exampleJob(worker) q.task_done() q = Queue() for x in range(10) : #10 workers t = threading.Thread(target = threader) t.daemon = True t.start() start = time.time() for worker in range(20) : #20 jobs q.put(worker) q.join() print("Entire job took : ", time.time()-start)
def netService(a, cmd): while True: data = listen() print(data[1].decode("utf-8")[0]) if data[1].decode("utf-8")[0] == 'd': dry = ['Not dry', 'Dry', 'Too dry'] send('{"Temperature":%d,"Humidity":%d,"Dry Status":"%s","Light":%d}' % (a[0], a[1], dry[a[2]], a[3]), data) else: send("a", data) def add(l, cmd): for i in range(4): l.append(i) if __name__ == "__main__": q = Queue() m = Manager() cmd='1' l = m.list() p = Process(target=netService, args=(l, cmd)) q = Process(target=add, args=(l, cmd)) p.start() q.start() p.join() q.join()
#esponse = client.upload(, is_dir=True) # send to this node else: end_pointer = current_photo_pointer+photos_per_node if end_pointer > num_photos: end_pointer = num_photos photolist = photo_list[current_photo_pointer:end_pointer] current_photo_pointer+=photos_per_node print(str(eachnode) + " : " + str(photolist)) job_map = {'title': 'upload_image', 'client': node_client[each_node], 'list': photo_list, 'filepath': images_filepath+'/images'} job_queue.put(job_map) # send regular number of photos job_queue.join() # extract # client = lib.FileClient('localhost:8080') # # demo for file uploading # in_file_name = 'IMG_2331.JPG' # response = client.upload('dataset/images/images/**', is_dir=True) # print("response" + str(response)) #execute a task on the server
if __name__ == '__main__': try: os.mkdir("image") except: pass q = Queue(100) q_for_v = Queue(50) app = QtWidgets.QApplication(sys.argv) ui = Ui_MainWindow(q, q_for_v) p1 = Process(target=producer, args=(q, q_for_v)) c = Process(target=video_load, args=(q_for_v, r'./gopro8(1)_best.mp4', 0)) c1 = Process(target=video_load_1, args=(q_for_v, r'./gopro8(2)_best.mp4', 1)) ui.show() p1.start() c.start() c1.start() sys.exit(app.exec_()) p1.join() c.join() c1.join() q_for_v.join() q.join() # q_for_v.close() # q.close()
class DeployManager(object): def __init__(self, args): self._args = args self.awsutils = AwsUtils(access_key=args.key, secret_key=args.secret, region=args.region) self.task_queue = Queue() self.cluster_list = self.awsutils.list_clusters() self.threads_count = args.threads_count self.service_wait_max_attempts = args.service_wait_max_attempts self.service_wait_delay = args.service_wait_delay self.key = args.key self.secret = args.secret self.region = args.region self.error = False # 削除対象 self.delete_service_list = [] # 全サービス self.all_service_list = [] # デプロイ対象の全サービス self.all_deploy_target_service_list = [] # 優先付きのstopBeforeDeployサービス self.primary_stop_before_deploy_service_list = [] # stopBeforeDeployサービス self.stop_before_deploy_service_list = [] # 優先付きのサービス self.primary_deploy_service_list = [] # それ以外 self.remain_deploy_service_list = [] self.delete_scheduled_task_list = [] self.scheduled_task_list = [] self.environment = None self.template_group = None self.is_service_zero_keep = True self.is_stop_before_deploy = True self.is_delete_unused_service = True self.force = False def _service_config(self): self.all_service_list,\ self.all_deploy_target_service_list,\ self.scheduled_task_list,\ self.deploy_scheduled_task_list,\ self.environment = get_deploy_list( services_yaml=self._args.services_yaml, environment_yaml=self._args.environment_yaml, task_definition_template_dir=self._args.task_definition_template_dir, task_definition_config_json=self._args.task_definition_config_json, task_definition_config_env=self._args.task_definition_config_env, deploy_service_group=self._args.deploy_service_group, template_group=self._args.template_group ) # thread数がタスクの数を超えているなら減らす deploy_size = len(self.deploy_scheduled_task_list) + len(self.all_deploy_target_service_list) if deploy_size < self.threads_count: self.threads_count = deploy_size self.is_service_zero_keep = self._args.service_zero_keep self.template_group = self._args.template_group self.is_delete_unused_service = self._args.delete_unused_service self.is_stop_before_deploy = self._args.stop_before_deploy def _set_deploy_list(self): for service in self.all_deploy_target_service_list: if self.is_stop_before_deploy and service.stop_before_deploy: if service.is_primary_placement: self.primary_stop_before_deploy_service_list.append(service) else: self.stop_before_deploy_service_list.append(service) else: if service.is_primary_placement: self.primary_deploy_service_list.append(service) else: self.remain_deploy_service_list.append(service) def _unstopped_primary_stop_before_deploy_service_list(self) -> list: return [x for x in self.primary_stop_before_deploy_service_list if x.origin_desired_count > 0] def _unstopped_stop_before_deploy_service_list(self) -> list: return [x for x in self.stop_before_deploy_service_list if x.origin_desired_count > 0] def _start_threads(self): # threadの開始 for i in range(self.threads_count): thread = DeployProcess( task_queue=self.task_queue, key=self.key, secret=self.secret, region=self.region, is_service_zero_keep=self.is_service_zero_keep, is_stop_before_deploy=self.is_stop_before_deploy, service_wait_max_attempts=self.service_wait_max_attempts, service_wait_delay=self.service_wait_delay ) thread.setDaemon(True) thread.start() def run(self): self._service_config() self._start_threads() self._fetch_ecs_information() # Step: Delete Unused Service self._delete_unused() self._check_deploy() self._set_deploy_list() self._stop_scheduled_task() self._stop_before_deploy() self._deploy_service() self._start_after_deploy() self._deploy_scheduled_task() self._result_check() def dry_run(self): self._service_config() self._start_threads() self._fetch_ecs_information() # Step: Check Delete Service self._delete_unused(dry_run=True) # Step: Check Service self._check_deploy() self._set_deploy_list() def delete(self): self.environment = self._args.environment self.force = self._args.force self._start_threads() self._fetch_ecs_information(is_all=True) if self.delete_service_list == 0 and self.delete_scheduled_task_list == 0: info("No delete service or scheduled task") return h1("Delete Service or Scheduled Task") for service in self.delete_service_list: print("* %s" % service.service_name) for task in self.delete_scheduled_task_list: print("* %s" % task.family) if not self.force: reply = input("\nWould you like delete all ecs service in %s (y/n)\n" % self.environment) if reply != 'y': return self._delete_unused() def _stop_scheduled_task(self): if len(self.deploy_scheduled_task_list) > 0: h1("Step: Stop ECS Scheduled Task") for task in self.deploy_scheduled_task_list: self.task_queue.put([task, ProcessMode.stopScheduledTask]) self.task_queue.join() def _stop_before_deploy(self): if len(self._unstopped_primary_stop_before_deploy_service_list()) > 0 \ or len(self._unstopped_stop_before_deploy_service_list()) > 0: h1("Step: Stop ECS Service Before Deploy") for service in self._unstopped_primary_stop_before_deploy_service_list(): self.task_queue.put([service, ProcessMode.stopBeforeDeploy]) for service in self._unstopped_stop_before_deploy_service_list(): self.task_queue.put([service, ProcessMode.stopBeforeDeploy]) self.task_queue.join() h2("Wait for Service Status 'Stable'") self._wait_for_stable(self._unstopped_primary_stop_before_deploy_service_list()) self._wait_for_stable(self._unstopped_stop_before_deploy_service_list()) def _start_after_deploy(self): if len(self.primary_stop_before_deploy_service_list) > 0: h1("Step: Start Primary ECS Service After Deploy") for service in self.primary_stop_before_deploy_service_list: self.task_queue.put([service, ProcessMode.deployService]) self.task_queue.join() h2("Wait for Service Status 'Stable'") self._wait_for_stable(self.primary_stop_before_deploy_service_list) if len(self.stop_before_deploy_service_list) > 0: h1("Step: Start ECS Service After Deploy") for service in self.stop_before_deploy_service_list: self.task_queue.put([service, ProcessMode.deployService]) self.task_queue.join() h2("Wait for Service Status 'Stable'") self._wait_for_stable(self.stop_before_deploy_service_list) def _deploy_scheduled_task(self): if len(self.deploy_scheduled_task_list) > 0: h1("Step: Deploy ECS Scheduled Task") for task in self.deploy_scheduled_task_list: self.task_queue.put([task, ProcessMode.deployScheduledTask]) self.task_queue.join() def _delete_unused(self, dry_run=False): if dry_run: h1("Step: Check Delete Unused") else: h1("Step: Delete Unused") if not self.is_delete_unused_service: info("Do not delete unused") return if len(self.delete_service_list) == 0 and len(self.delete_scheduled_task_list) == 0: info("There was no service or task to delete.") for delete_service in self.delete_service_list: success("Delete service '{delete_service.service_name}'".format(delete_service=delete_service)) if not dry_run: self.awsutils.delete_service(delete_service.cluster_name, delete_service.service_name) for delete_scheduled_task in self.delete_scheduled_task_list: success("Delete scheduled task '{delete_scheduled_task.name}'" .format(delete_scheduled_task=delete_scheduled_task)) if not dry_run: self.awsutils.delete_scheduled_task( name=delete_scheduled_task.name, target_arn=delete_scheduled_task.task_environment.target_lambda_arn ) def _fetch_ecs_information(self, is_all=False): h1("Step: Fetch ECS Information") describe_service_list = [] if len(self.all_service_list) > 0 or is_all: describe_service_list = ecs.service.fetch_aws_service( cluster_list=self.cluster_list, awsutils=self.awsutils ) for s in describe_service_list: self.task_queue.put([s, ProcessMode.fetchServices]) cloud_watch_rule_list = [] if len(self.scheduled_task_list) > 0 or is_all: rules = self.awsutils.list_cloudwatch_event_rules() for r in rules: if r.get('Description') == scheduled_task_managed_description: c = CloudwatchEventRule(r) cloud_watch_rule_list.append(c) self.task_queue.put([c, ProcessMode.fetchCloudwatchEvents]) while self.task_queue.qsize() > 0: print('.', end='', flush=True) time.sleep(3) self.task_queue.join() info("") # set service description and get delete servicelist for describe_service in describe_service_list: if self.environment != describe_service.task_environment.environment: continue if self.template_group is not None: if self.template_group != describe_service.task_environment.template_group: continue is_delete = True for service in self.all_service_list: if service.service_name == describe_service.service_name: if service.task_environment.cluster_name == describe_service.cluster_name: service.set_from_describe_service(describe_service=describe_service) is_delete = False break if is_delete: self.delete_service_list.append(describe_service) for cloud_watch_rule in cloud_watch_rule_list: if self.environment != cloud_watch_rule.task_environment.environment: continue if self.template_group is not None: if self.template_group != cloud_watch_rule.task_environment.template_group: continue is_delete = True for scheduled_task in self.scheduled_task_list: if scheduled_task.family == cloud_watch_rule.family: scheduled_task.set_from_cloudwatch_event_rule(cloud_watch_rule) is_delete = False break if is_delete: self.delete_scheduled_task_list.append(cloud_watch_rule) success("Check succeeded") def _deploy_service(self): if len(self.primary_deploy_service_list) > 0: h1("Step: Deploy Primary ECS Service") for service in self.primary_deploy_service_list: self.task_queue.put([service, ProcessMode.deployService]) self.task_queue.join() h2("Wait for Service Status 'Stable'") self._wait_for_stable(self.primary_deploy_service_list) if len(self.remain_deploy_service_list) > 0: h1("Step: Deploy ECS Service") for service in self.remain_deploy_service_list: self.task_queue.put([service, ProcessMode.deployService]) self.task_queue.join() h2("Wait for Service Status 'Stable'") self._wait_for_stable(self.remain_deploy_service_list) def _check_deploy(self): h1("Step: Check Deploy ECS Service and Scheduled tasks") for service in self.all_deploy_target_service_list: self.task_queue.put([service, ProcessMode.checkDeployService]) for scheduled_task in self.deploy_scheduled_task_list: self.task_queue.put([scheduled_task, ProcessMode.checkDeployScheduledTask]) self.task_queue.join() def _wait_for_stable(self, service_list: list): if len(service_list) > 0: for service in service_list: self.task_queue.put([service, ProcessMode.waitForStable]) self.task_queue.join() def _result_check(self): error_service_list = list(filter( lambda service: service.status == ProcessStatus.error, self.all_deploy_target_service_list )) error_scheduled_task_list = list(filter( lambda task: task.status == ProcessStatus.error, self.deploy_scheduled_task_list )) # エラーが一個でもあれば失敗としておく if len(error_service_list) > 0 or len(error_scheduled_task_list) > 0: sys.exit(1) if self.error: sys.exit(1)
def run_inputs(self, inputs, print_progress=True): """ Process SCAPS run parameters in parallel. Takes in a dictionary of inputs, structured as {'id1':run_params_1, 'id2':run_params_2, ...} where run_params_1, run_params_2, ... should be the argument to the pre-specified input processor method. Returns a dictionary structured as {'id1':output_1, 'id2':output_2, ...} where output_1, output_2, ... are the objectrs returned by the pre-specified output processor method """ inq = Queue() outq = Queue() output_dict = {} num_total = len(inputs.keys()) num_done = 0 proc_list = [] config_all = {'SCAPS_ROOT':self.SCAPS_ROOT, 'SCAPS_CMD':self.SCAPS_CMD, 'SCAPS_EXEC_DIR':self.scaps_exec_dir, 'INPUT_PROC':self.input_processor, 'OUTPUT_PROC':self.output_processor} for proc_i in range(self.ncores): config_proc = deepcopy(config_all) config_proc['CORE'] = proc_i proc = Process(target=SCAPSrunner.run_process, args=(config_proc, inq, outq)) proc.start() proc_list.append(proc) inputiter = inputs.iteritems() while True: running = any(proc.is_alive() for proc in proc_list) if not running: break while inq.empty(): try: (id, input) = inputiter.next() inq.put({'id':id,'calc_param':input}) except: inq.put({'id':'done'}) while not outq.empty(): pt = outq.get() output_dict[pt['id']] = pt['output'] num_done += 1 if print_progress: print("Finished input ID{} [{}/{} total]".format(pt['id'], num_done, num_total)) for proc_i, proc in enumerate(proc_list): proc.join() # Garbage collect while not inq.empty(): inq.get() while not outq.empty(): outq.join() # Give queues time to close time.sleep(5) if not (set(inputs.keys())==set(output_dict.keys())): print("Warning: Not all inputs seem to have gotten outputs") return output_dict
def main_no_coro(q: mp.Queue): run() q.put(None) q.join()
import queue # 不能进行多进程之间的数据传输 (1)from multiprocessing import Queue 借助Queue解决生产者消费者模型 队列是安全的。 q = Queue(num) num : 队列的最大长度 q.get()# 阻塞等待获取数据,如果有数据直接获取,如果没有数据,阻塞等待 q.put()# 阻塞,如果可以继续往队列中放数据,就直接放,不能放就阻塞等待 q.get_nowait()# 不阻塞,如果有数据直接获取,没有数据就报错 q.put_nowait()# 不阻塞,如果可以继续往队列中放数据,就直接放,不能放就报错 (2)from multiprocessing import JoinableQueue#可连接的队列 JoinableQueue是继承Queue,所以可以使用Queue中的方法 并且JoinableQueue又多了两个方法 q.join()# 用于生产者。等待 q.task_done的返回结果,通过返回结果,生产者就能获得消费者当前消费了多少个数据 q.task_done() # 用于消费者,是指每消费队列中一个数据,就给join返回一个标识。 2 管道(了解) from multiprocessing import Pipe con1,con2 = Pipe() 管道是不安全的。 管道是用于多进程之间通信的一种方式。 如果在单进程中使用管道,那么就是con1收数据,就是con2发数据。 如果是con1发数据,就是con2收数据 如果在多进程中使用管道,那么就必须是父进程使用con1收,子进程就必须使用con2发 父进程使用con1发,子进程就必须使用con2收 父进程使用con2收,子进程就必须使用con1发 父进程使用con2发,子进程就必须使用con1收 在管道中有一个著名的错误叫做EOFError。是指,父进程中如果关闭了发送端,子进程还继续接收数据,那么就会引发EOFError。
class PlasmaShmQueue: def __init__(self, maxsize: int = 0): r""" Use pyarrow in-memory plasma store to implement shared memory queue. Compared to native `multiprocess.Queue`, `PlasmaShmQueue` avoid pickle/unpickle and communication overhead, leading to better performance in multi-process application. :type maxsize: int :param maxsize: maximum size of the queue, `None` means no limit. (default: ``None``) """ # Lazy start the plasma store manager global MGE_PLASMA_STORE_MANAGER if MGE_PLASMA_STORE_MANAGER is None: try: MGE_PLASMA_STORE_MANAGER = _PlasmaStoreManager() except Exception as e: err_info = ( "Please make sure pyarrow installed correctly!\n" "You can try reinstall pyarrow and see if you can run " "`plasma_store -s /tmp/mge_plasma_xxx -m 1000` normally.") raise RuntimeError( "Exception happened in starting plasma_store: {}\n" "Tips: {}".format(str(e), err_info)) else: MGE_PLASMA_STORE_MANAGER.refcount += 1 self.socket_name = MGE_PLASMA_STORE_MANAGER.socket_name # TODO: how to catch the exception happened in `plasma.connect`? self.client = None # Used to store the header for the data.(ObjectIDs) self.queue = Queue(maxsize) # type: Queue def put(self, data, block=True, timeout=None): if self.client is None: self.client = plasma.connect(self.socket_name) try: object_id = self.client.put(data) except plasma.PlasmaStoreFull: raise RuntimeError("plasma store out of memory!") try: self.queue.put(object_id, block, timeout) except queue.Full: self.client.delete([object_id]) raise queue.Full def get(self, block=True, timeout=None): if self.client is None: self.client = plasma.connect(self.socket_name) object_id = self.queue.get(block, timeout) if not self.client.contains(object_id): raise RuntimeError( "ObjectID: {} not found in plasma store".format(object_id)) data = self.client.get(object_id) self.client.delete([object_id]) return data def qsize(self): return self.queue.qsize() def empty(self): return self.queue.empty() def join(self): self.queue.join() def disconnect_client(self): if self.client is not None: self.client.disconnect() def close(self): self.queue.close() self.disconnect_client() global MGE_PLASMA_STORE_MANAGER MGE_PLASMA_STORE_MANAGER.refcount -= 1 _clear_plasma_store() def cancel_join_thread(self): self.queue.cancel_join_thread()
while True: time.sleep(6) #self.test.print_self() for sss in self.test.ooos: print('keep print {}'.format(sss.status)) if __name__ == '__main__': current = test('aaa', 'bbb', 'ccc') qqq = Queue(10) for aa in current.ooos: qqq.put(aa) #qqq.put(None) threads = [] gg = keep_print(current) gg.start() for i in range(3): tt = update_thread(qqq) tt.start() threads.append(tt) [ts.join() for ts in threads] #gg.join() qqq.join() print("finished")